Renamed packages com.metaweb.* to com.google.*.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1130 7d457c2a-affb-35e4-300a-418c747d4874

broker/appengine/.classpath

@@ -3,12 +3,13 @@
 	<classpathentry kind="src" path="src"/>
 	<classpathentry kind="con" path="com.google.appengine.eclipse.core.GAE_CONTAINER"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
-	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk.jar"/>	
+	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk.jar"/>
 	<classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/httpcore-4.0.1.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/httpcore-4.0.1-sources.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/slf4j-api-1.5.6.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/slf4j-api-1.5.6-sources.jar"/>
 	<classpathentry combineaccessrules="false" kind="src" path="/gridworks-broker"/>
+	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/rhino-1.7R2.jar"/>
 	<classpathentry kind="output" path="module/MOD-INF/classes"/>
 </classpath>

broker/appengine/.settings/com.google.appengine.eclipse.core.prefs

@@ -1,3 +1,3 @@
-#Wed May 26 15:13:15 PDT 2010
+#Tue Aug 03 15:54:32 PDT 2010
 eclipse.preferences.version=1
-validationExclusions=src/com/metaweb/gridworks/appengine/*ClientConnection*.java
+validationExclusions=src/com/google/gridworks/appengine/*ClientConnection*.java

broker/appengine/module/MOD-INF/module.properties

@@ -1,4 +1,4 @@
 name = broker
 description = Google App Engine implementation of Gridworks Broker
-module-impl = com.metaweb.gridworks.broker.AppEngineGridworksBrokerImpl
+module-impl = com.google.gridworks.broker.AppEngineGridworksBrokerImpl
 templating = false

broker/appengine/src/com/google/gridworks/appengine/AppEngineClientConnection.java

@@ -0,0 +1,243 @@
+package com.google.gridworks.appengine;
+
+import static com.google.appengine.api.urlfetch.FetchOptions.Builder.allowTruncate;
+
+import java.io.ByteArrayOutputStream;
+import java.net.InetAddress;
+import java.net.URL;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.SSLSession;
+
+import org.apache.http.Header;
+import org.apache.http.HttpConnectionMetrics;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.conn.ManagedClientConnection;
+import org.apache.http.conn.routing.HttpRoute;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.message.BasicHttpResponse;
+import org.apache.http.params.HttpParams;
+import org.apache.http.protocol.HttpContext;
+
+import com.google.appengine.api.urlfetch.HTTPHeader;
+import com.google.appengine.api.urlfetch.HTTPMethod;
+import com.google.appengine.api.urlfetch.HTTPRequest;
+import com.google.appengine.api.urlfetch.HTTPResponse;
+import com.google.appengine.api.urlfetch.URLFetchService;
+import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
+
+class AppEngineClientConnection implements ManagedClientConnection {
+    // Managed is the composition of ConnectionReleaseTrigger,
+    //     HttpClientConnection, HttpConnection, HttpInetConnection
+
+    private HttpRoute _route;
+    private Object _state;
+    private boolean _reuseable;
+
+    public AppEngineClientConnection(HttpRoute route, Object state) {
+        _route = route;
+        _state = state;
+    }
+
+    // ManagedClientConnection methods
+
+    public HttpRoute getRoute() {
+        return _route;
+    }
+
+    public Object getState() {
+        return _state;
+    }
+
+    public SSLSession getSSLSession() {
+        return null;
+    }
+
+    public boolean isSecure() {
+        // XXX maybe parse the url to see if it's https?
+        return false;
+    }
+
+    public boolean isMarkedReusable() {
+        return _reuseable;
+    }
+
+    public void markReusable() {
+        _reuseable = true;
+    }
+
+    public void layerProtocol(HttpContext context, HttpParams params) {
+        return;
+    }
+
+    public void open(HttpRoute route, HttpContext context, HttpParams params) {
+        return;
+    }
+
+    public void setIdleDuration(long duration, TimeUnit unit) {
+        return;
+    }
+
+    public void setState(Object state) {
+        _state = state;
+    }
+
+    public void tunnelProxy(HttpHost next, boolean secure, HttpParams params) {
+        return;
+    }
+
+    public void tunnelTarget(boolean secure, HttpParams params) {
+        return;
+    }
+
+    public void unmarkReusable() {
+        _reuseable = false;
+    }
+
+
+    // ConnectionReleaseTrigger methods
+
+    public void releaseConnection() {
+        return;
+    }
+
+    public void abortConnection() {
+        return;
+    }
+
+    // HttpClientConnection methods
+
+    private HTTPRequest _appengine_hrequest;
+    private HTTPResponse _appengine_hresponse;
+
+    public void flush() {
+        return;
+    }
+
+    public boolean isResponseAvailable(int timeout) {
+        // XXX possibly use Async fetcher
+        return true;
+    }
+
+    public void receiveResponseEntity(org.apache.http.HttpResponse apache_response) {
+        byte[] data = _appengine_hresponse.getContent();
+
+        if (data != null) {
+            apache_response.setEntity(new ByteArrayEntity(data));
+        }
+    }
+
+    public HttpResponse receiveResponseHeader() {
+        URLFetchService ufs = URLFetchServiceFactory.getURLFetchService();
+        try {
+            _appengine_hresponse = ufs.fetch(_appengine_hrequest);
+        } catch (java.io.IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        org.apache.http.HttpResponse apache_response =
+            new BasicHttpResponse(new ProtocolVersion("HTTP", 1, 0),
+                                  _appengine_hresponse.getResponseCode(),
+                                  null);
+
+        for (HTTPHeader h : _appengine_hresponse.getHeaders()) {
+            apache_response.addHeader(h.getName(), h.getValue());
+        }
+
+        return apache_response;
+    }
+
+    public void sendRequestEntity(org.apache.http.HttpEntityEnclosingRequest request) {
+        ByteArrayOutputStream os = new ByteArrayOutputStream();
+
+        org.apache.http.HttpEntity ent = request.getEntity();
+        if (ent != null) {
+            try {
+                ent.writeTo(os);
+            } catch (java.io.IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        _appengine_hrequest.setPayload(os.toByteArray());
+    }
+
+    public void sendRequestHeader(org.apache.http.HttpRequest apache_request) {
+        URL request_url;
+
+        HttpHost host = _route.getTargetHost();
+
+        String protocol = host.getSchemeName();
+        String addr = host.getHostName();
+        int port = host.getPort();
+
+        String path = apache_request.getRequestLine().getUri();
+
+        try {
+            request_url = new URL(protocol, addr, port, path);
+        } catch (java.net.MalformedURLException e) {
+            throw new RuntimeException(e);
+        }
+
+        HTTPMethod method = HTTPMethod.valueOf(apache_request.getRequestLine().getMethod());
+        _appengine_hrequest = new HTTPRequest(request_url, method, allowTruncate()
+                                              .doNotFollowRedirects());
+
+        Header[] apache_headers = apache_request.getAllHeaders();
+        for (int i = 0; i < apache_headers.length; i++) {
+            Header h = apache_headers[i];
+            _appengine_hrequest
+                .setHeader(new HTTPHeader(h.getName(), h.getValue()));
+        }
+    }
+
+    // HttpConnection methods
+
+    public void close() {
+        return;
+    }
+
+    public HttpConnectionMetrics getMetrics() {
+        return null;
+    }
+
+    public int getSocketTimeout()  {
+        return -1;
+    }
+
+    public boolean isOpen() {
+        return true;
+    }
+
+    public boolean isStale() {
+        return false;
+    }
+
+    public void setSocketTimeout(int timeout) {
+        return;
+    }
+
+    public void shutdown() {
+        return;
+    }
+
+    // HttpInetConnection methods
+
+    public InetAddress getLocalAddress() {
+        return null;
+    }
+           
+    public int getLocalPort() {
+        return -1;
+    }
+           
+    public InetAddress getRemoteAddress() {
+        return null;
+    }
+           
+    public int getRemotePort() {
+        return -1;
+    }
+}

broker/appengine/src/com/google/gridworks/appengine/AppEngineClientConnectionManager.java

@@ -0,0 +1,76 @@
+package com.google.gridworks.appengine;
+
+import java.net.InetAddress;
+import java.net.Socket;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.conn.ClientConnectionRequest;
+import org.apache.http.conn.ManagedClientConnection;
+import org.apache.http.conn.routing.HttpRoute;
+import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.conn.scheme.SchemeRegistry;
+import org.apache.http.conn.scheme.SocketFactory;
+import org.apache.http.params.HttpParams;
+
+public class AppEngineClientConnectionManager implements ClientConnectionManager {
+
+    private SchemeRegistry schemes;
+
+    class NoopSocketFactory implements SocketFactory {
+        public Socket connectSocket(Socket sock, String host, int port, InetAddress addr, int lport, HttpParams params) {
+            return null;
+        }
+
+        public Socket createSocket() {
+            return null;
+        }
+
+        public boolean isSecure(Socket sock) {
+            return false;
+        }
+    }
+
+    public AppEngineClientConnectionManager() {
+        SocketFactory noop_sf = new NoopSocketFactory();
+        schemes = new SchemeRegistry();
+        schemes.register(new Scheme("http",  noop_sf, 80));
+        schemes.register(new Scheme("https", noop_sf, 443));
+    }
+
+    public void closeExpiredConnections() {
+        return;
+    }
+
+    public void closeIdleConnections(long idletime, TimeUnit tunit) {
+        return;
+    }
+
+    public ManagedClientConnection getConnection(HttpRoute route, Object state) {
+        return new AppEngineClientConnection(route, state);
+    }
+
+    public SchemeRegistry getSchemeRegistry() {
+        return schemes;
+    }
+
+    public void releaseConnection(ManagedClientConnection conn, long valid, TimeUnit tuint) {
+        return;
+    }
+
+    public ClientConnectionRequest requestConnection(final HttpRoute route, final Object state) {
+        return new ClientConnectionRequest() {
+            public void abortRequest() {
+                return;
+            }
+
+            public ManagedClientConnection getConnection(long idletime, TimeUnit tunit) {
+                return AppEngineClientConnectionManager.this.getConnection(route, state);
+            }
+        };
+    }
+
+    public void shutdown() {
+        return;
+    }
+}

broker/appengine/src/com/google/gridworks/broker/AppEngineGridworksBrokerImpl.java

@@ -0,0 +1,365 @@
+package com.google.gridworks.broker;
+
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.jdo.Extent;
+import javax.jdo.JDOHelper;
+import javax.jdo.PersistenceManager;
+import javax.jdo.PersistenceManagerFactory;
+import javax.jdo.Transaction;
+import javax.jdo.annotations.IdGeneratorStrategy;
+import javax.jdo.annotations.PersistenceCapable;
+import javax.jdo.annotations.Persistent;
+import javax.jdo.annotations.PrimaryKey;
+import javax.servlet.ServletConfig;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.http.client.HttpClient;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.appengine.api.datastore.Text;
+import com.google.gridworks.appengine.AppEngineClientConnectionManager;
+import com.google.gridworks.broker.GridworksBroker;
+
+public class AppEngineGridworksBrokerImpl extends GridworksBroker {
+                
+    protected static final Logger logger = LoggerFactory.getLogger("gridworks.broker.appengine");
+    
+    PersistenceManagerFactory pmfInstance;
+    
+    @Override
+    public void init(ServletConfig config) throws Exception {
+        super.init(config);
+        
+        pmfInstance = JDOHelper.getPersistenceManagerFactory("transactional");
+    }
+    
+    @Override
+    public void destroy() throws Exception {
+    }
+
+    // ---------------------------------------------------------------------------------
+
+    protected HttpClient getHttpClient() {
+        ClientConnectionManager cm = new AppEngineClientConnectionManager();
+        return new DefaultHttpClient(cm, null);
+    }
+    
+    // ---------------------------------------------------------------------------------
+    
+    protected void expireLocks(HttpServletResponse response) throws Exception {
+        
+        PersistenceManager pm = pmfInstance.getPersistenceManager();
+        
+        try {
+            Extent<Lock> extent = pm.getExtent(Lock.class, false);
+
+            try {
+                for (Lock lock : extent) {
+                    if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
+                        Transaction tx = pm.currentTransaction();
+                        try {
+                            tx.begin();
+                            pm.deletePersistent(lock);
+                            tx.commit();
+                        } finally {
+                            if (tx.isActive()) {
+                                tx.rollback();
+                            }
+                        }
+                    }
+                }
+            } finally {
+                extent.closeAll();                
+            }            
+            
+            respond(response, OK);
+            
+        } finally {
+            pm.close();
+        }
+    }
+    
+    protected void getState(HttpServletResponse response, String pid, String rev) throws Exception {
+        PersistenceManager pm = pmfInstance.getPersistenceManager();
+        
+        try {
+            // FIXME
+            respond(response, lockToJSON(getLock(pm,pid)));
+        } finally {
+            pm.close();
+        }
+    }
+
+    protected void obtainLock(HttpServletResponse response, String pid, String uid, String type) throws Exception {
+        PersistenceManager pm = pmfInstance.getPersistenceManager();
+        
+        // FIXME (use type)
+        
+        try {
+            Lock lock = getLock(pm, pid);
+            if (lock == null) {
+                Transaction tx = pm.currentTransaction();
+                
+                try {
+                    tx.begin();
+                    lock = new Lock(Long.toHexString(tx.hashCode()), pid, uid);
+                    pm.makePersistent(lock);
+                    tx.commit();
+                } finally {
+                    if (tx.isActive()) {
+                        tx.rollback();
+                    }
+                }
+            }
+            
+            respond(response, lockToJSON(lock));
+            
+        } finally {
+            pm.close();
+        }
+    }
+    
+    protected void releaseLock(HttpServletResponse response, String pid, String uid, String lid) throws Exception {
+
+        PersistenceManager pm = pmfInstance.getPersistenceManager();
+        
+        try {
+            Lock lock = getLock(pm, pid);
+            if (lock != null) {
+                if (!lock.id.equals(lid)) {
+                    throw new RuntimeException("Lock id doesn't match, can't release the lock");
+                }
+                if (!lock.uid.equals(uid)) {
+                    throw new RuntimeException("User id doesn't match the lock owner, can't release the lock");
+                }
+
+                Transaction tx = pm.currentTransaction();
+                
+                try {
+                    tx.begin();
+                    pm.deletePersistent(lock);
+                    tx.commit();
+                } finally {
+                    if (tx.isActive()) {
+                        tx.rollback();
+                    }
+                }
+            }
+            
+            respond(response, OK);
+            
+        } finally {
+            pm.close();
+        }
+    }
+    
+    // ----------------------------------------------------------------------------------------------------
+    
+    protected void startProject(HttpServletResponse response, String pid, String uid, String lid, String data) throws Exception {
+        PersistenceManager pm = pmfInstance.getPersistenceManager();
+        
+        try {
+            checkLock(pm, pid, uid, lid);
+            
+            Project project = getProject(pm, pid);
+            
+            if (project != null) {
+                throw new RuntimeException("Project '" + pid + "' already exists");
+            }
+            
+            Transaction tx = pm.currentTransaction();
+            
+            try {
+                tx.begin();
+                project = new Project(pid, data);
+                pm.makePersistent(project);
+                tx.commit();
+            } finally {
+                if (tx.isActive()) {
+                    tx.rollback();
+                }
+            }
+            
+            respond(response, OK);
+        } finally {
+            pm.close();
+        }
+    }
+
+    protected void addTransformations(HttpServletResponse response, String pid, String uid, String lid, List<String> transformations) throws Exception {
+        PersistenceManager pm = pmfInstance.getPersistenceManager();
+        
+        try {
+            checkLock(pm, pid, uid, lid);
+
+            Project project = getProject(pm, pid);
+            
+            if (project == null) {
+                throw new RuntimeException("Project '" + pid + "' not found");
+            }
+
+            Transaction tx = pm.currentTransaction();
+
+            try {
+                for (String s : transformations) {
+                    project.transformations.add(new Text(s));
+                }
+                tx.commit();
+            } finally {
+                if (tx.isActive()) {
+                    tx.rollback();
+                }
+            }
+            
+            respond(response, OK);
+        } finally {
+            pm.close();
+        }
+    }
+
+    // ---------------------------------------------------------------------------------
+    
+    protected void openProject(HttpServletResponse response, String pid) throws Exception {
+        PersistenceManager pm = pmfInstance.getPersistenceManager();
+        
+        try {
+            Project project = getProject(pm, pid);
+
+            Writer w = response.getWriter();
+            JSONWriter writer = new JSONWriter(w);
+            writer.object();
+                writer.key("data"); writer.value(project.data.toString());
+                writer.key("transformations"); 
+                writer.array();
+                    for (Text s : project.transformations) {
+                        writer.value(s.toString());
+                    }
+                writer.endArray();
+            writer.endObject();
+            w.flush();
+            w.close();
+        } finally {
+            pm.close();
+        }
+    }
+    
+    protected void getHistory(HttpServletResponse response, String pid, int tindex) throws Exception {
+        PersistenceManager pm = pmfInstance.getPersistenceManager();
+        
+        try {
+            Project project = getProject(pm, pid);
+
+            Writer w = response.getWriter();
+            JSONWriter writer = new JSONWriter(w);
+            writer.object();
+            writer.key("transformations"); 
+            writer.array();
+                int size = project.transformations.size();
+                for (int i = tindex; i < size; i++) {
+                    writer.value(project.transformations.get(i).toString());
+                }
+            writer.endArray();
+            writer.endObject();
+            w.flush();
+            w.close();
+        } finally {
+            pm.close();
+        }
+    }
+    
+    // ---------------------------------------------------------------------------------
+    
+    Project getProject(PersistenceManager pm, String pid) {
+        Project project = pm.getObjectById(Project.class, pid);
+        if (project == null) {
+            throw new RuntimeException("Project '" + pid + "' is not managed by this broker");
+        }
+        return project;
+    }
+        
+    @PersistenceCapable    
+    static class Project {
+        
+        @PrimaryKey
+        @Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
+        String pid;
+
+        @Persistent
+        List<Text> transformations = new ArrayList<Text>(); 
+
+        @Persistent
+        Text data;
+
+        Project(String pid, String data) {
+            this.pid = pid;
+            this.data = new Text(data);
+        }
+    }
+
+    // ---------------------------------------------------------------------------------
+    
+    Lock getLock(PersistenceManager pm, String pid) {
+        return pm.getObjectById(Lock.class, pid);
+    }
+
+    void checkLock(PersistenceManager pm, String pid, String uid, String lid) {
+        Lock lock = getLock(pm, pid);
+    
+        if (lock == null) {
+            throw new RuntimeException("No lock was found with the given Lock id '" + lid + "', you have to have a valid lock on a project in order to start it");
+        }
+        
+        if (!lock.pid.equals(pid)) {
+            throw new RuntimeException("Lock '" + lid + "' is for another project: " + pid);
+        }
+        
+        if (!lock.uid.equals(uid)) {
+            throw new RuntimeException("Lock '" + lid + "' is owned by another user: " + uid);
+        }
+    }
+    
+    JSONObject lockToJSON(Lock lock) throws JSONException {
+        JSONObject o = new JSONObject();
+        if (lock != null) {
+            o.put("lock_id", lock.id);
+            o.put("project_id", lock.pid);
+            o.put("user_id", lock.uid);
+            o.put("timestamp", lock.timestamp);
+        }
+        return o;
+    }
+    
+    @PersistenceCapable    
+    static class Lock {
+
+        @Persistent
+        String id;
+        
+        @PrimaryKey
+        @Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
+        String pid;
+        
+        @Persistent
+        String uid;
+        
+        @Persistent
+        long timestamp;
+        
+        Lock(String id, String pid, String uid) {
+            this.id = id;
+            this.pid = pid;
+            this.uid = uid;
+            this.timestamp = System.currentTimeMillis();
+        }
+    }
+    
+}

broker/appengine/src/com/metaweb/gridworks/appengine/AppEngineClientConnection.java

@@ -1,243 +0,0 @@
-package com.metaweb.gridworks.appengine;
-
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.allowTruncate;
-
-import java.io.ByteArrayOutputStream;
-import java.net.InetAddress;
-import java.net.URL;
-import java.util.concurrent.TimeUnit;
-
-import javax.net.ssl.SSLSession;
-
-import org.apache.http.Header;
-import org.apache.http.HttpConnectionMetrics;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.ProtocolVersion;
-import org.apache.http.conn.ManagedClientConnection;
-import org.apache.http.conn.routing.HttpRoute;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.message.BasicHttpResponse;
-import org.apache.http.params.HttpParams;
-import org.apache.http.protocol.HttpContext;
-
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPMethod;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
-
-class AppEngineClientConnection implements ManagedClientConnection {
-    // Managed is the composition of ConnectionReleaseTrigger,
-    //     HttpClientConnection, HttpConnection, HttpInetConnection
-
-    private HttpRoute _route;
-    private Object _state;
-    private boolean _reuseable;
-
-    public AppEngineClientConnection(HttpRoute route, Object state) {
-        _route = route;
-        _state = state;
-    }
-
-    // ManagedClientConnection methods
-
-    public HttpRoute getRoute() {
-        return _route;
-    }
-
-    public Object getState() {
-        return _state;
-    }
-
-    public SSLSession getSSLSession() {
-        return null;
-    }
-
-    public boolean isSecure() {
-        // XXX maybe parse the url to see if it's https?
-        return false;
-    }
-
-    public boolean isMarkedReusable() {
-        return _reuseable;
-    }
-
-    public void markReusable() {
-        _reuseable = true;
-    }
-
-    public void layerProtocol(HttpContext context, HttpParams params) {
-        return;
-    }
-
-    public void open(HttpRoute route, HttpContext context, HttpParams params) {
-        return;
-    }
-
-    public void setIdleDuration(long duration, TimeUnit unit) {
-        return;
-    }
-
-    public void setState(Object state) {
-        _state = state;
-    }
-
-    public void tunnelProxy(HttpHost next, boolean secure, HttpParams params) {
-        return;
-    }
-
-    public void tunnelTarget(boolean secure, HttpParams params) {
-        return;
-    }
-
-    public void unmarkReusable() {
-        _reuseable = false;
-    }
-
-
-    // ConnectionReleaseTrigger methods
-
-    public void releaseConnection() {
-        return;
-    }
-
-    public void abortConnection() {
-        return;
-    }
-
-    // HttpClientConnection methods
-
-    private HTTPRequest _appengine_hrequest;
-    private HTTPResponse _appengine_hresponse;
-
-    public void flush() {
-        return;
-    }
-
-    public boolean isResponseAvailable(int timeout) {
-        // XXX possibly use Async fetcher
-        return true;
-    }
-
-    public void receiveResponseEntity(org.apache.http.HttpResponse apache_response) {
-        byte[] data = _appengine_hresponse.getContent();
-
-        if (data != null) {
-            apache_response.setEntity(new ByteArrayEntity(data));
-        }
-    }
-
-    public HttpResponse receiveResponseHeader() {
-        URLFetchService ufs = URLFetchServiceFactory.getURLFetchService();
-        try {
-            _appengine_hresponse = ufs.fetch(_appengine_hrequest);
-        } catch (java.io.IOException e) {
-            throw new RuntimeException(e);
-        }
-
-        org.apache.http.HttpResponse apache_response =
-            new BasicHttpResponse(new ProtocolVersion("HTTP", 1, 0),
-                                  _appengine_hresponse.getResponseCode(),
-                                  null);
-
-        for (HTTPHeader h : _appengine_hresponse.getHeaders()) {
-            apache_response.addHeader(h.getName(), h.getValue());
-        }
-
-        return apache_response;
-    }
-
-    public void sendRequestEntity(org.apache.http.HttpEntityEnclosingRequest request) {
-        ByteArrayOutputStream os = new ByteArrayOutputStream();
-
-        org.apache.http.HttpEntity ent = request.getEntity();
-        if (ent != null) {
-            try {
-                ent.writeTo(os);
-            } catch (java.io.IOException e) {
-                throw new RuntimeException(e);
-            }
-        }
-
-        _appengine_hrequest.setPayload(os.toByteArray());
-    }
-
-    public void sendRequestHeader(org.apache.http.HttpRequest apache_request) {
-        URL request_url;
-
-        HttpHost host = _route.getTargetHost();
-
-        String protocol = host.getSchemeName();
-        String addr = host.getHostName();
-        int port = host.getPort();
-
-        String path = apache_request.getRequestLine().getUri();
-
-        try {
-            request_url = new URL(protocol, addr, port, path);
-        } catch (java.net.MalformedURLException e) {
-            throw new RuntimeException(e);
-        }
-
-        HTTPMethod method = HTTPMethod.valueOf(apache_request.getRequestLine().getMethod());
-        _appengine_hrequest = new HTTPRequest(request_url, method, allowTruncate()
-                                              .doNotFollowRedirects());
-
-        Header[] apache_headers = apache_request.getAllHeaders();
-        for (int i = 0; i < apache_headers.length; i++) {
-            Header h = apache_headers[i];
-            _appengine_hrequest
-                .setHeader(new HTTPHeader(h.getName(), h.getValue()));
-        }
-    }
-
-    // HttpConnection methods
-
-    public void close() {
-        return;
-    }
-
-    public HttpConnectionMetrics getMetrics() {
-        return null;
-    }
-
-    public int getSocketTimeout()  {
-        return -1;
-    }
-
-    public boolean isOpen() {
-        return true;
-    }
-
-    public boolean isStale() {
-        return false;
-    }
-
-    public void setSocketTimeout(int timeout) {
-        return;
-    }
-
-    public void shutdown() {
-        return;
-    }
-
-    // HttpInetConnection methods
-
-    public InetAddress getLocalAddress() {
-        return null;
-    }
-           
-    public int getLocalPort() {
-        return -1;
-    }
-           
-    public InetAddress getRemoteAddress() {
-        return null;
-    }
-           
-    public int getRemotePort() {
-        return -1;
-    }
-}

broker/appengine/src/com/metaweb/gridworks/appengine/AppEngineClientConnectionManager.java

@@ -1,76 +0,0 @@
-package com.metaweb.gridworks.appengine;
-
-import java.net.InetAddress;
-import java.net.Socket;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.http.conn.ClientConnectionManager;
-import org.apache.http.conn.ClientConnectionRequest;
-import org.apache.http.conn.ManagedClientConnection;
-import org.apache.http.conn.routing.HttpRoute;
-import org.apache.http.conn.scheme.Scheme;
-import org.apache.http.conn.scheme.SchemeRegistry;
-import org.apache.http.conn.scheme.SocketFactory;
-import org.apache.http.params.HttpParams;
-
-public class AppEngineClientConnectionManager implements ClientConnectionManager {
-
-    private SchemeRegistry schemes;
-
-    class NoopSocketFactory implements SocketFactory {
-        public Socket connectSocket(Socket sock, String host, int port, InetAddress addr, int lport, HttpParams params) {
-            return null;
-        }
-
-        public Socket createSocket() {
-            return null;
-        }
-
-        public boolean isSecure(Socket sock) {
-            return false;
-        }
-    }
-
-    public AppEngineClientConnectionManager() {
-        SocketFactory noop_sf = new NoopSocketFactory();
-        schemes = new SchemeRegistry();
-        schemes.register(new Scheme("http",  noop_sf, 80));
-        schemes.register(new Scheme("https", noop_sf, 443));
-    }
-
-    public void closeExpiredConnections() {
-        return;
-    }
-
-    public void closeIdleConnections(long idletime, TimeUnit tunit) {
-        return;
-    }
-
-    public ManagedClientConnection getConnection(HttpRoute route, Object state) {
-        return new AppEngineClientConnection(route, state);
-    }
-
-    public SchemeRegistry getSchemeRegistry() {
-        return schemes;
-    }
-
-    public void releaseConnection(ManagedClientConnection conn, long valid, TimeUnit tuint) {
-        return;
-    }
-
-    public ClientConnectionRequest requestConnection(final HttpRoute route, final Object state) {
-        return new ClientConnectionRequest() {
-            public void abortRequest() {
-                return;
-            }
-
-            public ManagedClientConnection getConnection(long idletime, TimeUnit tunit) {
-                return AppEngineClientConnectionManager.this.getConnection(route, state);
-            }
-        };
-    }
-
-    public void shutdown() {
-        return;
-    }
-}

broker/appengine/src/com/metaweb/gridworks/broker/AppEngineGridworksBrokerImpl.java

@@ -1,364 +0,0 @@
-package com.metaweb.gridworks.broker;
-
-import java.io.Writer;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.jdo.Extent;
-import javax.jdo.JDOHelper;
-import javax.jdo.PersistenceManager;
-import javax.jdo.PersistenceManagerFactory;
-import javax.jdo.Transaction;
-import javax.jdo.annotations.IdGeneratorStrategy;
-import javax.jdo.annotations.PersistenceCapable;
-import javax.jdo.annotations.Persistent;
-import javax.jdo.annotations.PrimaryKey;
-import javax.servlet.ServletConfig;
-import javax.servlet.http.HttpServletResponse;
-
-import org.apache.http.client.HttpClient;
-import org.apache.http.conn.ClientConnectionManager;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.appengine.api.datastore.Text;
-import com.metaweb.gridworks.appengine.AppEngineClientConnectionManager;
-
-public class AppEngineGridworksBrokerImpl extends GridworksBroker {
-                
-    protected static final Logger logger = LoggerFactory.getLogger("gridworks.broker.appengine");
-    
-    PersistenceManagerFactory pmfInstance;
-    
-    @Override
-    public void init(ServletConfig config) throws Exception {
-        super.init(config);
-        
-        pmfInstance = JDOHelper.getPersistenceManagerFactory("transactional");
-    }
-    
-    @Override
-    public void destroy() throws Exception {
-    }
-
-    // ---------------------------------------------------------------------------------
-
-    protected HttpClient getHttpClient() {
-        ClientConnectionManager cm = new AppEngineClientConnectionManager();
-        return new DefaultHttpClient(cm, null);
-    }
-    
-    // ---------------------------------------------------------------------------------
-    
-    protected void expireLocks(HttpServletResponse response) throws Exception {
-        
-        PersistenceManager pm = pmfInstance.getPersistenceManager();
-        
-        try {
-            Extent<Lock> extent = pm.getExtent(Lock.class, false);
-
-            try {
-                for (Lock lock : extent) {
-                    if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
-                        Transaction tx = pm.currentTransaction();
-                        try {
-                            tx.begin();
-                            pm.deletePersistent(lock);
-                            tx.commit();
-                        } finally {
-                            if (tx.isActive()) {
-                                tx.rollback();
-                            }
-                        }
-                    }
-                }
-            } finally {
-                extent.closeAll();                
-            }            
-            
-            respond(response, OK);
-            
-        } finally {
-            pm.close();
-        }
-    }
-    
-    protected void getState(HttpServletResponse response, String pid, String rev) throws Exception {
-        PersistenceManager pm = pmfInstance.getPersistenceManager();
-        
-        try {
-            // FIXME
-            respond(response, lockToJSON(getLock(pm,pid)));
-        } finally {
-            pm.close();
-        }
-    }
-
-    protected void obtainLock(HttpServletResponse response, String pid, String uid, String type) throws Exception {
-        PersistenceManager pm = pmfInstance.getPersistenceManager();
-        
-        // FIXME (use type)
-        
-        try {
-            Lock lock = getLock(pm, pid);
-            if (lock == null) {
-                Transaction tx = pm.currentTransaction();
-                
-                try {
-                    tx.begin();
-                    lock = new Lock(Long.toHexString(tx.hashCode()), pid, uid);
-                    pm.makePersistent(lock);
-                    tx.commit();
-                } finally {
-                    if (tx.isActive()) {
-                        tx.rollback();
-                    }
-                }
-            }
-            
-            respond(response, lockToJSON(lock));
-            
-        } finally {
-            pm.close();
-        }
-    }
-    
-    protected void releaseLock(HttpServletResponse response, String pid, String uid, String lid) throws Exception {
-
-        PersistenceManager pm = pmfInstance.getPersistenceManager();
-        
-        try {
-            Lock lock = getLock(pm, pid);
-            if (lock != null) {
-                if (!lock.id.equals(lid)) {
-                    throw new RuntimeException("Lock id doesn't match, can't release the lock");
-                }
-                if (!lock.uid.equals(uid)) {
-                    throw new RuntimeException("User id doesn't match the lock owner, can't release the lock");
-                }
-
-                Transaction tx = pm.currentTransaction();
-                
-                try {
-                    tx.begin();
-                    pm.deletePersistent(lock);
-                    tx.commit();
-                } finally {
-                    if (tx.isActive()) {
-                        tx.rollback();
-                    }
-                }
-            }
-            
-            respond(response, OK);
-            
-        } finally {
-            pm.close();
-        }
-    }
-    
-    // ----------------------------------------------------------------------------------------------------
-    
-    protected void startProject(HttpServletResponse response, String pid, String uid, String lid, String data) throws Exception {
-        PersistenceManager pm = pmfInstance.getPersistenceManager();
-        
-        try {
-            checkLock(pm, pid, uid, lid);
-            
-            Project project = getProject(pm, pid);
-            
-            if (project != null) {
-                throw new RuntimeException("Project '" + pid + "' already exists");
-            }
-            
-            Transaction tx = pm.currentTransaction();
-            
-            try {
-                tx.begin();
-                project = new Project(pid, data);
-                pm.makePersistent(project);
-                tx.commit();
-            } finally {
-                if (tx.isActive()) {
-                    tx.rollback();
-                }
-            }
-            
-            respond(response, OK);
-        } finally {
-            pm.close();
-        }
-    }
-
-    protected void addTransformations(HttpServletResponse response, String pid, String uid, String lid, List<String> transformations) throws Exception {
-        PersistenceManager pm = pmfInstance.getPersistenceManager();
-        
-        try {
-            checkLock(pm, pid, uid, lid);
-
-            Project project = getProject(pm, pid);
-            
-            if (project == null) {
-                throw new RuntimeException("Project '" + pid + "' not found");
-            }
-
-            Transaction tx = pm.currentTransaction();
-
-            try {
-                for (String s : transformations) {
-                    project.transformations.add(new Text(s));
-                }
-                tx.commit();
-            } finally {
-                if (tx.isActive()) {
-                    tx.rollback();
-                }
-            }
-            
-            respond(response, OK);
-        } finally {
-            pm.close();
-        }
-    }
-
-    // ---------------------------------------------------------------------------------
-    
-    protected void openProject(HttpServletResponse response, String pid) throws Exception {
-        PersistenceManager pm = pmfInstance.getPersistenceManager();
-        
-        try {
-            Project project = getProject(pm, pid);
-
-            Writer w = response.getWriter();
-            JSONWriter writer = new JSONWriter(w);
-            writer.object();
-                writer.key("data"); writer.value(project.data.toString());
-                writer.key("transformations"); 
-                writer.array();
-                    for (Text s : project.transformations) {
-                        writer.value(s.toString());
-                    }
-                writer.endArray();
-            writer.endObject();
-            w.flush();
-            w.close();
-        } finally {
-            pm.close();
-        }
-    }
-    
-    protected void getHistory(HttpServletResponse response, String pid, int tindex) throws Exception {
-        PersistenceManager pm = pmfInstance.getPersistenceManager();
-        
-        try {
-            Project project = getProject(pm, pid);
-
-            Writer w = response.getWriter();
-            JSONWriter writer = new JSONWriter(w);
-            writer.object();
-            writer.key("transformations"); 
-            writer.array();
-                int size = project.transformations.size();
-                for (int i = tindex; i < size; i++) {
-                    writer.value(project.transformations.get(i).toString());
-                }
-            writer.endArray();
-            writer.endObject();
-            w.flush();
-            w.close();
-        } finally {
-            pm.close();
-        }
-    }
-    
-    // ---------------------------------------------------------------------------------
-    
-    Project getProject(PersistenceManager pm, String pid) {
-        Project project = pm.getObjectById(Project.class, pid);
-        if (project == null) {
-            throw new RuntimeException("Project '" + pid + "' is not managed by this broker");
-        }
-        return project;
-    }
-        
-    @PersistenceCapable    
-    static class Project {
-        
-        @PrimaryKey
-        @Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
-        String pid;
-
-        @Persistent
-        List<Text> transformations = new ArrayList<Text>(); 
-
-        @Persistent
-        Text data;
-
-        Project(String pid, String data) {
-            this.pid = pid;
-            this.data = new Text(data);
-        }
-    }
-
-    // ---------------------------------------------------------------------------------
-    
-    Lock getLock(PersistenceManager pm, String pid) {
-        return pm.getObjectById(Lock.class, pid);
-    }
-
-    void checkLock(PersistenceManager pm, String pid, String uid, String lid) {
-        Lock lock = getLock(pm, pid);
-    
-        if (lock == null) {
-            throw new RuntimeException("No lock was found with the given Lock id '" + lid + "', you have to have a valid lock on a project in order to start it");
-        }
-        
-        if (!lock.pid.equals(pid)) {
-            throw new RuntimeException("Lock '" + lid + "' is for another project: " + pid);
-        }
-        
-        if (!lock.uid.equals(uid)) {
-            throw new RuntimeException("Lock '" + lid + "' is owned by another user: " + uid);
-        }
-    }
-    
-    JSONObject lockToJSON(Lock lock) throws JSONException {
-        JSONObject o = new JSONObject();
-        if (lock != null) {
-            o.put("lock_id", lock.id);
-            o.put("project_id", lock.pid);
-            o.put("user_id", lock.uid);
-            o.put("timestamp", lock.timestamp);
-        }
-        return o;
-    }
-    
-    @PersistenceCapable    
-    static class Lock {
-
-        @Persistent
-        String id;
-        
-        @PrimaryKey
-        @Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
-        String pid;
-        
-        @Persistent
-        String uid;
-        
-        @Persistent
-        long timestamp;
-        
-        Lock(String id, String pid, String uid) {
-            this.id = id;
-            this.pid = pid;
-            this.uid = uid;
-            this.timestamp = System.currentTimeMillis();
-        }
-    }
-    
-}

broker/core/.classpath

@@ -14,10 +14,10 @@
 	<classpathentry kind="lib" path="/gridworks/tests/server/lib/testng-5.12.1.jar" sourcepath="/gridworks/tests/server/lib-src/testng-5.12.1-sources.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/log4j-1.2.15.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/log4j-1.2.15-sources.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/slf4j-log4j12-1.5.6.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/slf4j-log4j12-1.5.6-sources.jar"/>
-	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/velocity-1.5.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/rhino-1.7R2.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/lessen-trunk-r8.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/commons-collections-3.2.1.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/jcl-over-slf4j-1.5.6.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/jcl-over-slf4j-1.5.6-sources.jar"/>
+	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/velocity-1.5.jar"/>
 	<classpathentry kind="output" path="module/MOD-INF/classes"/>
 </classpath>

broker/core/IDEs/eclipse/GridworksBrokerTests.launch

@@ -7,10 +7,10 @@
 <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.testng.remote.RemoteTestNG"/>
 <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="gridworks-broker"/>
 <mapAttribute key="org.testng.eclipse.ALL_CLASS_METHODS">
-<mapEntry key="com.metaweb.gridworks.broker.tests.GridworksBrokerTests" value=""/>
+<mapEntry key="com.google.gridworks.broker.tests.GridworksBrokerTests" value=""/>
 </mapAttribute>
 <listAttribute key="org.testng.eclipse.CLASS_TEST_LIST">
-<listEntry value="com.metaweb.gridworks.broker.tests.GridworksBrokerTests"/>
+<listEntry value="com.google.gridworks.broker.tests.GridworksBrokerTests"/>
 </listAttribute>
 <stringAttribute key="org.testng.eclipse.COMPLIANCE_LEVEL" value="JDK"/>
 <listAttribute key="org.testng.eclipse.GROUP_LIST"/>

broker/core/module/MOD-INF/module.properties

@@ -1,4 +1,4 @@
 name = broker
 description = Local Gridworks Broker
-module-impl = com.metaweb.gridworks.broker.GridworksBrokerImpl
+module-impl = com.google.gridworks.broker.GridworksBrokerImpl
 templating = false

broker/core/src/com/google/gridworks/broker/GridworksBroker.java

@@ -0,0 +1,311 @@
+
+package com.google.gridworks.broker;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.http.NameValuePair;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.BasicResponseHandler;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.params.CoreProtocolPNames;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import edu.mit.simile.butterfly.ButterflyModuleImpl;
+
+/**
+ * This class contains all the code shared by various implementations of a Gridworks Broker.
+ * 
+ * A broker is a server used by multiple Gridworks installations to enable collaborative
+ * development over the same project.
+ * 
+ * Broker implementations differ in how they store their state but all of them are required
+ * to extend this abstract class and implement the services that are called via HTTP.
+ * 
+ */
+public abstract class GridworksBroker extends ButterflyModuleImpl {
+    
+    static final public String GET_STATE = "get_state";
+    static final public String EXPIRE = "expire";
+    static final public String OBTAIN_LOCK = "obtain_lock";
+    static final public String RELEASE_LOCK = "release_lock";
+    static final public String TRANSFORM = "transform";
+    static final public String START = "start";
+    static final public String OPEN = "open";
+
+    static final public int ALL = 0;
+    static final public int COL = 1;
+    static final public int CELL = 2;
+    
+    static final protected Logger logger = LoggerFactory.getLogger("gridworks.broker");
+    
+    static final protected String USER_INFO_URL = "http://www.freebase.com/api/service/user_info";
+    static final protected String DELEGATED_OAUTH_HEADER = "X-Freebase-Credentials";
+    static final protected String OAUTH_HEADER = "Authorization";
+
+    static protected String OK; 
+    
+    static {
+        try {
+            JSONObject o = new JSONObject();
+            o.put("status","ok");
+            OK = o.toString();
+        } catch (JSONException e) {
+            // not going to happen;
+        }
+    }
+    
+    static public final long LOCK_DURATION = 60 * 1000; // 1 minute
+    static public final long USER_DURATION = 5 * 60 * 1000; // 1 minute
+    static public final long LOCK_EXPIRATION_CHECK_DELAY = 5 * 1000; // 5 seconds
+    
+    protected HttpClient httpclient;
+
+    protected boolean developmentMode;
+    
+    @Override
+    public void init(ServletConfig config) throws Exception {
+        super.init(config);
+        httpclient = getHttpClient();
+        developmentMode = Boolean.parseBoolean(config.getInitParameter("gridworks.development"));
+        if (developmentMode) logger.warn("Running in development mode");
+    }
+
+    @Override
+    public void destroy() throws Exception {
+        httpclient.getConnectionManager().shutdown();       
+    }
+    
+    @Override
+    public boolean process(String path, HttpServletRequest request, HttpServletResponse response) throws Exception {
+        if (logger.isDebugEnabled()) {
+            logger.debug("> process '{}'", path);
+        } else {
+            logger.info("process '{}'", path);
+        }
+
+        try {
+
+            if (GET_STATE.equals(path)) {
+                response.setCharacterEncoding("UTF-8");
+                response.setHeader("Content-Type", "application/json");
+                getState(response, getParameter(request, "pid"), getUserId(request), getInteger(request, "rev"));
+            } else if (EXPIRE.equals(path)) {
+                response.setCharacterEncoding("UTF-8");
+                response.setHeader("Content-Type", "application/json");
+                expire(response);
+            } else if (OBTAIN_LOCK.equals(path)) {
+                response.setCharacterEncoding("UTF-8");
+                response.setHeader("Content-Type", "application/json");
+                obtainLock(response, getParameter(request, "pid"), getUserId(request), getInteger(request, "locktype"), getParameter(request, "lockvalue"));
+            } else if (RELEASE_LOCK.equals(path)) {
+                response.setCharacterEncoding("UTF-8");
+                response.setHeader("Content-Type", "application/json");
+                releaseLock(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"));
+            } else if (TRANSFORM.equals(path)) {
+                response.setCharacterEncoding("UTF-8");
+                response.setHeader("Content-Type", "application/json");
+                addTransformations(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"), getList(request, "transformations"));
+            } else if (START.equals(path)) {
+                response.setCharacterEncoding("UTF-8");
+                response.setHeader("Content-Type", "application/json");
+                startProject(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"), getData(request), getParameter(request, "metadata"), getList(request, "transformations"));
+            } else if (OPEN.equals(path)) {
+                response.setCharacterEncoding("UTF-8");
+                response.setHeader("Content-Type", "application/json");
+                openProject(response, getParameter(request, "pid"));
+            } else {
+                boolean value = super.process(path, request, response);
+                if (logger.isDebugEnabled()) logger.debug("< process '{}'", path);
+                return value;
+            }
+                
+        } catch (RuntimeException e) {
+            logger.error("runtime error", e.getMessage());
+            respondError(response, e.getMessage());
+        } catch (Exception e) {
+            logger.error("internal error", e);
+            respondException(response, e);
+        }
+        
+        if (logger.isDebugEnabled()) logger.debug("< process '{}'", path);
+        
+        return true;
+    }
+    
+    // ----------------------------------------------------------------------------------------
+    
+    protected abstract HttpClient getHttpClient();
+
+    protected abstract void expire(HttpServletResponse response) throws Exception;
+
+    protected abstract void getState(HttpServletResponse response, String pid, String uid, int rev) throws Exception;
+
+    protected abstract void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception;
+    
+    protected abstract void releaseLock(HttpServletResponse response, String pid, String uid, String lock) throws Exception;
+
+    protected abstract void startProject(HttpServletResponse response, String pid, String uid, String lock, byte[] data, String metadata, List<String> transformations) throws Exception;
+
+    protected abstract void addTransformations(HttpServletResponse response, String pid, String uid, String lock, List<String> transformations) throws Exception;
+    
+    protected abstract void openProject(HttpServletResponse response, String pid) throws Exception;
+    
+    // ----------------------------------------------------------------------------------------
+    
+    @SuppressWarnings("unchecked")
+    protected String getUserId(HttpServletRequest request) throws Exception {
+
+        // This is useful for testing
+        if (developmentMode) {
+            return getParameter(request, "uid");
+        }
+        
+        String oauth = request.getHeader(DELEGATED_OAUTH_HEADER);
+        if (oauth == null) {
+            throw new RuntimeException("The request needs to contain the '" + DELEGATED_OAUTH_HEADER + "' header set to obtain user identity via Freebase.");
+        }
+        
+        List<NameValuePair> formparams = new ArrayList<NameValuePair>();
+        Map<String,String> params = (Map<String,String>) request.getParameterMap();
+        for (Entry<String,String> e : params.entrySet()) {
+            formparams.add(new BasicNameValuePair((String) e.getKey(), (String) e.getValue()));
+        }
+        UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
+
+        HttpPost httpRequest = new HttpPost(USER_INFO_URL);
+        httpRequest.setHeader(OAUTH_HEADER, oauth);
+        httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks Broker");
+        httpRequest.setEntity(entity);
+                
+        ResponseHandler<String> responseHandler = new BasicResponseHandler();
+        String responseBody = httpclient.execute(httpRequest, responseHandler);
+        JSONObject o = new JSONObject(responseBody);
+        
+        return o.getString("username");
+    }
+
+    // ----------------------------------------------------------------------------------------
+
+    static protected String getParameter(HttpServletRequest request, String name) throws ServletException {
+        String param = request.getParameter(name);
+        if (param == null) {
+            throw new RuntimeException("request must come with a '" + name + "' parameter");
+        }
+        return param;
+    }
+    
+    static protected List<String> getList(HttpServletRequest request, String name) throws ServletException, JSONException {
+        String param = getParameter(request, name);
+        JSONArray a = new JSONArray(param);
+        List<String> result = new ArrayList<String>(a.length());
+        for (int i = 0; i < a.length(); i++) {
+            result.add(a.getString(i));
+        }
+        return result;
+    }
+    
+    static protected int getInteger(HttpServletRequest request, String name) throws ServletException, JSONException {
+        return Integer.parseInt(getParameter(request, name));
+    }
+    
+    static protected byte[] getData(HttpServletRequest request) throws ServletException, IOException {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        InputStream input = request.getInputStream();
+        byte[] buffer = new byte[4096];
+        int count = 0;
+        int n = 0;
+        while (-1 != (n = input.read(buffer))) {
+            output.write(buffer, 0, n);
+            count += n;
+        }
+        return output.toByteArray();
+        
+    }
+    
+    static protected void respondError(HttpServletResponse response, String error) throws IOException, ServletException {
+
+        if (response == null) {
+            throw new ServletException("Response object can't be null");
+        }
+        
+        try {
+            JSONObject o = new JSONObject();
+            o.put("status", "error");
+            o.put("message", error);
+            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+            respond(response, o.toString());
+        } catch (JSONException e) {
+            e.printStackTrace(response.getWriter());
+        }
+    }
+    
+    static protected void respondException(HttpServletResponse response, Exception e) throws IOException, ServletException {
+
+        if (response == null) {
+            throw new ServletException("Response object can't be null");
+        }
+        
+        try {
+            JSONObject o = new JSONObject();
+            o.put("status", "error");
+            o.put("message", e.getMessage());
+        
+            StringWriter sw = new StringWriter();
+            PrintWriter pw = new PrintWriter(sw);
+            e.printStackTrace(pw);
+            pw.flush();
+            sw.flush();
+        
+            o.put("stack", sw.toString());
+        
+            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+            respond(response, o.toString());
+        } catch (JSONException e1) {
+            e.printStackTrace(response.getWriter());
+        }
+    }
+
+    static protected void respond(HttpServletResponse response, JSONObject content) throws IOException, ServletException {
+        if (content == null) {
+            throw new ServletException("Content object can't be null");
+        }
+        
+        respond(response, content.toString());
+    }
+
+    static protected void respond(HttpServletResponse response, String content) throws IOException, ServletException {
+        if (response == null) {
+            throw new ServletException("Response object can't be null");
+        }
+        
+        Writer w = response.getWriter();
+        if (w != null) {
+            w.write(content);
+            w.flush();
+            w.close();
+        } else {
+            throw new ServletException("response returned a null writer");
+        }
+    }
+}

broker/core/src/com/google/gridworks/broker/GridworksBrokerImpl.java

@@ -0,0 +1,592 @@
+package com.google.gridworks.broker;
+
+import static com.sleepycat.persist.model.Relationship.MANY_TO_ONE;
+
+import java.io.File;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.sleepycat.je.Environment;
+import com.sleepycat.je.EnvironmentConfig;
+import com.sleepycat.je.Transaction;
+import com.sleepycat.persist.EntityCursor;
+import com.sleepycat.persist.EntityStore;
+import com.sleepycat.persist.PrimaryIndex;
+import com.sleepycat.persist.SecondaryIndex;
+import com.sleepycat.persist.StoreConfig;
+import com.sleepycat.persist.model.Entity;
+import com.sleepycat.persist.model.PrimaryKey;
+import com.sleepycat.persist.model.SecondaryKey;
+
+public class GridworksBrokerImpl extends GridworksBroker {
+                
+    protected static final Logger logger = LoggerFactory.getLogger("gridworks.broker.local");
+    
+    Environment env;
+    
+    EntityStore projectStore;
+    EntityStore lockStore;
+    EntityStore userStore;
+    
+    PrimaryIndex<String,Project> projectById;
+    PrimaryIndex<String,Lock> lockById;
+
+    SecondaryIndex<String,String,Lock> locksByProject;
+    
+    Timer timer;
+    Expirer expirer;
+
+    @Override
+    public void init(ServletConfig config) throws Exception {
+        logger.trace("> init");
+        super.init(config);
+
+        timer = new Timer();
+        expirer = new Expirer();
+        timer.schedule(expirer, 0, LOCK_EXPIRATION_CHECK_DELAY);
+        
+        String dataDir = config.getInitParameter("gridworks.data");
+        if (dataDir == null) dataDir = "data";
+        File dataPath = new File(dataDir);
+        if (!dataPath.exists()) dataPath.mkdirs();
+        
+        EnvironmentConfig envConfig = new EnvironmentConfig();
+        envConfig.setAllowCreate(true);
+        envConfig.setTransactional(true);
+        env = new Environment(dataPath, envConfig); 
+
+        StoreConfig storeConfig = new StoreConfig();
+        storeConfig.setAllowCreate(true);
+        storeConfig.setTransactional(true);
+        projectStore = new EntityStore(env, "ProjectsStore", storeConfig);
+        lockStore = new EntityStore(env, "LockStore", storeConfig);
+        
+        projectById = projectStore.getPrimaryIndex(String.class, Project.class);
+        lockById = lockStore.getPrimaryIndex(String.class, Lock.class);
+        
+        locksByProject = lockStore.getSecondaryIndex(lockById, String.class, "pid"); 
+        logger.trace("< init");
+    }
+    
+    @Override
+    public void destroy() throws Exception {
+        logger.trace("> destroy");
+        super.destroy();
+        
+        if (projectStore != null) {
+            projectStore.close();
+            projectById = null;            
+        } 
+
+        if (lockStore != null) {
+            lockStore.close();
+            lockById = null;
+        } 
+        
+        if (timer != null) {
+            timer.cancel();
+            timer.purge();
+            timer = null;
+        }
+        
+        if (env != null) {
+            env.close();
+            env = null;
+        }
+        logger.trace("< destroy");
+    }
+
+    class Expirer extends TimerTask {
+        public void run() {
+            if (lockById != null) {
+                logger.trace("> expire");
+                Transaction txn = env.beginTransaction(null, null);
+                try {
+                    EntityCursor<Lock> cursor = lockById.entities();
+                    try {
+                        for (Lock lock : cursor) {
+                            if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
+                                logger.trace("Found expired lock {}", lock.id);
+                                try {
+                                    releaseLock(null, lock.pid, lock.uid, lock.id);
+                                } catch (Exception e) {
+                                    logger.error("Exception while expiring lock for project '" + lock.pid + "'", e);
+                                }
+                            }
+                        }
+                    } finally {
+                        cursor.close();
+                    }
+                } finally {
+                    if (txn != null) {
+                        txn.abort();
+                        txn = null;
+                    }
+                }
+                logger.trace("< expire");
+            }
+        }
+    }
+    
+    // ---------------------------------------------------------------------------------
+
+    @Override
+    protected HttpClient getHttpClient() {
+        return new DefaultHttpClient();
+    }
+    
+    // ---------------------------------------------------------------------------------
+    
+    @Override
+    protected void expire(HttpServletResponse response) throws Exception {
+        expirer.run();
+        respond(response, OK);
+    }
+        
+    @Override
+    protected void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception {
+        logger.trace("> obtain lock");
+        Lock lock = null;
+        Lock blocker = null;
+
+        Transaction txn = env.beginTransaction(null, null);
+
+        try {
+
+            EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities(); 
+            
+            /*
+             *  ALL
+             *       blocked -> somebody else's lock
+             *       reuse -> you already have an ALL lock
+             *       new -> else
+             *   
+             *   COL
+             *       blocked -> somebody else's all lock || a lock on the same col
+             *       reuse -> you have an ALL lock || a lock on the same col 
+             *       new -> else
+             *   
+             *   CELL
+             *       blocked -> somebody else's all lock || a lock on the same col || a lock on the same cell
+             *       reuse -> you have a lock on the same cell            
+             *       yes -> (you have a lock on the same cell) && (nobody else has a lock on the same cell || the same col || all)
+             *       new -> else 
+             *       
+             */
+            
+            try {
+                if (locktype == ALL) {
+                    if (lockvalue.length() > 0) {
+                        throw new RuntimeException("Hmm, seems like you're calling an ALL with a specific value, are you sure you didn't want another type of lock?");
+                    }
+
+                    for (Lock l : cursor) {
+                        if (!l.uid.equals(uid)) {
+                            blocker = l;
+                            break;
+                        } else {
+                            if (l.type == ALL) {
+                                lock = l;
+                                break;
+                            }
+                        }
+                    }
+                } else if (locktype == COL) {
+                    if (lockvalue.indexOf(',') > -1) {
+                        throw new RuntimeException("Hmm, seems like you're calling a COL lock with a CELL value");
+                    }
+
+                    for (Lock l : cursor) {
+                        if (!l.uid.equals(uid)) {
+                            if (l.type == ALL || 
+                               (l.type == COL && l.value.equals(lockvalue)) ||
+                               (l.type == CELL && l.value.split(",")[0].equals(lockvalue))) {
+                                blocker = l;
+                                break;
+                            }
+                        } else {
+                            if (l.type == ALL || 
+                               (l.type == COL && l.value.equals(lockvalue))) {
+                                lock = l;
+                                break;
+                            }
+                        }
+                    }
+                } else if (locktype == CELL) {
+                    if (lockvalue.indexOf(',') == -1) {
+                        throw new RuntimeException("Hmm, seems like you're calling a CELL lock without specifying row and column: format must be 'row,column'");
+                    }
+
+                    for (Lock l : cursor) {
+                        if (!l.uid.equals(uid)) {
+                            if (l.type == ALL || 
+                               (l.type == COL && l.value.equals(lockvalue.split(",")[0])) || 
+                               (l.type == CELL && l.value.equals(lockvalue))) {
+                                blocker = l;
+                                break;
+                            }
+                        } else {
+                            if (l.type == ALL || 
+                               (l.type == COL && l.value.equals(lockvalue.split(",")[0])) || 
+                               (l.type == CELL && l.value.equals(lockvalue))) {
+                                lock = l;
+                                break;
+                            }
+                        }
+                    }
+                }
+            } finally {
+                cursor.close(); 
+            } 
+    
+            if (blocker != null) {
+                logger.info("found a blocking lock {}", lockToString(blocker));
+                throw new RuntimeException("Can't obtain lock, it is blocked by a type '" + blocker.type + "' lock owned by '" + blocker.uid + "'");
+            }
+            
+            if (lock == null) {
+                logger.info("no comparable lock already exists, creating a new one");
+                lock = new Lock(Long.toHexString(txn.getId()), pid, uid, locktype, lockvalue);
+                lockById.put(txn, lock);
+                txn.commit();
+            }
+
+        } finally {
+            if (txn != null) {
+                txn.abort();
+                txn = null;
+            }
+        }
+        
+        JSONObject o = lockToJSON(lock, uid);
+        o.put("status", "ok");
+        respond(response, o);
+        
+        logger.trace("< obtain lock");
+    }
+    
+    @Override
+    protected void releaseLock(HttpServletResponse response, String pid, String uid, String lid) throws Exception {
+
+        Transaction txn = env.beginTransaction(null, null);
+
+        try {
+            Lock lock = getLock(lid, pid, uid);
+            if (lock != null) {
+                if (!lock.uid.equals(uid)) {
+                    throw new RuntimeException("User id doesn't match the lock owner, can't release the lock");
+                }
+                lockById.delete(lid);
+                txn.commit();
+            }
+        } finally {
+            if (txn != null) {
+                txn.abort();
+                txn = null;
+            }
+        }
+                
+        if (response != null) { // this because the expiration thread can call this method without a real response
+            respond(response, OK);
+        }
+    }
+    
+    // ----------------------------------------------------------------------------------------------------
+
+    @Override
+    protected void startProject(HttpServletResponse response, String pid, String uid, String lid, byte[] data, String metadata, List<String> transformations) throws Exception {
+        
+        Transaction txn = env.beginTransaction(null, null);
+
+        try {
+            if (projectById.contains(pid)) {
+                throw new RuntimeException("Project '" + pid + "' already exists");
+            }
+            
+            Lock lock = getLock(lid, pid, uid);
+            
+            if (lock.type != ALL) {
+                throw new RuntimeException("The lock you have is not enough to start a project");
+            }
+            
+            projectById.put(txn, new Project(pid, data, metadata, transformations));
+            txn.commit();
+        } finally {
+            if (txn != null) {
+                txn.abort();
+                txn = null;
+            }
+        }
+        
+        respond(response, OK);
+    }
+    
+    @Override
+    protected void addTransformations(HttpServletResponse response, String pid, String uid, String lid, List<String> transformations) throws Exception {
+
+        Transaction txn = env.beginTransaction(null, null);
+        
+        try {
+            Project project = getProject(pid);
+            
+            if (project == null) {
+                throw new RuntimeException("Project '" + pid + "' not found");
+            }
+
+            Lock lock = getLock(lid, pid, uid);
+
+            logger.info("obtained lock: {}", lockToString(lock));
+            
+            if (lock.type == ALL) {
+                project.transformations.addAll(transformations);
+            } else {
+                for (String s : transformations) {
+                    JSONObject o = new JSONObject(s);
+                    
+                    int type = o.getInt("op_type");
+                    String value = o.getString("op_value");
+                    if (lock.type == COL) {
+                        if (type == COL) {
+                            if (value != null && value.equals(lock.value)) {
+                               project.transformations.add(s);
+                            } else {
+                                throw new RuntimeException("Can't apply '" + s + "': you have a lock for column '" + lock.value + "' and you're attempting to modify column '" + value + "'.");
+                            }
+                        } else if (type == CELL) {
+                            String column = value.split(",")[0];
+                            if (column != null && column.equals(lock.value)) {
+                                project.transformations.add(s);
+                            } else {
+                                throw new RuntimeException("Can't apply '" + s + "': you have a lock for column '" + lock.value + "' and you're attempting to modify cell '" + value + "' in another column.");
+                            }
+                        }
+                    } else if (lock.type == CELL) {
+                        if (type == COL) {
+                            throw new RuntimeException("Can't apply '" + s + "': you offered a lock for a single cell and you're attempting an operation for the entire column.");
+                        } else if (type == CELL) {
+                            if (value != null && value.equals(lock.value)) {
+                                project.transformations.add(s);
+                            } else {
+                                throw new RuntimeException("Can't apply '" + s + "': you have a lock for cell '" + lock.value + "' and you're attempting to modify cell '" + value + "'.");
+                            }
+                        }
+                    }
+                }
+            }
+
+            projectById.put(txn, project);
+            
+            txn.commit();
+        } finally {
+            if (txn != null) {
+                txn.abort();
+                txn = null;
+            }
+        }
+        
+        respond(response, OK);
+    }
+
+    // ---------------------------------------------------------------------------------
+    
+    @Override
+    protected void openProject(HttpServletResponse response, String pid) throws Exception {
+        Project project = getProject(pid);
+
+        Writer w = response.getWriter();
+        JSONWriter writer = new JSONWriter(w);
+        writer.object();
+            writer.key("status"); writer.value("ok");
+            writer.key("data"); writer.value(project.data);
+            writer.key("metadata"); writer.value(new JSONObject(project.metadata));
+            writer.key("transformations"); 
+            writer.array();
+                for (String s : project.transformations) {
+                    writer.value(new JSONObject(s));
+                }
+            writer.endArray();
+        writer.endObject();
+        w.flush();
+        w.close();
+    }
+        
+    // ---------------------------------------------------------------------------------
+
+    @Override
+    protected void getState(HttpServletResponse response, String pid, String uid, int rev) throws Exception {
+
+        Project project = getProject(pid);
+        
+        Writer w = response.getWriter();
+        JSONWriter writer = new JSONWriter(w);
+        
+        writer.object();
+        writer.key("status"); writer.value("ok");
+        writer.key("transformations"); 
+        writer.array();
+            int size = project.transformations.size();
+            for (int i = rev; i < size; i++) {
+                writer.value(new JSONObject(project.transformations.get(i)));
+            }
+        writer.endArray();
+
+        EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities(); 
+        
+        try {
+            writer.key("locks"); 
+            writer.array();
+            for (Lock lock : cursor) {
+                writer.value(lockToJSON(lock, uid));
+            }
+            writer.endArray();
+            writer.endObject();
+            
+            w.flush();
+            w.close();
+        } finally {
+            cursor.close(); 
+        } 
+    }
+    
+    // ---------------------------------------------------------------------------------
+
+    Project getProject(String pid) {
+        Project project = projectById.get(pid);
+        if (project == null) {
+            throw new RuntimeException("Project '" + pid + "' could not be found: are you sure is not managed by another broker?");
+        }
+        return project;
+    }
+        
+    @Entity
+    static class Project {
+        
+        @PrimaryKey
+        String pid;
+
+        List<String> transformations; 
+        
+        byte[] data;
+
+        String metadata;
+        
+        int rev;
+
+        Project(String pid, byte[] data, String metadata, List<String> transformations) {
+            this.pid = pid;
+            this.data = data;
+            this.metadata = metadata;
+            this.transformations = (transformations != null) ? transformations : new ArrayList<String>();
+            this.rev = this.transformations.size();
+        }
+        
+        @SuppressWarnings("unused")
+        private Project() {}
+    }
+    
+    // ---------------------------------------------------------------------------------
+    
+    Lock getLock(String lid, String pid, String uid) {
+        Lock lock = lockById.get(lid);
+        checkLock(lock, lid, pid, uid);
+        return lock;
+    }
+
+    void checkLock(Lock lock, String lid, String pid, String uid) {
+        if (lock == null) {
+            throw new RuntimeException("No lock was found with the given Lock id '" + lid + "', you have to have a valid lock on a project in order to start it");
+        }
+        
+        if (!lock.pid.equals(pid)) {
+            throw new RuntimeException("Lock '" + lock.id + "' is for another project: " + lock.pid);
+        }
+        
+        if (!lock.uid.equals(uid)) {
+            throw new RuntimeException("Lock '" + lock.id + "' is owned by another user: " + lock.uid);
+        }
+    }
+
+    Lock getLock(String pid, String uid, int locktype) {
+        Lock lock = null;
+        EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities(); 
+        
+        try {
+            for (Lock l : cursor) {
+                if (uid.equals(l.uid) && (locktype == l.type)) {
+                    lock = l;
+                    break;
+                }
+            }
+        } finally {
+            cursor.close(); 
+        } 
+        
+        return lock;
+    }    
+    
+    JSONObject lockToJSON(Lock lock, String uid) throws JSONException {
+        JSONObject o = new JSONObject();
+        if (lock != null) {
+            // NOTE: only the owner of the lock should get the ID, 
+            // otherwise others can just fake ownership of other people's locks
+            if (lock.uid.equals(uid)) { 
+                o.put("lock", lock.id);
+            }
+            
+            o.put("pid", lock.pid);
+            o.put("uid", lock.uid);
+            o.put("type", lock.type);
+            o.put("value", lock.value);
+            o.put("timestamp", lock.timestamp);
+        }
+        return o;
+    }
+    
+    String lockToString(Lock lock) {
+        return lock.id + "," + lock.pid + "," + lock.uid + "," + lock.type + "," + lock.value;
+    }
+    
+    @Entity
+    static class Lock {
+
+        @PrimaryKey
+        String id;
+        
+        @SecondaryKey(relate=MANY_TO_ONE)
+        String pid;
+        
+        String uid;
+        
+        int type;
+        
+        String value;
+        
+        long timestamp;
+        
+        Lock(String id, String pid, String uid, int type, String value) {
+            this.id = id;
+            this.pid = pid;
+            this.uid = uid;
+            this.type = type;
+            this.value = value;
+            this.timestamp = System.currentTimeMillis();
+        }
+                
+        @SuppressWarnings("unused")
+        private Lock() {}
+    }
+}

broker/core/src/com/metaweb/gridworks/broker/GridworksBroker.java

@@ -1,311 +0,0 @@
-
-package com.metaweb.gridworks.broker;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.io.Writer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import javax.servlet.ServletConfig;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.apache.http.NameValuePair;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.ResponseHandler;
-import org.apache.http.client.entity.UrlEncodedFormEntity;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.BasicResponseHandler;
-import org.apache.http.message.BasicNameValuePair;
-import org.apache.http.params.CoreProtocolPNames;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import edu.mit.simile.butterfly.ButterflyModuleImpl;
-
-/**
- * This class contains all the code shared by various implementations of a Gridworks Broker.
- * 
- * A broker is a server used by multiple Gridworks installations to enable collaborative
- * development over the same project.
- * 
- * Broker implementations differ in how they store their state but all of them are required
- * to extend this abstract class and implement the services that are called via HTTP.
- * 
- */
-public abstract class GridworksBroker extends ButterflyModuleImpl {
-    
-    static final public String GET_STATE = "get_state";
-    static final public String EXPIRE = "expire";
-    static final public String OBTAIN_LOCK = "obtain_lock";
-    static final public String RELEASE_LOCK = "release_lock";
-    static final public String TRANSFORM = "transform";
-    static final public String START = "start";
-    static final public String OPEN = "open";
-
-    static final public int ALL = 0;
-    static final public int COL = 1;
-    static final public int CELL = 2;
-    
-    static final protected Logger logger = LoggerFactory.getLogger("gridworks.broker");
-    
-    static final protected String USER_INFO_URL = "http://www.freebase.com/api/service/user_info";
-    static final protected String DELEGATED_OAUTH_HEADER = "X-Freebase-Credentials";
-    static final protected String OAUTH_HEADER = "Authorization";
-
-    static protected String OK; 
-    
-    static {
-        try {
-            JSONObject o = new JSONObject();
-            o.put("status","ok");
-            OK = o.toString();
-        } catch (JSONException e) {
-            // not going to happen;
-        }
-    }
-    
-    static public final long LOCK_DURATION = 60 * 1000; // 1 minute
-    static public final long USER_DURATION = 5 * 60 * 1000; // 1 minute
-    static public final long LOCK_EXPIRATION_CHECK_DELAY = 5 * 1000; // 5 seconds
-    
-    protected HttpClient httpclient;
-
-    protected boolean developmentMode;
-    
-    @Override
-    public void init(ServletConfig config) throws Exception {
-        super.init(config);
-        httpclient = getHttpClient();
-        developmentMode = Boolean.parseBoolean(config.getInitParameter("gridworks.development"));
-        if (developmentMode) logger.warn("Running in development mode");
-    }
-
-    @Override
-    public void destroy() throws Exception {
-        httpclient.getConnectionManager().shutdown();       
-    }
-    
-    @Override
-    public boolean process(String path, HttpServletRequest request, HttpServletResponse response) throws Exception {
-        if (logger.isDebugEnabled()) {
-            logger.debug("> process '{}'", path);
-        } else {
-            logger.info("process '{}'", path);
-        }
-
-        try {
-
-            if (GET_STATE.equals(path)) {
-                response.setCharacterEncoding("UTF-8");
-                response.setHeader("Content-Type", "application/json");
-                getState(response, getParameter(request, "pid"), getUserId(request), getInteger(request, "rev"));
-            } else if (EXPIRE.equals(path)) {
-                response.setCharacterEncoding("UTF-8");
-                response.setHeader("Content-Type", "application/json");
-                expire(response);
-            } else if (OBTAIN_LOCK.equals(path)) {
-                response.setCharacterEncoding("UTF-8");
-                response.setHeader("Content-Type", "application/json");
-                obtainLock(response, getParameter(request, "pid"), getUserId(request), getInteger(request, "locktype"), getParameter(request, "lockvalue"));
-            } else if (RELEASE_LOCK.equals(path)) {
-                response.setCharacterEncoding("UTF-8");
-                response.setHeader("Content-Type", "application/json");
-                releaseLock(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"));
-            } else if (TRANSFORM.equals(path)) {
-                response.setCharacterEncoding("UTF-8");
-                response.setHeader("Content-Type", "application/json");
-                addTransformations(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"), getList(request, "transformations"));
-            } else if (START.equals(path)) {
-                response.setCharacterEncoding("UTF-8");
-                response.setHeader("Content-Type", "application/json");
-                startProject(response, getParameter(request, "pid"), getUserId(request), getParameter(request, "lock"), getData(request), getParameter(request, "metadata"), getList(request, "transformations"));
-            } else if (OPEN.equals(path)) {
-                response.setCharacterEncoding("UTF-8");
-                response.setHeader("Content-Type", "application/json");
-                openProject(response, getParameter(request, "pid"));
-            } else {
-                boolean value = super.process(path, request, response);
-                if (logger.isDebugEnabled()) logger.debug("< process '{}'", path);
-                return value;
-            }
-                
-        } catch (RuntimeException e) {
-            logger.error("runtime error", e.getMessage());
-            respondError(response, e.getMessage());
-        } catch (Exception e) {
-            logger.error("internal error", e);
-            respondException(response, e);
-        }
-        
-        if (logger.isDebugEnabled()) logger.debug("< process '{}'", path);
-        
-        return true;
-    }
-    
-    // ----------------------------------------------------------------------------------------
-    
-    protected abstract HttpClient getHttpClient();
-
-    protected abstract void expire(HttpServletResponse response) throws Exception;
-
-    protected abstract void getState(HttpServletResponse response, String pid, String uid, int rev) throws Exception;
-
-    protected abstract void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception;
-    
-    protected abstract void releaseLock(HttpServletResponse response, String pid, String uid, String lock) throws Exception;
-
-    protected abstract void startProject(HttpServletResponse response, String pid, String uid, String lock, byte[] data, String metadata, List<String> transformations) throws Exception;
-
-    protected abstract void addTransformations(HttpServletResponse response, String pid, String uid, String lock, List<String> transformations) throws Exception;
-    
-    protected abstract void openProject(HttpServletResponse response, String pid) throws Exception;
-    
-    // ----------------------------------------------------------------------------------------
-    
-    @SuppressWarnings("unchecked")
-    protected String getUserId(HttpServletRequest request) throws Exception {
-
-        // This is useful for testing
-        if (developmentMode) {
-            return getParameter(request, "uid");
-        }
-        
-        String oauth = request.getHeader(DELEGATED_OAUTH_HEADER);
-        if (oauth == null) {
-            throw new RuntimeException("The request needs to contain the '" + DELEGATED_OAUTH_HEADER + "' header set to obtain user identity via Freebase.");
-        }
-        
-        List<NameValuePair> formparams = new ArrayList<NameValuePair>();
-        Map<String,String> params = (Map<String,String>) request.getParameterMap();
-        for (Entry<String,String> e : params.entrySet()) {
-            formparams.add(new BasicNameValuePair((String) e.getKey(), (String) e.getValue()));
-        }
-        UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
-
-        HttpPost httpRequest = new HttpPost(USER_INFO_URL);
-        httpRequest.setHeader(OAUTH_HEADER, oauth);
-        httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks Broker");
-        httpRequest.setEntity(entity);
-                
-        ResponseHandler<String> responseHandler = new BasicResponseHandler();
-        String responseBody = httpclient.execute(httpRequest, responseHandler);
-        JSONObject o = new JSONObject(responseBody);
-        
-        return o.getString("username");
-    }
-
-    // ----------------------------------------------------------------------------------------
-
-    static protected String getParameter(HttpServletRequest request, String name) throws ServletException {
-        String param = request.getParameter(name);
-        if (param == null) {
-            throw new RuntimeException("request must come with a '" + name + "' parameter");
-        }
-        return param;
-    }
-    
-    static protected List<String> getList(HttpServletRequest request, String name) throws ServletException, JSONException {
-        String param = getParameter(request, name);
-        JSONArray a = new JSONArray(param);
-        List<String> result = new ArrayList<String>(a.length());
-        for (int i = 0; i < a.length(); i++) {
-            result.add(a.getString(i));
-        }
-        return result;
-    }
-    
-    static protected int getInteger(HttpServletRequest request, String name) throws ServletException, JSONException {
-        return Integer.parseInt(getParameter(request, name));
-    }
-    
-    static protected byte[] getData(HttpServletRequest request) throws ServletException, IOException {
-        ByteArrayOutputStream output = new ByteArrayOutputStream();
-        InputStream input = request.getInputStream();
-        byte[] buffer = new byte[4096];
-        int count = 0;
-        int n = 0;
-        while (-1 != (n = input.read(buffer))) {
-            output.write(buffer, 0, n);
-            count += n;
-        }
-        return output.toByteArray();
-        
-    }
-    
-    static protected void respondError(HttpServletResponse response, String error) throws IOException, ServletException {
-
-        if (response == null) {
-            throw new ServletException("Response object can't be null");
-        }
-        
-        try {
-            JSONObject o = new JSONObject();
-            o.put("status", "error");
-            o.put("message", error);
-            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
-            respond(response, o.toString());
-        } catch (JSONException e) {
-            e.printStackTrace(response.getWriter());
-        }
-    }
-    
-    static protected void respondException(HttpServletResponse response, Exception e) throws IOException, ServletException {
-
-        if (response == null) {
-            throw new ServletException("Response object can't be null");
-        }
-        
-        try {
-            JSONObject o = new JSONObject();
-            o.put("status", "error");
-            o.put("message", e.getMessage());
-        
-            StringWriter sw = new StringWriter();
-            PrintWriter pw = new PrintWriter(sw);
-            e.printStackTrace(pw);
-            pw.flush();
-            sw.flush();
-        
-            o.put("stack", sw.toString());
-        
-            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
-            respond(response, o.toString());
-        } catch (JSONException e1) {
-            e.printStackTrace(response.getWriter());
-        }
-    }
-
-    static protected void respond(HttpServletResponse response, JSONObject content) throws IOException, ServletException {
-        if (content == null) {
-            throw new ServletException("Content object can't be null");
-        }
-        
-        respond(response, content.toString());
-    }
-
-    static protected void respond(HttpServletResponse response, String content) throws IOException, ServletException {
-        if (response == null) {
-            throw new ServletException("Response object can't be null");
-        }
-        
-        Writer w = response.getWriter();
-        if (w != null) {
-            w.write(content);
-            w.flush();
-            w.close();
-        } else {
-            throw new ServletException("response returned a null writer");
-        }
-    }
-}

broker/core/src/com/metaweb/gridworks/broker/GridworksBrokerImpl.java

@@ -1,592 +0,0 @@
-package com.metaweb.gridworks.broker;
-
-import static com.sleepycat.persist.model.Relationship.MANY_TO_ONE;
-
-import java.io.File;
-import java.io.Writer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
-
-import javax.servlet.ServletConfig;
-import javax.servlet.http.HttpServletResponse;
-
-import org.apache.http.client.HttpClient;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.sleepycat.je.Environment;
-import com.sleepycat.je.EnvironmentConfig;
-import com.sleepycat.je.Transaction;
-import com.sleepycat.persist.EntityCursor;
-import com.sleepycat.persist.EntityStore;
-import com.sleepycat.persist.PrimaryIndex;
-import com.sleepycat.persist.SecondaryIndex;
-import com.sleepycat.persist.StoreConfig;
-import com.sleepycat.persist.model.Entity;
-import com.sleepycat.persist.model.PrimaryKey;
-import com.sleepycat.persist.model.SecondaryKey;
-
-public class GridworksBrokerImpl extends GridworksBroker {
-                
-    protected static final Logger logger = LoggerFactory.getLogger("gridworks.broker.local");
-    
-    Environment env;
-    
-    EntityStore projectStore;
-    EntityStore lockStore;
-    EntityStore userStore;
-    
-    PrimaryIndex<String,Project> projectById;
-    PrimaryIndex<String,Lock> lockById;
-
-    SecondaryIndex<String,String,Lock> locksByProject;
-    
-    Timer timer;
-    Expirer expirer;
-
-    @Override
-    public void init(ServletConfig config) throws Exception {
-        logger.trace("> init");
-        super.init(config);
-
-        timer = new Timer();
-        expirer = new Expirer();
-        timer.schedule(expirer, 0, LOCK_EXPIRATION_CHECK_DELAY);
-        
-        String dataDir = config.getInitParameter("gridworks.data");
-        if (dataDir == null) dataDir = "data";
-        File dataPath = new File(dataDir);
-        if (!dataPath.exists()) dataPath.mkdirs();
-        
-        EnvironmentConfig envConfig = new EnvironmentConfig();
-        envConfig.setAllowCreate(true);
-        envConfig.setTransactional(true);
-        env = new Environment(dataPath, envConfig); 
-
-        StoreConfig storeConfig = new StoreConfig();
-        storeConfig.setAllowCreate(true);
-        storeConfig.setTransactional(true);
-        projectStore = new EntityStore(env, "ProjectsStore", storeConfig);
-        lockStore = new EntityStore(env, "LockStore", storeConfig);
-        
-        projectById = projectStore.getPrimaryIndex(String.class, Project.class);
-        lockById = lockStore.getPrimaryIndex(String.class, Lock.class);
-        
-        locksByProject = lockStore.getSecondaryIndex(lockById, String.class, "pid"); 
-        logger.trace("< init");
-    }
-    
-    @Override
-    public void destroy() throws Exception {
-        logger.trace("> destroy");
-        super.destroy();
-        
-        if (projectStore != null) {
-            projectStore.close();
-            projectById = null;            
-        } 
-
-        if (lockStore != null) {
-            lockStore.close();
-            lockById = null;
-        } 
-        
-        if (timer != null) {
-            timer.cancel();
-            timer.purge();
-            timer = null;
-        }
-        
-        if (env != null) {
-            env.close();
-            env = null;
-        }
-        logger.trace("< destroy");
-    }
-
-    class Expirer extends TimerTask {
-        public void run() {
-            if (lockById != null) {
-                logger.trace("> expire");
-                Transaction txn = env.beginTransaction(null, null);
-                try {
-                    EntityCursor<Lock> cursor = lockById.entities();
-                    try {
-                        for (Lock lock : cursor) {
-                            if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
-                                logger.trace("Found expired lock {}", lock.id);
-                                try {
-                                    releaseLock(null, lock.pid, lock.uid, lock.id);
-                                } catch (Exception e) {
-                                    logger.error("Exception while expiring lock for project '" + lock.pid + "'", e);
-                                }
-                            }
-                        }
-                    } finally {
-                        cursor.close();
-                    }
-                } finally {
-                    if (txn != null) {
-                        txn.abort();
-                        txn = null;
-                    }
-                }
-                logger.trace("< expire");
-            }
-        }
-    }
-    
-    // ---------------------------------------------------------------------------------
-
-    @Override
-    protected HttpClient getHttpClient() {
-        return new DefaultHttpClient();
-    }
-    
-    // ---------------------------------------------------------------------------------
-    
-    @Override
-    protected void expire(HttpServletResponse response) throws Exception {
-        expirer.run();
-        respond(response, OK);
-    }
-        
-    @Override
-    protected void obtainLock(HttpServletResponse response, String pid, String uid, int locktype, String lockvalue) throws Exception {
-        logger.trace("> obtain lock");
-        Lock lock = null;
-        Lock blocker = null;
-
-        Transaction txn = env.beginTransaction(null, null);
-
-        try {
-
-            EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities(); 
-            
-            /*
-             *  ALL
-             *       blocked -> somebody else's lock
-             *       reuse -> you already have an ALL lock
-             *       new -> else
-             *   
-             *   COL
-             *       blocked -> somebody else's all lock || a lock on the same col
-             *       reuse -> you have an ALL lock || a lock on the same col 
-             *       new -> else
-             *   
-             *   CELL
-             *       blocked -> somebody else's all lock || a lock on the same col || a lock on the same cell
-             *       reuse -> you have a lock on the same cell            
-             *       yes -> (you have a lock on the same cell) && (nobody else has a lock on the same cell || the same col || all)
-             *       new -> else 
-             *       
-             */
-            
-            try {
-                if (locktype == ALL) {
-                    if (lockvalue.length() > 0) {
-                        throw new RuntimeException("Hmm, seems like you're calling an ALL with a specific value, are you sure you didn't want another type of lock?");
-                    }
-
-                    for (Lock l : cursor) {
-                        if (!l.uid.equals(uid)) {
-                            blocker = l;
-                            break;
-                        } else {
-                            if (l.type == ALL) {
-                                lock = l;
-                                break;
-                            }
-                        }
-                    }
-                } else if (locktype == COL) {
-                    if (lockvalue.indexOf(',') > -1) {
-                        throw new RuntimeException("Hmm, seems like you're calling a COL lock with a CELL value");
-                    }
-
-                    for (Lock l : cursor) {
-                        if (!l.uid.equals(uid)) {
-                            if (l.type == ALL || 
-                               (l.type == COL && l.value.equals(lockvalue)) ||
-                               (l.type == CELL && l.value.split(",")[0].equals(lockvalue))) {
-                                blocker = l;
-                                break;
-                            }
-                        } else {
-                            if (l.type == ALL || 
-                               (l.type == COL && l.value.equals(lockvalue))) {
-                                lock = l;
-                                break;
-                            }
-                        }
-                    }
-                } else if (locktype == CELL) {
-                    if (lockvalue.indexOf(',') == -1) {
-                        throw new RuntimeException("Hmm, seems like you're calling a CELL lock without specifying row and column: format must be 'row,column'");
-                    }
-
-                    for (Lock l : cursor) {
-                        if (!l.uid.equals(uid)) {
-                            if (l.type == ALL || 
-                               (l.type == COL && l.value.equals(lockvalue.split(",")[0])) || 
-                               (l.type == CELL && l.value.equals(lockvalue))) {
-                                blocker = l;
-                                break;
-                            }
-                        } else {
-                            if (l.type == ALL || 
-                               (l.type == COL && l.value.equals(lockvalue.split(",")[0])) || 
-                               (l.type == CELL && l.value.equals(lockvalue))) {
-                                lock = l;
-                                break;
-                            }
-                        }
-                    }
-                }
-            } finally {
-                cursor.close(); 
-            } 
-    
-            if (blocker != null) {
-                logger.info("found a blocking lock {}", lockToString(blocker));
-                throw new RuntimeException("Can't obtain lock, it is blocked by a type '" + blocker.type + "' lock owned by '" + blocker.uid + "'");
-            }
-            
-            if (lock == null) {
-                logger.info("no comparable lock already exists, creating a new one");
-                lock = new Lock(Long.toHexString(txn.getId()), pid, uid, locktype, lockvalue);
-                lockById.put(txn, lock);
-                txn.commit();
-            }
-
-        } finally {
-            if (txn != null) {
-                txn.abort();
-                txn = null;
-            }
-        }
-        
-        JSONObject o = lockToJSON(lock, uid);
-        o.put("status", "ok");
-        respond(response, o);
-        
-        logger.trace("< obtain lock");
-    }
-    
-    @Override
-    protected void releaseLock(HttpServletResponse response, String pid, String uid, String lid) throws Exception {
-
-        Transaction txn = env.beginTransaction(null, null);
-
-        try {
-            Lock lock = getLock(lid, pid, uid);
-            if (lock != null) {
-                if (!lock.uid.equals(uid)) {
-                    throw new RuntimeException("User id doesn't match the lock owner, can't release the lock");
-                }
-                lockById.delete(lid);
-                txn.commit();
-            }
-        } finally {
-            if (txn != null) {
-                txn.abort();
-                txn = null;
-            }
-        }
-                
-        if (response != null) { // this because the expiration thread can call this method without a real response
-            respond(response, OK);
-        }
-    }
-    
-    // ----------------------------------------------------------------------------------------------------
-
-    @Override
-    protected void startProject(HttpServletResponse response, String pid, String uid, String lid, byte[] data, String metadata, List<String> transformations) throws Exception {
-        
-        Transaction txn = env.beginTransaction(null, null);
-
-        try {
-            if (projectById.contains(pid)) {
-                throw new RuntimeException("Project '" + pid + "' already exists");
-            }
-            
-            Lock lock = getLock(lid, pid, uid);
-            
-            if (lock.type != ALL) {
-                throw new RuntimeException("The lock you have is not enough to start a project");
-            }
-            
-            projectById.put(txn, new Project(pid, data, metadata, transformations));
-            txn.commit();
-        } finally {
-            if (txn != null) {
-                txn.abort();
-                txn = null;
-            }
-        }
-        
-        respond(response, OK);
-    }
-    
-    @Override
-    protected void addTransformations(HttpServletResponse response, String pid, String uid, String lid, List<String> transformations) throws Exception {
-
-        Transaction txn = env.beginTransaction(null, null);
-        
-        try {
-            Project project = getProject(pid);
-            
-            if (project == null) {
-                throw new RuntimeException("Project '" + pid + "' not found");
-            }
-
-            Lock lock = getLock(lid, pid, uid);
-
-            logger.info("obtained lock: {}", lockToString(lock));
-            
-            if (lock.type == ALL) {
-                project.transformations.addAll(transformations);
-            } else {
-                for (String s : transformations) {
-                    JSONObject o = new JSONObject(s);
-                    
-                    int type = o.getInt("op_type");
-                    String value = o.getString("op_value");
-                    if (lock.type == COL) {
-                        if (type == COL) {
-                            if (value != null && value.equals(lock.value)) {
-                               project.transformations.add(s);
-                            } else {
-                                throw new RuntimeException("Can't apply '" + s + "': you have a lock for column '" + lock.value + "' and you're attempting to modify column '" + value + "'.");
-                            }
-                        } else if (type == CELL) {
-                            String column = value.split(",")[0];
-                            if (column != null && column.equals(lock.value)) {
-                                project.transformations.add(s);
-                            } else {
-                                throw new RuntimeException("Can't apply '" + s + "': you have a lock for column '" + lock.value + "' and you're attempting to modify cell '" + value + "' in another column.");
-                            }
-                        }
-                    } else if (lock.type == CELL) {
-                        if (type == COL) {
-                            throw new RuntimeException("Can't apply '" + s + "': you offered a lock for a single cell and you're attempting an operation for the entire column.");
-                        } else if (type == CELL) {
-                            if (value != null && value.equals(lock.value)) {
-                                project.transformations.add(s);
-                            } else {
-                                throw new RuntimeException("Can't apply '" + s + "': you have a lock for cell '" + lock.value + "' and you're attempting to modify cell '" + value + "'.");
-                            }
-                        }
-                    }
-                }
-            }
-
-            projectById.put(txn, project);
-            
-            txn.commit();
-        } finally {
-            if (txn != null) {
-                txn.abort();
-                txn = null;
-            }
-        }
-        
-        respond(response, OK);
-    }
-
-    // ---------------------------------------------------------------------------------
-    
-    @Override
-    protected void openProject(HttpServletResponse response, String pid) throws Exception {
-        Project project = getProject(pid);
-
-        Writer w = response.getWriter();
-        JSONWriter writer = new JSONWriter(w);
-        writer.object();
-            writer.key("status"); writer.value("ok");
-            writer.key("data"); writer.value(project.data);
-            writer.key("metadata"); writer.value(new JSONObject(project.metadata));
-            writer.key("transformations"); 
-            writer.array();
-                for (String s : project.transformations) {
-                    writer.value(new JSONObject(s));
-                }
-            writer.endArray();
-        writer.endObject();
-        w.flush();
-        w.close();
-    }
-        
-    // ---------------------------------------------------------------------------------
-
-    @Override
-    protected void getState(HttpServletResponse response, String pid, String uid, int rev) throws Exception {
-
-        Project project = getProject(pid);
-        
-        Writer w = response.getWriter();
-        JSONWriter writer = new JSONWriter(w);
-        
-        writer.object();
-        writer.key("status"); writer.value("ok");
-        writer.key("transformations"); 
-        writer.array();
-            int size = project.transformations.size();
-            for (int i = rev; i < size; i++) {
-                writer.value(new JSONObject(project.transformations.get(i)));
-            }
-        writer.endArray();
-
-        EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities(); 
-        
-        try {
-            writer.key("locks"); 
-            writer.array();
-            for (Lock lock : cursor) {
-                writer.value(lockToJSON(lock, uid));
-            }
-            writer.endArray();
-            writer.endObject();
-            
-            w.flush();
-            w.close();
-        } finally {
-            cursor.close(); 
-        } 
-    }
-    
-    // ---------------------------------------------------------------------------------
-
-    Project getProject(String pid) {
-        Project project = projectById.get(pid);
-        if (project == null) {
-            throw new RuntimeException("Project '" + pid + "' could not be found: are you sure is not managed by another broker?");
-        }
-        return project;
-    }
-        
-    @Entity
-    static class Project {
-        
-        @PrimaryKey
-        String pid;
-
-        List<String> transformations; 
-        
-        byte[] data;
-
-        String metadata;
-        
-        int rev;
-
-        Project(String pid, byte[] data, String metadata, List<String> transformations) {
-            this.pid = pid;
-            this.data = data;
-            this.metadata = metadata;
-            this.transformations = (transformations != null) ? transformations : new ArrayList<String>();
-            this.rev = this.transformations.size();
-        }
-        
-        @SuppressWarnings("unused")
-        private Project() {}
-    }
-    
-    // ---------------------------------------------------------------------------------
-    
-    Lock getLock(String lid, String pid, String uid) {
-        Lock lock = lockById.get(lid);
-        checkLock(lock, lid, pid, uid);
-        return lock;
-    }
-
-    void checkLock(Lock lock, String lid, String pid, String uid) {
-        if (lock == null) {
-            throw new RuntimeException("No lock was found with the given Lock id '" + lid + "', you have to have a valid lock on a project in order to start it");
-        }
-        
-        if (!lock.pid.equals(pid)) {
-            throw new RuntimeException("Lock '" + lock.id + "' is for another project: " + lock.pid);
-        }
-        
-        if (!lock.uid.equals(uid)) {
-            throw new RuntimeException("Lock '" + lock.id + "' is owned by another user: " + lock.uid);
-        }
-    }
-
-    Lock getLock(String pid, String uid, int locktype) {
-        Lock lock = null;
-        EntityCursor<Lock> cursor = locksByProject.subIndex(pid).entities(); 
-        
-        try {
-            for (Lock l : cursor) {
-                if (uid.equals(l.uid) && (locktype == l.type)) {
-                    lock = l;
-                    break;
-                }
-            }
-        } finally {
-            cursor.close(); 
-        } 
-        
-        return lock;
-    }    
-    
-    JSONObject lockToJSON(Lock lock, String uid) throws JSONException {
-        JSONObject o = new JSONObject();
-        if (lock != null) {
-            // NOTE: only the owner of the lock should get the ID, 
-            // otherwise others can just fake ownership of other people's locks
-            if (lock.uid.equals(uid)) { 
-                o.put("lock", lock.id);
-            }
-            
-            o.put("pid", lock.pid);
-            o.put("uid", lock.uid);
-            o.put("type", lock.type);
-            o.put("value", lock.value);
-            o.put("timestamp", lock.timestamp);
-        }
-        return o;
-    }
-    
-    String lockToString(Lock lock) {
-        return lock.id + "," + lock.pid + "," + lock.uid + "," + lock.type + "," + lock.value;
-    }
-    
-    @Entity
-    static class Lock {
-
-        @PrimaryKey
-        String id;
-        
-        @SecondaryKey(relate=MANY_TO_ONE)
-        String pid;
-        
-        String uid;
-        
-        int type;
-        
-        String value;
-        
-        long timestamp;
-        
-        Lock(String id, String pid, String uid, int type, String value) {
-            this.id = id;
-            this.pid = pid;
-            this.uid = uid;
-            this.type = type;
-            this.value = value;
-            this.timestamp = System.currentTimeMillis();
-        }
-                
-        @SuppressWarnings("unused")
-        private Lock() {}
-    }
-}

broker/core/tests/conf/tests.xml

@@ -3,7 +3,7 @@
 <suite name="Gridworks Broker Unit Tests">
   <test name="tests">
     <packages>
-      <package name="com.metaweb.gridworks.broker.tests.*"/>
+      <package name="com.google.gridworks.broker.tests.*"/>
     </packages>
     <groups>
       <run>

broker/core/tests/src/com/google/gridworks/broker/tests/GridworksBrokerTests.java

@@ -0,0 +1,452 @@
+package com.google.gridworks.broker.tests;
+
+import static com.google.gridworks.broker.GridworksBroker.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletInputStream;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.AfterSuite;
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeSuite;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import com.google.gridworks.broker.GridworksBroker;
+import com.google.gridworks.broker.GridworksBrokerImpl;
+
+public class GridworksBrokerTests {
+
+    Logger logger;
+    File data;
+    
+    @BeforeSuite public void suite_init() {
+        System.setProperty("log4j.configuration", "tests.log4j.properties");
+        data = new File("data");
+        if (!data.exists()) data.mkdirs();
+    }
+
+    @AfterSuite public void suite_destroy() {
+        for (File f : data.listFiles()) {
+            f.delete();
+        }
+        data.delete();
+    }
+
+    // ------------------------------------------------------------------------------------
+    
+    ServletConfig config = null;
+    GridworksBroker broker = null;
+    
+    @BeforeTest public void test_init() throws Exception {
+        logger = LoggerFactory.getLogger(this.getClass());
+        config = mock(ServletConfig.class);
+        when(config.getInitParameter("gridworks.data")).thenReturn(data.getAbsolutePath());
+        when(config.getInitParameter("gridworks.development")).thenReturn("true");
+
+        broker = new GridworksBrokerImpl();
+        broker.init(config);
+    }
+
+    @AfterTest public void test_destroy() throws Exception {
+        broker.destroy();
+        broker = null;
+        config = null;
+    }
+    
+    // ------------------------------------------------------------------------------------
+
+    HttpServletRequest request = null;
+    HttpServletResponse response = null;
+    StringWriter writer = null;
+    
+    @BeforeMethod public void setup() throws Exception {
+        request = mock(HttpServletRequest.class);
+        response = mock(HttpServletResponse.class);
+    }
+
+    @AfterMethod public void teardown() throws Exception {
+        response = null;
+        request = null;
+    }
+    
+    // ------------------------------------------------------------------------------------
+
+    @Test public void testLifeCycle() {
+        Assert.assertTrue(true);
+    }
+
+    @Test public void testService() {
+        try {
+            success(broker, request, response, EXPIRE);
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+    @Test public void testObtainLockFailure() {
+        try {
+            failure(broker, request, response, OBTAIN_LOCK);
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+    @Test public void testReleaseLockFailure() {
+        try {
+            failure(broker, request, response, RELEASE_LOCK);
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+    @Test public void testGetStateFailure() {
+        try {
+            failure(broker, request, response, GET_STATE, "pid", "project1934983948", "uid", "testuser", "rev", "0");
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+    @Test public void testBrokenAllLockFailure() {
+        try {
+            failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(ALL), "lockvalue", "1");
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+    
+    @Test public void testBrokenColLockFailure() {
+        try {
+            failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(COL), "lockvalue", "1,1");
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+    
+    @Test public void testBrokenCellLockFailure() {
+        try {
+            failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(CELL), "lockvalue", "1");
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+    
+    @Test public void testLockSimple() {
+        String project = "proj0";
+        String user = "testuser";
+        
+        try {
+            logger.info("--- obtain ALL lock on project ---");
+            JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
+            assertJSON(result, "uid", "testuser");
+            String lock = result.getString("lock");
+    
+            logger.info("--- obtain ALL lock on project ---");
+            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
+
+            logger.info("--- obtain COL lock on project ---");
+            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
+            assertJSON(result, "uid", "testuser");
+            lock = result.getString("lock");
+    
+            logger.info("--- release COL lock on project ---");
+            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
+
+            logger.info("--- obtain CELL lock on project ---");
+            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
+            assertJSON(result, "uid", "testuser");
+            lock = result.getString("lock");
+            
+            logger.info("--- release CELL lock on project ---");
+            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+    
+    @Test public void testLocksAllBlocks() {
+        String project = "proj1";
+        String user = "testuser";
+        String user2 = "testuser2";
+
+        try {
+            logger.info("--- obtain ALL lock on project ---");
+            JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
+            assertJSON(result, "uid", user);
+            String lock = result.getString("lock");
+
+            logger.info("--- another using asking for any lock will fail ---");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
+
+            logger.info("--- same user asking for lower capable locks will return the ALL one ---");
+            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
+            String lock2 = result.getString("lock");
+            Assert.assertEquals(lock, lock2);
+            
+            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
+            lock2 = result.getString("lock");
+            Assert.assertEquals(lock, lock2);
+            
+            logger.info("--- release the ALL lock ---");
+            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+    @Test public void testLocksColBlocks() {
+        String project = "proj2";
+        String user = "testuser";
+        String user2 = "testuser2";
+
+        try {
+            logger.info("--- obtain COL lock on project ---");
+            JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
+            String lock = result.getString("lock");
+
+            logger.info("--- other user must fail to obtain lock on the same COL or ALL ---");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
+            
+            logger.info("--- but succeed in getting a COL lock on another column or cell ---");
+            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "2");
+            String lock2 = result.getString("lock");
+            
+            logger.info("--- now it's our first user's turn to fail to get lock ---");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "2");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
+            
+            logger.info("--- release the locks ---");
+            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
+            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", lock2);
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+
+    @Test public void testLocksCellBlocks() {
+        String project = "proj3";
+        String user = "testuser";
+        String user2 = "testuser2";
+
+        try {
+            logger.info("--- obtain CELL lock on project ---");
+            JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
+            String lock = result.getString("lock");
+
+            logger.info("--- other user must fail to obtain lock on the same CELL, COL or ALL ---");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
+            
+            logger.info("--- but succeed in getting a CELL lock on a cell in another column ---");
+            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
+            String lock2 = result.getString("lock");
+            
+            logger.info("--- now it's our first user's turn to fail to get lock ---");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "2");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
+            
+            logger.info("--- release the locks ---");
+            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
+            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", lock2);
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+    
+    @Test public void testCompleteProjectLifeCycle() {
+        try {
+            String project = "proj4";
+            String user = "testuser";
+            String user2 = "testuser2";
+            String data = "blah";
+            String metadata = "{}";
+            String transformations = "[]";
+            String rev = "0";
+            
+            logger.info("--- obtain ALL lock on project ---");
+            JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
+            assertJSON(result, "uid", user);
+            String lock = result.getString("lock");
+
+            logger.info("--- start project ---");
+            success(broker, request, response, START, "pid", project, "uid", user, "lock", lock, "data", data, "metadata", metadata, "transformations", transformations);
+
+            logger.info("--- verify project state contains lock ---");
+            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev);
+            JSONArray locks = result.getJSONArray("locks");
+            Assert.assertEquals(locks.length(), 1);
+            JSONObject l = locks.getJSONObject(0);
+            assertJSON(l, "uid", "testuser");
+            Assert.assertEquals(l.getInt("type"), ALL);
+            
+            logger.info("--- release ALL lock on project ---");
+            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
+            
+            logger.info("--- verify no locks are present ---");
+            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev);
+            locks = result.getJSONArray("locks");
+            Assert.assertEquals(locks.length(), 0);
+            
+            logger.info("--- open project and verify data was loaded correctly ---");
+            result = success(broker, request, response, OPEN, "pid", project, "uid", user, "rev", rev);
+            JSONArray result_data = result.getJSONArray("data");
+            Assert.assertEquals(result_data.length(),data.getBytes("UTF-8").length);
+
+            JSONArray tt;
+            JSONObject t;
+            
+            logger.info("--- obtain column lock ---");
+            String column = "1";
+            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", column);
+            String col_lock = result.getString("lock");
+
+            logger.info("--- perform column transformation ---");
+            t = new JSONObject();
+            t.put("op_type", COL);
+            t.put("op_value", column); // operate on col 1
+            t.put("value", new JSONObject());
+            tt = new JSONArray();
+            tt.put(t);
+            result = success(broker, request, response, TRANSFORM, "pid", project, "uid", user, "lock", col_lock, "transformations", tt.toString());
+
+            logger.info("--- make sure transformation was recorded properly ---");
+            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "0");
+            tt = result.getJSONArray("transformations");
+            Assert.assertEquals(tt.length(), 1);
+            t = tt.getJSONObject(0);
+            assertJSON(t, "op_value", column);
+
+            logger.info("--- make sure revision numbers in state management work as expected ---");
+            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "1");
+            tt = result.getJSONArray("transformations");
+            Assert.assertEquals(tt.length(), 0);
+            
+            logger.info("--- perform cell transformation ---");
+            String cell = "1";
+            t = new JSONObject();
+            t.put("op_type", CELL);
+            t.put("op_value", column + "," + cell); // operate on cell at row 1 column 1
+            t.put("value", new JSONObject());
+            tt = new JSONArray();
+            tt.put(t);
+            result = success(broker, request, response, TRANSFORM, "pid", project, "uid", user, "lock", col_lock, "transformations", tt.toString());
+            
+            logger.info("--- make sure transformation was recorded properly ---");
+            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "0");
+            tt = result.getJSONArray("transformations");
+            Assert.assertEquals(tt.length(), 2);
+
+            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "1");
+            tt = result.getJSONArray("transformations");
+            Assert.assertEquals(tt.length(), 1);
+            t = tt.getJSONObject(0);
+            assertJSON(t, "op_value", column + "," + cell);
+            
+            logger.info("--- make sure another user fails to acquire ALL lock ---");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
+
+            logger.info("--- make sure another user fails to acquire COL lock on the same column ---");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", column);
+
+            logger.info("--- make sure another user manages to acquire COL lock on another column ---");
+            String column2 = "2";
+            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", column2);
+            String col_lock2 = result.getString("lock");
+            
+            logger.info("--- make sure that both locks are present ---");
+            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "2");
+            locks = result.getJSONArray("locks");
+            Assert.assertEquals(locks.length(), 2);
+            
+            logger.info("--- make sure we can't escalate our current COL lock to an ALL lock ---");
+            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
+            
+            logger.info("--- release column locks ---");
+            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", col_lock);
+            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", col_lock2);
+            
+            logger.info("--- make sure the project has no locks ---");
+            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "2");
+            locks = result.getJSONArray("locks");
+            Assert.assertEquals(locks.length(), 0);
+            
+        } catch (Exception e) {
+            Assert.fail();
+        }
+    }
+    
+    // ------------------------------------------------------------------------------------
+    
+    private void assertJSON(JSONObject o, String name, String value) throws JSONException {
+        Assert.assertEquals(o.get(name), value);
+    }
+    
+    private JSONObject success(GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
+        return call(true, broker, request, response, service, params);
+    }
+
+    private JSONObject failure(GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
+        return call(false, broker, request, response, service, params);
+    }
+    
+    private JSONObject call(boolean successful, GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
+        if (params != null) {
+            for (int i = 0; i < params.length; ) {
+                String name = params[i++];
+                String value = params[i++];
+                if ("data".equals(name)) {
+                    final ByteArrayInputStream inputStream = new ByteArrayInputStream(value.getBytes("UTF-8"));
+                    when(request.getInputStream()).thenReturn(new ServletInputStream() {
+                        public int read() throws IOException {
+                            return inputStream.read();
+                        }
+                    });                    
+                } else {
+                    when(request.getParameter(name)).thenReturn(value);
+                }
+            }
+        }
+
+        StringWriter writer = new StringWriter();
+        when(response.getWriter()).thenReturn(new PrintWriter(writer));
+        
+        broker.process(service, request, response);
+
+        JSONObject result = new JSONObject(writer.toString());
+        
+        if (successful) {
+            assertJSON(result, "status", "ok");
+        } else {
+            assertJSON(result, "status", "error");
+        }
+        
+        logger.info(result.toString());
+
+        return result;
+    }
+}

broker/core/tests/src/com/metaweb/gridworks/broker/tests/GridworksBrokerTests.java

@@ -1,458 +0,0 @@
-package com.metaweb.gridworks.broker.tests;
-
-import static com.metaweb.gridworks.broker.GridworksBroker.*;
-import static com.metaweb.gridworks.broker.GridworksBroker.EXPIRE;
-import static com.metaweb.gridworks.broker.GridworksBroker.GET_STATE;
-import static com.metaweb.gridworks.broker.GridworksBroker.OBTAIN_LOCK;
-import static com.metaweb.gridworks.broker.GridworksBroker.RELEASE_LOCK;
-import static com.metaweb.gridworks.broker.GridworksBroker.START;
-import static com.metaweb.gridworks.broker.GridworksBroker.OPEN;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-
-import javax.servlet.ServletConfig;
-import javax.servlet.ServletInputStream;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.AfterSuite;
-import org.testng.annotations.AfterTest;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.BeforeSuite;
-import org.testng.annotations.BeforeTest;
-import org.testng.annotations.Test;
-
-import com.metaweb.gridworks.broker.GridworksBroker;
-import com.metaweb.gridworks.broker.GridworksBrokerImpl;
-
-public class GridworksBrokerTests {
-
-    Logger logger;
-    File data;
-    
-    @BeforeSuite public void suite_init() {
-        System.setProperty("log4j.configuration", "tests.log4j.properties");
-        data = new File("data");
-        if (!data.exists()) data.mkdirs();
-    }
-
-    @AfterSuite public void suite_destroy() {
-        for (File f : data.listFiles()) {
-            f.delete();
-        }
-        data.delete();
-    }
-
-    // ------------------------------------------------------------------------------------
-    
-    ServletConfig config = null;
-    GridworksBroker broker = null;
-    
-    @BeforeTest public void test_init() throws Exception {
-        logger = LoggerFactory.getLogger(this.getClass());
-        config = mock(ServletConfig.class);
-        when(config.getInitParameter("gridworks.data")).thenReturn(data.getAbsolutePath());
-        when(config.getInitParameter("gridworks.development")).thenReturn("true");
-
-        broker = new GridworksBrokerImpl();
-        broker.init(config);
-    }
-
-    @AfterTest public void test_destroy() throws Exception {
-        broker.destroy();
-        broker = null;
-        config = null;
-    }
-    
-    // ------------------------------------------------------------------------------------
-
-    HttpServletRequest request = null;
-    HttpServletResponse response = null;
-    StringWriter writer = null;
-    
-    @BeforeMethod public void setup() throws Exception {
-        request = mock(HttpServletRequest.class);
-        response = mock(HttpServletResponse.class);
-    }
-
-    @AfterMethod public void teardown() throws Exception {
-        response = null;
-        request = null;
-    }
-    
-    // ------------------------------------------------------------------------------------
-
-    @Test public void testLifeCycle() {
-        Assert.assertTrue(true);
-    }
-
-    @Test public void testService() {
-        try {
-            success(broker, request, response, EXPIRE);
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-
-    @Test public void testObtainLockFailure() {
-        try {
-            failure(broker, request, response, OBTAIN_LOCK);
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-
-    @Test public void testReleaseLockFailure() {
-        try {
-            failure(broker, request, response, RELEASE_LOCK);
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-
-    @Test public void testGetStateFailure() {
-        try {
-            failure(broker, request, response, GET_STATE, "pid", "project1934983948", "uid", "testuser", "rev", "0");
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-
-    @Test public void testBrokenAllLockFailure() {
-        try {
-            failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(ALL), "lockvalue", "1");
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-    
-    @Test public void testBrokenColLockFailure() {
-        try {
-            failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(COL), "lockvalue", "1,1");
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-    
-    @Test public void testBrokenCellLockFailure() {
-        try {
-            failure(broker, request, response, OBTAIN_LOCK, "pid", "project", "uid", "testuser", "locktype", Integer.toString(CELL), "lockvalue", "1");
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-    
-    @Test public void testLockSimple() {
-        String project = "proj0";
-        String user = "testuser";
-        
-        try {
-            logger.info("--- obtain ALL lock on project ---");
-            JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
-            assertJSON(result, "uid", "testuser");
-            String lock = result.getString("lock");
-    
-            logger.info("--- obtain ALL lock on project ---");
-            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
-
-            logger.info("--- obtain COL lock on project ---");
-            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
-            assertJSON(result, "uid", "testuser");
-            lock = result.getString("lock");
-    
-            logger.info("--- release COL lock on project ---");
-            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
-
-            logger.info("--- obtain CELL lock on project ---");
-            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
-            assertJSON(result, "uid", "testuser");
-            lock = result.getString("lock");
-            
-            logger.info("--- release CELL lock on project ---");
-            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-    
-    @Test public void testLocksAllBlocks() {
-        String project = "proj1";
-        String user = "testuser";
-        String user2 = "testuser2";
-
-        try {
-            logger.info("--- obtain ALL lock on project ---");
-            JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
-            assertJSON(result, "uid", user);
-            String lock = result.getString("lock");
-
-            logger.info("--- another using asking for any lock will fail ---");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
-
-            logger.info("--- same user asking for lower capable locks will return the ALL one ---");
-            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
-            String lock2 = result.getString("lock");
-            Assert.assertEquals(lock, lock2);
-            
-            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
-            lock2 = result.getString("lock");
-            Assert.assertEquals(lock, lock2);
-            
-            logger.info("--- release the ALL lock ---");
-            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-
-    @Test public void testLocksColBlocks() {
-        String project = "proj2";
-        String user = "testuser";
-        String user2 = "testuser2";
-
-        try {
-            logger.info("--- obtain COL lock on project ---");
-            JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "1");
-            String lock = result.getString("lock");
-
-            logger.info("--- other user must fail to obtain lock on the same COL or ALL ---");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
-            
-            logger.info("--- but succeed in getting a COL lock on another column or cell ---");
-            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "2");
-            String lock2 = result.getString("lock");
-            
-            logger.info("--- now it's our first user's turn to fail to get lock ---");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "2");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
-            
-            logger.info("--- release the locks ---");
-            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
-            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", lock2);
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-
-    @Test public void testLocksCellBlocks() {
-        String project = "proj3";
-        String user = "testuser";
-        String user2 = "testuser2";
-
-        try {
-            logger.info("--- obtain CELL lock on project ---");
-            JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
-            String lock = result.getString("lock");
-
-            logger.info("--- other user must fail to obtain lock on the same CELL, COL or ALL ---");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", "1");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "1,1");
-            
-            logger.info("--- but succeed in getting a CELL lock on a cell in another column ---");
-            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
-            String lock2 = result.getString("lock");
-            
-            logger.info("--- now it's our first user's turn to fail to get lock ---");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", "2");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(CELL), "lockvalue", "2,1");
-            
-            logger.info("--- release the locks ---");
-            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
-            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", lock2);
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-    
-    @Test public void testCompleteProjectLifeCycle() {
-        try {
-            String project = "proj4";
-            String user = "testuser";
-            String user2 = "testuser2";
-            String data = "blah";
-            String metadata = "{}";
-            String transformations = "[]";
-            String rev = "0";
-            
-            logger.info("--- obtain ALL lock on project ---");
-            JSONObject result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
-            assertJSON(result, "uid", user);
-            String lock = result.getString("lock");
-
-            logger.info("--- start project ---");
-            success(broker, request, response, START, "pid", project, "uid", user, "lock", lock, "data", data, "metadata", metadata, "transformations", transformations);
-
-            logger.info("--- verify project state contains lock ---");
-            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev);
-            JSONArray locks = result.getJSONArray("locks");
-            Assert.assertEquals(locks.length(), 1);
-            JSONObject l = locks.getJSONObject(0);
-            assertJSON(l, "uid", "testuser");
-            Assert.assertEquals(l.getInt("type"), ALL);
-            
-            logger.info("--- release ALL lock on project ---");
-            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", lock);
-            
-            logger.info("--- verify no locks are present ---");
-            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", rev);
-            locks = result.getJSONArray("locks");
-            Assert.assertEquals(locks.length(), 0);
-            
-            logger.info("--- open project and verify data was loaded correctly ---");
-            result = success(broker, request, response, OPEN, "pid", project, "uid", user, "rev", rev);
-            JSONArray result_data = result.getJSONArray("data");
-            Assert.assertEquals(result_data.length(),data.getBytes("UTF-8").length);
-
-            JSONArray tt;
-            JSONObject t;
-            
-            logger.info("--- obtain column lock ---");
-            String column = "1";
-            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(COL), "lockvalue", column);
-            String col_lock = result.getString("lock");
-
-            logger.info("--- perform column transformation ---");
-            t = new JSONObject();
-            t.put("op_type", COL);
-            t.put("op_value", column); // operate on col 1
-            t.put("value", new JSONObject());
-            tt = new JSONArray();
-            tt.put(t);
-            result = success(broker, request, response, TRANSFORM, "pid", project, "uid", user, "lock", col_lock, "transformations", tt.toString());
-
-            logger.info("--- make sure transformation was recorded properly ---");
-            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "0");
-            tt = result.getJSONArray("transformations");
-            Assert.assertEquals(tt.length(), 1);
-            t = tt.getJSONObject(0);
-            assertJSON(t, "op_value", column);
-
-            logger.info("--- make sure revision numbers in state management work as expected ---");
-            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "1");
-            tt = result.getJSONArray("transformations");
-            Assert.assertEquals(tt.length(), 0);
-            
-            logger.info("--- perform cell transformation ---");
-            String cell = "1";
-            t = new JSONObject();
-            t.put("op_type", CELL);
-            t.put("op_value", column + "," + cell); // operate on cell at row 1 column 1
-            t.put("value", new JSONObject());
-            tt = new JSONArray();
-            tt.put(t);
-            result = success(broker, request, response, TRANSFORM, "pid", project, "uid", user, "lock", col_lock, "transformations", tt.toString());
-            
-            logger.info("--- make sure transformation was recorded properly ---");
-            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "0");
-            tt = result.getJSONArray("transformations");
-            Assert.assertEquals(tt.length(), 2);
-
-            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "1");
-            tt = result.getJSONArray("transformations");
-            Assert.assertEquals(tt.length(), 1);
-            t = tt.getJSONObject(0);
-            assertJSON(t, "op_value", column + "," + cell);
-            
-            logger.info("--- make sure another user fails to acquire ALL lock ---");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(ALL), "lockvalue", "");
-
-            logger.info("--- make sure another user fails to acquire COL lock on the same column ---");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", column);
-
-            logger.info("--- make sure another user manages to acquire COL lock on another column ---");
-            String column2 = "2";
-            result = success(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user2, "locktype", Integer.toString(COL), "lockvalue", column2);
-            String col_lock2 = result.getString("lock");
-            
-            logger.info("--- make sure that both locks are present ---");
-            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "2");
-            locks = result.getJSONArray("locks");
-            Assert.assertEquals(locks.length(), 2);
-            
-            logger.info("--- make sure we can't escalate our current COL lock to an ALL lock ---");
-            failure(broker, request, response, OBTAIN_LOCK, "pid", project, "uid", user, "locktype", Integer.toString(ALL), "lockvalue", "");
-            
-            logger.info("--- release column locks ---");
-            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user, "lock", col_lock);
-            success(broker, request, response, RELEASE_LOCK, "pid", project, "uid", user2, "lock", col_lock2);
-            
-            logger.info("--- make sure the project has no locks ---");
-            result = success(broker, request, response, GET_STATE, "pid", project, "uid", user, "rev", "2");
-            locks = result.getJSONArray("locks");
-            Assert.assertEquals(locks.length(), 0);
-            
-        } catch (Exception e) {
-            Assert.fail();
-        }
-    }
-    
-    // ------------------------------------------------------------------------------------
-    
-    private void assertJSON(JSONObject o, String name, String value) throws JSONException {
-        Assert.assertEquals(o.get(name), value);
-    }
-    
-    private JSONObject success(GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
-        return call(true, broker, request, response, service, params);
-    }
-
-    private JSONObject failure(GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
-        return call(false, broker, request, response, service, params);
-    }
-    
-    private JSONObject call(boolean successful, GridworksBroker broker, HttpServletRequest request, HttpServletResponse response, String service, String... params) throws Exception {
-        if (params != null) {
-            for (int i = 0; i < params.length; ) {
-                String name = params[i++];
-                String value = params[i++];
-                if ("data".equals(name)) {
-                    final ByteArrayInputStream inputStream = new ByteArrayInputStream(value.getBytes("UTF-8"));
-                    when(request.getInputStream()).thenReturn(new ServletInputStream() {
-                        public int read() throws IOException {
-                            return inputStream.read();
-                        }
-                    });                    
-                } else {
-                    when(request.getParameter(name)).thenReturn(value);
-                }
-            }
-        }
-
-        StringWriter writer = new StringWriter();
-        when(response.getWriter()).thenReturn(new PrintWriter(writer));
-        
-        broker.process(service, request, response);
-
-        JSONObject result = new JSONObject(writer.toString());
-        
-        if (successful) {
-            assertJSON(result, "status", "ok");
-        } else {
-            assertJSON(result, "status", "error");
-        }
-        
-        logger.info(result.toString());
-
-        return result;
-    }
-}

extensions/jython/module/MOD-INF/controller.js

@@ -1,10 +1,10 @@
 function init() {
     // Packages.java.lang.System.err.println("Initializing jython extension");
     
-    Packages.com.metaweb.gridworks.expr.MetaParser.registerLanguageParser(
+    Packages.com.google.gridworks.expr.MetaParser.registerLanguageParser(
         "jython",
         "Jython",
-        Packages.com.metaweb.gridworks.jython.JythonEvaluable.createParser(),
+        Packages.com.google.gridworks.jython.JythonEvaluable.createParser(),
         "return value"
     );
 }

extensions/jython/src/com/google/gridworks/jython/JythonEvaluable.java

@@ -0,0 +1,131 @@
+package com.google.gridworks.jython;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+
+import org.python.core.Py;
+import org.python.core.PyException;
+import org.python.core.PyFunction;
+import org.python.core.PyNone;
+import org.python.core.PyObject;
+import org.python.core.PyString;
+import org.python.util.PythonInterpreter;
+
+import com.google.gridworks.expr.EvalError;
+import com.google.gridworks.expr.Evaluable;
+import com.google.gridworks.expr.HasFields;
+import com.google.gridworks.expr.LanguageSpecificParser;
+import com.google.gridworks.expr.ParsingException;
+
+public class JythonEvaluable implements Evaluable {
+    
+    static public LanguageSpecificParser createParser() {
+        return new LanguageSpecificParser() {
+            
+            public Evaluable parse(String s) throws ParsingException {
+                return new JythonEvaluable(s);
+            }
+        };
+    }
+    
+    private static final String s_functionName = "___temp___";
+    
+    private static PythonInterpreter _engine; 
+    
+    // FIXME(SM): this initialization logic depends on the fact that the JVM's 
+    // current working directory is the root of the Gridworks distributions
+    // or the development checkouts. While this works in practice, it would
+    // be preferable to have a more reliable address space, but since we
+    // don't have access to the servlet context from this class this is
+    // the best we can do for now.
+    static {
+        File libPath = new File("webapp/WEB-INF/lib/jython");
+        if (!libPath.exists() && !libPath.canRead()) {
+            libPath = new File("main/webapp/WEB-INF/lib/jython");
+            if (!libPath.exists() && !libPath.canRead()) {
+                libPath = null;
+            }
+        }
+        
+        if (libPath != null) {
+            Properties props = new Properties();
+            props.setProperty("python.path", libPath.getAbsolutePath());
+            PythonInterpreter.initialize(System.getProperties(), props, new String[] { "" });
+        }
+        
+        _engine = new PythonInterpreter();
+    }
+
+    public JythonEvaluable(String s) {
+        // indent and create a function out of the code
+        String[] lines = s.split("\r\n|\r|\n");
+        
+        StringBuffer sb = new StringBuffer(1024);
+        sb.append("def ");
+        sb.append(s_functionName);
+        sb.append("(value, cell, cells, row, rowIndex):");
+        for (int i = 0; i < lines.length; i++) {
+            sb.append("\n  ");
+            sb.append(lines[i]);
+        }
+
+        _engine.exec(sb.toString());
+    }
+    
+    public Object evaluate(Properties bindings) {
+        try {
+            // call the temporary PyFunction directly
+            Object result = ((PyFunction)_engine.get(s_functionName)).__call__(
+                new PyObject[] {
+                    Py.java2py( bindings.get("value") ),
+                    new JythonHasFieldsWrapper((HasFields) bindings.get("cell"), bindings),
+                    new JythonHasFieldsWrapper((HasFields) bindings.get("cells"), bindings),
+                    new JythonHasFieldsWrapper((HasFields) bindings.get("row"), bindings),
+                    Py.java2py( bindings.get("rowIndex") )
+                }
+            );
+
+            return unwrap(result);
+        } catch (PyException e) {
+            return new EvalError(e.getMessage());
+        }
+    }
+    
+    protected Object unwrap(Object result) {
+        if (result != null) {
+            if (result instanceof JythonObjectWrapper) {
+                return ((JythonObjectWrapper) result)._obj;
+            } else if (result instanceof JythonHasFieldsWrapper) {
+                return ((JythonHasFieldsWrapper) result)._obj;
+            } else if (result instanceof PyString) {
+                return ((PyString) result).asString();
+            } else if (result instanceof PyObject) {
+                return unwrap((PyObject) result);
+            }
+        }
+        
+        return result;      
+    }
+    
+    protected Object unwrap(PyObject po) {
+        if (po instanceof PyNone) {
+            return null;
+        } else if (po.isNumberType()) {
+            return po.asDouble();
+        } else if (po.isSequenceType()) {
+            Iterator<PyObject> i = po.asIterable().iterator();
+            
+            List<Object> list = new ArrayList<Object>();
+            while (i.hasNext()) {
+                list.add(unwrap((Object) i.next()));
+            }
+            
+            return list.toArray();
+        } else {
+            return po;
+        }
+    }
+}

extensions/jython/src/com/google/gridworks/jython/JythonHasFieldsWrapper.java

@@ -0,0 +1,40 @@
+package com.google.gridworks.jython;
+
+import java.util.Properties;
+
+import org.python.core.Py;
+import org.python.core.PyObject;
+
+import com.google.gridworks.expr.HasFields;
+
+public class JythonHasFieldsWrapper extends PyObject {
+    private static final long serialVersionUID = -1275353513262385099L;
+
+    public HasFields _obj;
+
+    private Properties _bindings;
+
+    public JythonHasFieldsWrapper(HasFields obj, Properties bindings) {
+        _obj = obj;
+        _bindings = bindings;
+    }
+
+    public PyObject __finditem__(PyObject key) {
+        String k = (String) key.__tojava__(String.class);
+        Object v = _obj.getField(k, _bindings);
+        if (v != null) {
+            if (v instanceof PyObject) {
+                return (PyObject) v;
+            } else if (v instanceof HasFields) {
+                return new JythonHasFieldsWrapper((HasFields) v, _bindings);
+            } else if (Py.getAdapter().canAdapt(v)) {
+                return Py.java2py(v);
+            } else {
+                return new JythonObjectWrapper(v);
+            }
+        } else {
+            return null;
+        }
+    }
+
+}

extensions/jython/src/com/google/gridworks/jython/JythonObjectWrapper.java

@@ -0,0 +1,17 @@
+package com.google.gridworks.jython;
+
+import org.python.core.PyObject;
+
+public class JythonObjectWrapper extends PyObject {
+    private static final long serialVersionUID = -6608115027151667441L;
+
+    public Object _obj;
+
+    public JythonObjectWrapper(Object obj) {
+        _obj = obj;
+    }
+
+    public String toString() {
+        return _obj.getClass().getSimpleName();
+    }
+}

extensions/jython/src/com/metaweb/gridworks/jython/JythonEvaluable.java

@@ -1,131 +0,0 @@
-package com.metaweb.gridworks.jython;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Properties;
-
-import org.python.core.Py;
-import org.python.core.PyException;
-import org.python.core.PyFunction;
-import org.python.core.PyNone;
-import org.python.core.PyObject;
-import org.python.core.PyString;
-import org.python.util.PythonInterpreter;
-
-import com.metaweb.gridworks.expr.EvalError;
-import com.metaweb.gridworks.expr.Evaluable;
-import com.metaweb.gridworks.expr.HasFields;
-import com.metaweb.gridworks.expr.LanguageSpecificParser;
-import com.metaweb.gridworks.expr.ParsingException;
-
-public class JythonEvaluable implements Evaluable {
-    
-    static public LanguageSpecificParser createParser() {
-        return new LanguageSpecificParser() {
-            
-            public Evaluable parse(String s) throws ParsingException {
-                return new JythonEvaluable(s);
-            }
-        };
-    }
-    
-    private static final String s_functionName = "___temp___";
-    
-    private static PythonInterpreter _engine; 
-    
-    // FIXME(SM): this initialization logic depends on the fact that the JVM's 
-    // current working directory is the root of the Gridworks distributions
-    // or the development checkouts. While this works in practice, it would
-    // be preferable to have a more reliable address space, but since we
-    // don't have access to the servlet context from this class this is
-    // the best we can do for now.
-    static {
-        File libPath = new File("webapp/WEB-INF/lib/jython");
-        if (!libPath.exists() && !libPath.canRead()) {
-            libPath = new File("main/webapp/WEB-INF/lib/jython");
-            if (!libPath.exists() && !libPath.canRead()) {
-                libPath = null;
-            }
-        }
-        
-        if (libPath != null) {
-            Properties props = new Properties();
-            props.setProperty("python.path", libPath.getAbsolutePath());
-            PythonInterpreter.initialize(System.getProperties(), props, new String[] { "" });
-        }
-        
-        _engine = new PythonInterpreter();
-    }
-
-    public JythonEvaluable(String s) {
-        // indent and create a function out of the code
-        String[] lines = s.split("\r\n|\r|\n");
-        
-        StringBuffer sb = new StringBuffer(1024);
-        sb.append("def ");
-        sb.append(s_functionName);
-        sb.append("(value, cell, cells, row, rowIndex):");
-        for (int i = 0; i < lines.length; i++) {
-            sb.append("\n  ");
-            sb.append(lines[i]);
-        }
-
-        _engine.exec(sb.toString());
-    }
-    
-    public Object evaluate(Properties bindings) {
-        try {
-            // call the temporary PyFunction directly
-            Object result = ((PyFunction)_engine.get(s_functionName)).__call__(
-                new PyObject[] {
-                    Py.java2py( bindings.get("value") ),
-                    new JythonHasFieldsWrapper((HasFields) bindings.get("cell"), bindings),
-                    new JythonHasFieldsWrapper((HasFields) bindings.get("cells"), bindings),
-                    new JythonHasFieldsWrapper((HasFields) bindings.get("row"), bindings),
-                    Py.java2py( bindings.get("rowIndex") )
-                }
-            );
-
-            return unwrap(result);
-        } catch (PyException e) {
-            return new EvalError(e.getMessage());
-        }
-    }
-    
-    protected Object unwrap(Object result) {
-        if (result != null) {
-            if (result instanceof JythonObjectWrapper) {
-                return ((JythonObjectWrapper) result)._obj;
-            } else if (result instanceof JythonHasFieldsWrapper) {
-                return ((JythonHasFieldsWrapper) result)._obj;
-            } else if (result instanceof PyString) {
-                return ((PyString) result).asString();
-            } else if (result instanceof PyObject) {
-                return unwrap((PyObject) result);
-            }
-        }
-        
-        return result;      
-    }
-    
-    protected Object unwrap(PyObject po) {
-        if (po instanceof PyNone) {
-            return null;
-        } else if (po.isNumberType()) {
-            return po.asDouble();
-        } else if (po.isSequenceType()) {
-            Iterator<PyObject> i = po.asIterable().iterator();
-            
-            List<Object> list = new ArrayList<Object>();
-            while (i.hasNext()) {
-                list.add(unwrap((Object) i.next()));
-            }
-            
-            return list.toArray();
-        } else {
-            return po;
-        }
-    }
-}

extensions/jython/src/com/metaweb/gridworks/jython/JythonHasFieldsWrapper.java

@@ -1,40 +0,0 @@
-package com.metaweb.gridworks.jython;
-
-import java.util.Properties;
-
-import org.python.core.Py;
-import org.python.core.PyObject;
-
-import com.metaweb.gridworks.expr.HasFields;
-
-public class JythonHasFieldsWrapper extends PyObject {
-    private static final long serialVersionUID = -1275353513262385099L;
-
-    public HasFields _obj;
-
-    private Properties _bindings;
-
-    public JythonHasFieldsWrapper(HasFields obj, Properties bindings) {
-        _obj = obj;
-        _bindings = bindings;
-    }
-
-    public PyObject __finditem__(PyObject key) {
-        String k = (String) key.__tojava__(String.class);
-        Object v = _obj.getField(k, _bindings);
-        if (v != null) {
-            if (v instanceof PyObject) {
-                return (PyObject) v;
-            } else if (v instanceof HasFields) {
-                return new JythonHasFieldsWrapper((HasFields) v, _bindings);
-            } else if (Py.getAdapter().canAdapt(v)) {
-                return Py.java2py(v);
-            } else {
-                return new JythonObjectWrapper(v);
-            }
-        } else {
-            return null;
-        }
-    }
-
-}

extensions/jython/src/com/metaweb/gridworks/jython/JythonObjectWrapper.java

@@ -1,17 +0,0 @@
-package com.metaweb.gridworks.jython;
-
-import org.python.core.PyObject;
-
-public class JythonObjectWrapper extends PyObject {
-    private static final long serialVersionUID = -6608115027151667441L;
-
-    public Object _obj;
-
-    public JythonObjectWrapper(Object obj) {
-        _obj = obj;
-    }
-
-    public String toString() {
-        return _obj.getClass().getSimpleName();
-    }
-}

extensions/rdf-exporter/.project

@@ -1,17 +1,17 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <projectDescription>
-	<name>gridworks-rdf-exporter</name>
-	<comment></comment>
-	<projects>
-	</projects>
-	<buildSpec>
-		<buildCommand>
-			<name>org.eclipse.jdt.core.javabuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-	</buildSpec>
-	<natures>
-		<nature>org.eclipse.jdt.core.javanature</nature>
-	</natures>
-</projectDescription>
+        <name>gridworks-rdf-exporter</name>
+        <comment></comment>
+        <projects>
+        </projects>
+        <buildSpec>
+                <buildCommand>
+                        <name>org.eclipse.jdt.core.javabuilder</name>
+                        <arguments>
+                        </arguments>
+                </buildCommand>
+        </buildSpec>
+        <natures>
+                <nature>org.eclipse.jdt.core.javanature</nature>
+        </natures>
+</projectDescription>

extensions/rdf-exporter/module/MOD-INF/controller.js

@@ -1,5 +1,5 @@
 
-importPackage(com.metaweb.gridworks.rdf.commands);
+importPackage(com.google.gridworks.rdf.commands);
 
 /*
  * Function invoked to initialize the extension.
@@ -8,21 +8,21 @@ function init() {
     /*
      *  Attach an rdf schema to each project.
      */
-    Packages.com.metaweb.gridworks.model.Project.registerOverlayModel(
+    Packages.com.google.gridworks.model.Project.registerOverlayModel(
         "rdfSchema",
-        Packages.com.metaweb.gridworks.rdf.RdfSchema);
+        Packages.com.google.gridworks.rdf.RdfSchema);
     
     /*
      *  Operations
      */
-    Packages.com.metaweb.gridworks.operations.OperationRegistry.register(
-        "save-rdf-schema", Packages.com.metaweb.gridworks.rdf.operations.SaveRdfSchemaOperation);
+    Packages.com.google.gridworks.operations.OperationRegistry.register(
+        "save-rdf-schema", Packages.com.google.gridworks.rdf.operations.SaveRdfSchemaOperation);
     
     /*
      *  Exporters
      */
-    var ExportRowsCommand = Packages.com.metaweb.gridworks.commands.project.ExportRowsCommand;
-    var RdfExporter = Packages.com.metaweb.gridworks.rdf.exporters.RdfExporter;
+    var ExportRowsCommand = Packages.com.google.gridworks.commands.project.ExportRowsCommand;
+    var RdfExporter = Packages.com.google.gridworks.rdf.exporters.RdfExporter;
     
     ExportRowsCommand.registerExporter("rdf", new RdfExporter("RDF/XML"));
     ExportRowsCommand.registerExporter("n3", new RdfExporter("N3"));
@@ -30,16 +30,16 @@ function init() {
     /*
      *  GEL Functions and Binders
      */
-    Packages.com.metaweb.gridworks.gel.ControlFunctionRegistry.registerFunction(
-        "urlify", new Packages.com.metaweb.gridworks.rdf.expr.functions.strings.Urlify());
+    Packages.com.google.gridworks.gel.ControlFunctionRegistry.registerFunction(
+        "urlify", new Packages.com.google.gridworks.rdf.expr.functions.strings.Urlify());
         
-    Packages.com.metaweb.gridworks.expr.ExpressionUtils.registerBinder(
-        new Packages.com.metaweb.gridworks.rdf.expr.RdfBinder());
+    Packages.com.google.gridworks.expr.ExpressionUtils.registerBinder(
+        new Packages.com.google.gridworks.rdf.expr.RdfBinder());
         
     /*
      *  Commands
      */
-    var GridworksServlet = Packages.com.metaweb.gridworks.GridworksServlet;
+    var GridworksServlet = Packages.com.google.gridworks.GridworksServlet;
     GridworksServlet.registerCommand("save-rdf-schema", new SaveRdfSchemaCommand());
     GridworksServlet.registerCommand("preview-rdf",new PreviewRdfCommand());
     GridworksServlet.registerCommand("save-baseURI",new SaveBaseURI());
@@ -51,7 +51,7 @@ function init() {
     /*
      *  Client-side Resources
      */
-    var ClientSideResourceManager = Packages.com.metaweb.gridworks.ClientSideResourceManager;
+    var ClientSideResourceManager = Packages.com.google.gridworks.ClientSideResourceManager;
     
     // Script files to inject into /project page
     ClientSideResourceManager.addPaths(

extensions/rdf-exporter/src/com/google/gridworks/rdf/CellBlankNode.java

@@ -0,0 +1,39 @@
+package com.google.gridworks.rdf;
+
+import java.net.URI;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.Resource;
+
+public class CellBlankNode extends ResourceNode{
+
+    final public int columnIndex;
+    final public String columnName;
+    
+    public CellBlankNode(int i,String columnName){
+        this.columnIndex = i;
+        this.columnName = columnName;
+    }
+    
+    @Override
+    public Resource createResource(URI baseUri, Model model, Project project,
+            Row row, int rowIndex,Resource[] blanks) {
+        return model.createResource();
+    }
+
+    @Override
+    protected void writeNode(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.key("nodeType");writer.value("cell-as-blank");
+        writer.key("columnIndex");writer.value(columnIndex);
+        writer.key("columnName");writer.value(columnName);
+    }
+
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/CellLiteralNode.java

@@ -0,0 +1,78 @@
+package com.google.gridworks.rdf;
+
+import java.net.URI;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+import com.hp.hpl.jena.rdf.model.Literal;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.RDFNode;
+import com.hp.hpl.jena.rdf.model.Resource;
+
+public class CellLiteralNode extends CellNode{
+
+    private String valueType;
+    private String lang;
+    public String getValueType() {
+        return valueType;
+    }
+    public void setValueType(String valueType) {
+        this.valueType = valueType;
+    }
+    public String getLang() {
+        return lang;
+    }
+    public void setLang(String lang) {
+        this.lang = lang;
+    }
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.object();
+        writer.key("nodeType");writer.value("cell-as-literal");
+        writer.key("columnIndex");writer.value(columnIndex);
+        writer.key("columnName");writer.value(columnName);
+        if(valueType!=null){
+            writer.key("valueType");writer.value(valueType);
+        }
+        if(lang!=null){
+            writer.key("lang");writer.value(lang);
+        }
+        writer.endObject();
+    }
+    
+    public CellLiteralNode(int index,String columnName){
+        super(index,columnName);
+    }
+    
+    public CellLiteralNode(int index,String columnName,String valueType,String lang){
+        this(index,columnName);
+        this.lang = lang;
+        this.valueType = valueType;
+    }
+    public RDFNode createNode(URI baseUri, Model model, Project project,
+            Row row, int rowIndex,Resource[] blanks) {
+        String val;
+        try{
+            val= row.getCell(this.columnIndex).value.toString();
+        }catch(NullPointerException ne){
+            return null;
+        }
+        if(val!=null && val.length()>0){
+            //TODO language and datatype
+            Literal l;
+            if(this.valueType!=null){
+                l = model.createTypedLiteral(val, valueType);
+            }else{
+                l = model.createLiteral(val);
+            }
+            return l;
+        }else{
+            return null;
+        }
+    }
+
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/CellNode.java

@@ -0,0 +1,12 @@
+package com.google.gridworks.rdf;
+
+public abstract class CellNode implements Node{
+    final public int columnIndex;
+    final public String columnName;
+    
+    public CellNode(int i,String columnName){
+        this.columnIndex = i;
+        this.columnName = columnName; 
+    }
+
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/CellResourceNode.java

@@ -0,0 +1,80 @@
+package com.google.gridworks.rdf;
+
+import java.net.URI;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+
+import com.google.gridworks.expr.Evaluable;
+import com.google.gridworks.expr.ExpressionUtils;
+import com.google.gridworks.expr.MetaParser;
+import com.google.gridworks.model.Cell;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.Resource;
+
+
+public class CellResourceNode extends ResourceNode{
+
+    private String uriExpression;
+    final public String columnName;
+    public String getUriExpression() {
+        return uriExpression;
+    }
+
+    
+    final public int columnIndex;
+    
+    public CellResourceNode(int i,String columnName){
+        this.columnIndex = i;
+        this.columnName = columnName;
+    }
+    
+    public CellResourceNode(int columnIndex,String columnName,String exp) {
+        this(columnIndex,columnName);
+        this.uriExpression = exp;
+    }
+
+    @Override
+    public Resource createResource(URI baseUri,Model model,Project project,Row row,int rowIndex,Resource[] blanks) {
+        try{
+            Properties bindings = ExpressionUtils.createBindings(project);
+            Evaluable eval = MetaParser.parse(uriExpression);
+            Cell cell = row.getCell(this.columnIndex);
+            String colName = this.columnIndex>-1?project.columnModel.getColumnByCellIndex(this.columnIndex).getName():"";
+            ExpressionUtils.bind(bindings, row, rowIndex,colName , cell);
+            Object result = eval.evaluate(bindings);
+            if(result.toString().length()>0){
+                String uri = Util.getUri(baseUri, result.toString());
+                Resource r =  model.createResource(uri);
+                return r;
+            }else{
+                return null;
+            }
+        }catch(Exception e){
+//            e.printStackTrace();
+            //an empty cell might result in an exception out of evaluating URI expression... so it is intended to eat the exception
+            return null;
+        }
+        
+    }
+
+    @Override
+    protected void writeNode(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.key("nodeType"); writer.value("cell-as-resource");
+        writer.key("uriExpression"); writer.value(uriExpression);
+        writer.key("columnIndex"); writer.value(columnIndex);
+        if(columnIndex==-1){
+            //Row number
+            writer.key("isRowNumberCell"); writer.value(true);
+        } else {
+            writer.key("columnName"); writer.value(columnName);
+        }
+    }
+    
+    
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/ConstantBlankNode.java

@@ -0,0 +1,33 @@
+package com.google.gridworks.rdf;
+
+import java.net.URI;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.Resource;
+
+public class ConstantBlankNode extends ResourceNode{
+
+    private int _id;
+    ConstantBlankNode(int id){
+        this._id = id;
+    }
+    
+    @Override
+    public Resource createResource(URI baseUri, Model model, Project project,
+            Row row, int rowIndex,Resource[] blanks) {
+        return blanks[this._id];
+    }
+
+    @Override
+    protected void writeNode(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.key("nodeType");writer.value("blank");
+    }
+
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/ConstantLiteralNode.java

@@ -0,0 +1,89 @@
+package com.google.gridworks.rdf;
+
+import java.net.URI;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+import com.hp.hpl.jena.rdf.model.Literal;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.RDFNode;
+import com.hp.hpl.jena.rdf.model.Resource;
+
+public class ConstantLiteralNode implements Node{
+
+    private String valueType;
+    private String lang;
+    private String value;
+    
+    
+    public ConstantLiteralNode(String val,String type,String l){
+        this.value = val;
+        this.valueType = type;
+        this.lang = l;
+    }
+    public String getValueType() {
+        return valueType;
+    }
+
+
+    public void setValueType(String valueType) {
+        this.valueType = valueType;
+    }
+
+
+    public String getLang() {
+        return lang;
+    }
+
+
+    public void setLang(String lang) {
+        this.lang = lang;
+    }
+
+
+    public String getValue() {
+        return value;
+    }
+
+
+    public void setValue(String value) {
+        this.value = value;
+    }
+
+
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.object();
+        writer.key("nodeType"); writer.value("literal");
+        writer.key("value"); writer.value(value);
+        if(valueType!=null){
+            writer.key("valueType");
+            writer.value(valueType);
+        }
+        if(lang!=null){
+            writer.key("lang");
+            writer.value(lang);
+        }
+        writer.endObject();
+    }
+    public RDFNode createNode(URI baseUri, Model model, Project project,
+            Row row, int rowIndex,Resource[] blanks) {
+        if(this.value!=null && this.value.length()>0){
+            
+            Literal l ;
+            if(this.valueType!=null){
+                l = model.createTypedLiteral(this.value, valueType);
+            }else{
+                l = model.createLiteral(this.value);
+            }
+            return l;
+        }else{
+            return null;
+        }
+    }
+
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/ConstantResourceNode.java

@@ -0,0 +1,60 @@
+package com.google.gridworks.rdf;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.Resource;
+
+public class ConstantResourceNode extends ResourceNode{
+
+    private String uri;
+
+
+    public String getUri() {
+        return uri;
+    }
+
+    public void setUri(String uri) {
+        this.uri = uri;
+    }
+    
+
+    public ConstantResourceNode(String uri){
+        this.uri = uri;
+    }
+
+    @Override
+    public Resource createResource(URI baseUri, Model model, Project project,
+            Row row, int rowIndex,Resource[] blanks) {
+        if(this.uri!=null & this.uri.length()>0){
+            String tmp;
+            try {
+                tmp = Util.getUri(baseUri, this.uri);
+            } catch (URISyntaxException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+                return null;
+            }
+            Resource r =  model.createResource(tmp);
+            return r;
+        }else{
+            return null;
+        }
+    }
+
+    @Override
+    protected void writeNode(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.key("nodeType"); writer.value("resource");
+        writer.key("uri"); writer.value(uri);
+    }
+    
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/Link.java

@@ -0,0 +1,36 @@
+package com.google.gridworks.rdf;
+
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.Jsonizable;
+
+public class Link implements Jsonizable {
+
+    public final String propertyUri;
+    public final String curie;
+    public final Node target;
+
+    public Link(String uri,String curie,Node t){
+        this.propertyUri = uri;
+        this.target = t;
+        this.curie = curie;
+    }
+    
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+
+        writer.object();
+        writer.key("uri");
+        writer.value(propertyUri);
+        writer.key("curie");
+        writer.value(curie);
+        if (target != null) {
+            writer.key("target");
+            target.write(writer, options);
+        }
+        writer.endObject();
+    }
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/Node.java

@@ -0,0 +1,15 @@
+package com.google.gridworks.rdf;
+
+import java.net.URI;
+
+import com.google.gridworks.Jsonizable;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.RDFNode;
+import com.hp.hpl.jena.rdf.model.Resource;
+
+public interface Node extends Jsonizable{
+
+    RDFNode createNode(URI baseUri,Model model,Project project,Row row,int rowIndex,Resource[] blanks);
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/RdfSchema.java

@@ -0,0 +1,180 @@
+package com.google.gridworks.rdf;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.gridworks.model.OverlayModel;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.rdf.ResourceNode.RdfType;
+
+public class RdfSchema implements OverlayModel {
+
+    final protected List<Node> _rootNodes = new ArrayList<Node>();
+    final protected List<ConstantBlankNode> _blanks = new ArrayList<ConstantBlankNode>();
+
+    public List<ConstantBlankNode> get_blanks() {
+        return _blanks;
+    }
+
+    protected URI baseUri;
+
+    public void setBaseUri(URI baseUri) {
+        this.baseUri = baseUri;
+    }
+
+    public void setBaseUri(String base) throws URISyntaxException {
+        this.baseUri = new URI(base);
+    }
+
+    public RdfSchema() {
+        // FIXME
+        try {
+            this.baseUri = new URI("http://localhost:3333/");
+        } catch (URISyntaxException ue) {
+        }
+    }
+
+    public URI getBaseUri() {
+        return baseUri;
+    }
+
+    public Node getRoot() {
+        return _rootNodes.get(0);
+    }
+
+    static public RdfSchema reconstruct(JSONObject o) throws JSONException {
+        RdfSchema s = new RdfSchema();
+        // TODO
+        try {
+            s.baseUri = new URI(o.getString("baseUri"));
+        } catch (URISyntaxException me) {
+            me.printStackTrace();
+        }
+
+        JSONArray rootNodes = o.getJSONArray("rootNodes");
+        int count = rootNodes.length();
+
+        for (int i = 0; i < count; i++) {
+            JSONObject o2 = rootNodes.getJSONObject(i);
+            Node node = reconstructNode(o2, s);
+            if (node != null) {
+                s._rootNodes.add(node);
+            }
+        }
+
+        return s;
+    }
+
+    static protected Node reconstructNode(JSONObject o, RdfSchema s)
+            throws JSONException {
+        Node node = null;
+        int blanksCount = 0;
+        String nodeType = o.getString("nodeType");
+        if (nodeType.startsWith("cell-as-")) {
+            int columnIndex = o.getInt("columnIndex");
+            String columnName = null;
+            if(columnIndex!=-1){
+                columnName = o.getString("columnName");
+            }
+
+            if ("cell-as-resource".equals(nodeType)) {
+                String exp = o.getString("uriExpression");
+                node = new CellResourceNode(columnIndex, columnName, exp);
+                if (o.has("rdfTypes")) {
+                    List<RdfType> types = reconstructTypes(o
+                            .getJSONArray("rdfTypes"));
+                    ((CellResourceNode) node).setTypes(types);
+                }
+            } else if ("cell-as-literal".equals(nodeType)) {
+                String valueType = o.has("valueType")?Util.getDataType(o.getString("valueType")):null;
+                String lang = o.has("lang") ? o.getString("lang"):null;
+                node = new CellLiteralNode(columnIndex, columnName, valueType, lang);
+            } else if ("cell-as-blank".equals(nodeType)) {
+                node = new CellBlankNode(columnIndex,columnName);
+                if (o.has("rdfTypes")) {
+                    List<RdfType> types = reconstructTypes(o
+                            .getJSONArray("rdfTypes"));
+                    ((CellBlankNode) node).setTypes(types);
+                }
+            }
+        } else if ("resource".equals(nodeType)) {
+            node = new ConstantResourceNode(o.getString("uri"));
+            if (o.has("rdfTypes")) {
+                List<RdfType> types = reconstructTypes(o
+                        .getJSONArray("rdfTypes"));
+                ((ConstantResourceNode) node).setTypes(types);
+            }
+        } else if ("literal".equals(nodeType)) {
+            String valueType = o.has("valueType")?Util.getDataType(o.getString("valueType")):null;
+            String lang = o.has("lang") ? o.getString("lang"):null;
+            node = new ConstantLiteralNode(o.getString("value"), valueType,lang);
+        } else if ("blank".equals(nodeType)) {
+            node = new ConstantBlankNode(blanksCount);
+            blanksCount += 1;
+            s._blanks.add((ConstantBlankNode) node);
+            if (o.has("rdfTypes")) {
+                List<RdfType> types = reconstructTypes(o
+                        .getJSONArray("rdfTypes"));
+                ((ConstantBlankNode) node).setTypes(types);
+            }
+        }
+
+        if (node != null && node instanceof ResourceNode && o.has("links")) {
+            ResourceNode node2 = (ResourceNode) node;
+
+            JSONArray links = o.getJSONArray("links");
+            int linkCount = links.length();
+
+            for (int j = 0; j < linkCount; j++) {
+                JSONObject oLink = links.getJSONObject(j);
+
+                node2.addLink(new Link(oLink.getString("uri"), oLink.getString("curie"),oLink
+                        .has("target")
+                        && !oLink.isNull("target") ? reconstructNode(oLink
+                        .getJSONObject("target"), s) : null));
+            }
+        }
+
+        return node;
+    }
+
+    static private List<RdfType> reconstructTypes(JSONArray arr)
+            throws JSONException {
+        List<RdfType> lst = new ArrayList<RdfType>();
+        for (int i = 0; i < arr.length(); i++) {
+            String uri = arr.getJSONObject(i).getString("uri");
+            String curie = arr.getJSONObject(i).getString("curie");
+            lst.add(new RdfType(uri, curie));
+        }
+        return lst;
+    }
+
+    @Override
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.object();
+        writer.key("baseUri"); writer.value(baseUri);
+        
+        writer.key("rootNodes");
+        writer.array();
+        
+        for (Node node : _rootNodes) {
+            node.write(writer, options);
+        }
+        
+        writer.endArray();
+        writer.endObject();
+    }
+    
+    static public RdfSchema load(Project project, JSONObject obj) throws Exception {
+        return reconstruct(obj);
+    }
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/ResourceNode.java

@@ -0,0 +1,133 @@
+package com.google.gridworks.rdf;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.Property;
+import com.hp.hpl.jena.rdf.model.RDFNode;
+import com.hp.hpl.jena.rdf.model.Resource;
+import com.hp.hpl.jena.vocabulary.RDF;
+
+
+abstract public class ResourceNode implements Node {
+
+    private List<Link> links = new ArrayList<Link>();
+    
+    private List<RdfType> rdfTypes = new ArrayList<RdfType>();
+    
+    public void addLink(Link link) {
+        this.links.add(link);
+    }
+
+    public void addType(RdfType type) {
+        this.rdfTypes.add(type);
+    }
+
+    public Link getLink(int index) {
+        return this.links.get(index);
+    }
+
+    public int getLinkCount() {
+        return this.links.size();
+    }
+
+    public List<RdfType> getTypes() {
+        return this.rdfTypes;
+    }
+
+    protected abstract void writeNode(JSONWriter writer, Properties options) throws JSONException;
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.object();
+        //writer node
+        writeNode(writer,options);
+        //write types
+        writer.key("rdfTypes");
+        writer.array();
+        for(RdfType type:this.getTypes()){
+            writer.object();
+            writer.key("uri");writer.value(type.uri);
+            writer.key("curie");writer.value(type.curie);
+            writer.endObject();
+        }
+        writer.endArray();
+        //write links
+        writer.key("links");
+        writer.array();
+        for(int i=0;i<getLinkCount();i++){
+            Link l = getLink(i);
+            /*writer.object();
+            writer.key("uri");writer.value(l.propertyUri);
+            writer.key("target");
+            l.target.write(writer, options);
+            writer.endObject();*/
+            
+            l.write(writer, options);
+        }
+        writer.endArray();
+        
+        writer.endObject();
+        
+    }
+
+    
+    protected void addTypes(Resource r,Model model){
+        for(RdfType type:this.getTypes()){
+            r.addProperty(RDF.type, model.createResource(type.uri));
+        }
+    }
+    
+    protected Resource addLinks(Resource r,URI baseUri,Model model,Project project,Row row,int rowIndex,Resource[] blanks){
+        for(int i=0;i<getLinkCount();i++){
+            Link l = getLink(i);
+            String propertyUri;
+            try {
+                propertyUri = Util.getUri(baseUri, l.propertyUri);
+                Property p = model.createProperty(propertyUri);
+                RDFNode o = l.target.createNode(baseUri, model, project, row, rowIndex,blanks);
+                if(o!=null){
+                    r.addProperty(p, o);
+                }
+            } catch (URISyntaxException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+                return null;
+            }
+        }
+        return r;
+    }
+
+    public void setTypes(List<RdfType> types) {
+        this.rdfTypes = types;
+    }
+    
+    public RDFNode createNode(URI baseUri,Model model,Project project,Row row,int rowIndex,Resource[] blanks) {
+        Resource r = createResource(baseUri, model, project, row, rowIndex,blanks);
+        if(r==null){
+            return null;
+        }
+        addTypes(r, model);
+        return addLinks(r,baseUri,model,project,row,rowIndex,blanks);
+    }
+    
+    public abstract Resource createResource(URI baseUri,Model model,Project project,Row row,int rowIndex,Resource[] blanks) ;
+    
+    public static class RdfType{
+        String uri;
+        String curie;
+        public RdfType(String uri,String curie){
+            this.uri = uri;
+            this.curie = curie;
+            
+        }
+    }
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/Util.java

@@ -0,0 +1,47 @@
+package com.google.gridworks.rdf;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import com.google.gridworks.model.Project;
+
+public class Util {
+
+    private static final String XSD_INT_URI ="http://www.w3.org/2001/XMLSchema#int";
+    private static final String XSD_DOUBLE_URI ="http://www.w3.org/2001/XMLSchema#double";
+    private static final String XSD_DATE_URI ="http://www.w3.org/2001/XMLSchema#date";
+    
+    public static String getUri(URI base,String rel)throws URISyntaxException{
+        return base.resolve(rel).toString();
+    }
+    
+    public static String getDataType(String s){
+        if(s==null){
+            return null;
+        }
+        if(s.equals(XSD_INT_URI)){
+            return XSD_INT_URI;
+        }
+        if(s.equals(XSD_DOUBLE_URI)){
+            return XSD_DOUBLE_URI;
+        }
+        if(s.equals(XSD_DATE_URI)){
+            return XSD_DATE_URI;
+        }
+        return null;
+    }
+    
+    public static RdfSchema getProjectSchema(Project project) {
+        synchronized (project) {
+            RdfSchema rdfSchema = (RdfSchema) project.overlayModels.get("rdfSchema");
+            if (rdfSchema == null) {
+                rdfSchema = new RdfSchema();
+                
+                project.overlayModels.put("rdfSchema", rdfSchema);
+                project.getMetadata().updateModified();
+            }
+            
+            return rdfSchema;
+        }
+    }
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/commands/DeleteVocabularyCommand.java

@@ -0,0 +1,42 @@
+package com.google.gridworks.rdf.commands;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.Jsonizable;
+import com.google.gridworks.commands.Command;
+import com.google.gridworks.rdf.vocab.VocabularyManager;
+
+public class DeleteVocabularyCommand extends Command{
+
+    @Override
+    public void doPost(HttpServletRequest request, HttpServletResponse response)
+            throws ServletException, IOException {
+        String uri = request.getParameter("uri");
+        try {    
+            VocabularyManager.getSingleton(servlet).deleteVocabulary(uri);
+            respondJSON(response, new Jsonizable() {
+                
+                @Override
+                public void write(JSONWriter writer, Properties options)
+                        throws JSONException {
+                    writer.object();
+                    writer.key("code"); writer.value("ok");
+                    writer.endObject();
+                }
+            });
+        } catch (JSONException e) {
+            respondException(response, e);
+        } catch (Exception e){
+            respondException(response, e);
+        }
+    }
+
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/commands/ImportVocabularyCommand.java

@@ -0,0 +1,48 @@
+package com.google.gridworks.rdf.commands;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.Jsonizable;
+import com.google.gridworks.commands.Command;
+import com.google.gridworks.rdf.vocab.VocabularyExistException;
+import com.google.gridworks.rdf.vocab.VocabularyManager;
+
+public class ImportVocabularyCommand extends Command{
+
+    @Override
+    public void doGet(HttpServletRequest request, HttpServletResponse response)
+            throws ServletException, IOException {
+        String prefix = request.getParameter("prefix");
+        String url = request.getParameter("url");
+        String namespace = request.getParameter("namespace");
+        try {    
+            VocabularyManager.getSingleton(servlet).addVocabulary(url, prefix, namespace);
+            respondJSON(response, new Jsonizable() {
+                
+                @Override
+                public void write(JSONWriter writer, Properties options)
+                        throws JSONException {
+                    writer.object();
+                    writer.key("code"); writer.value("ok");
+                    writer.endObject();
+                }
+            });
+        } catch (JSONException e) {
+            respondException(response, e);
+        } catch (VocabularyExistException e) {
+            respondException(response, e);
+        } catch (Exception e){
+            respondException(response, e);
+        }
+    }
+
+    
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/commands/ListVocabulariesCommand.java

@@ -0,0 +1,37 @@
+package com.google.gridworks.rdf.commands;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONWriter;
+
+import com.google.gridworks.commands.Command;
+import com.google.gridworks.rdf.vocab.Vocabulary;
+import com.google.gridworks.rdf.vocab.VocabularyManager;
+
+public class ListVocabulariesCommand extends Command{
+    @Override
+    public void doGet(HttpServletRequest request, HttpServletResponse response)
+            throws ServletException, IOException {
+        response.setCharacterEncoding("UTF-8");
+        response.setHeader("Content-Type", "application/json");
+        try{
+            JSONWriter writer = new JSONWriter(response.getWriter());
+            writer.object();
+            writer.key("vocabularies");
+            writer.array();
+            Properties p = new Properties();
+            for(Vocabulary v:VocabularyManager.getSingleton(servlet).getVocabularies()){
+                v.write(writer, p);
+            }
+            writer.endArray();
+            writer.endObject();
+        } catch (Exception e) {
+            respondException(response, e);
+        }
+    }
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/commands/PreviewRdfCommand.java

@@ -0,0 +1,106 @@
+package com.google.gridworks.rdf.commands;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.URI;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.gridworks.browsing.Engine;
+import com.google.gridworks.browsing.FilteredRows;
+import com.google.gridworks.browsing.RowVisitor;
+import com.google.gridworks.commands.Command;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+import com.google.gridworks.rdf.Node;
+import com.google.gridworks.rdf.RdfSchema;
+import com.google.gridworks.util.ParsingUtilities;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.ModelFactory;
+import com.hp.hpl.jena.rdf.model.Resource;
+
+public class PreviewRdfCommand extends Command {
+
+    @Override
+    public void doPost(HttpServletRequest request, HttpServletResponse response)
+            throws ServletException, IOException {
+        try {
+            Project project = getProject(request);
+            Engine engine = getEngine(request, project);
+            FilteredRows filteredRows = engine.getAllFilteredRows();
+            
+            response.setCharacterEncoding("UTF-8");
+            response.setHeader("Content-Type", "application/json");
+            
+            String jsonString = request.getParameter("schema");
+            JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
+            RdfSchema schema = RdfSchema.reconstruct(json);
+            
+            Model model = ModelFactory.createDefaultModel();
+            URI baseUri = schema.getBaseUri();
+            Node root = schema.getRoot();
+            
+            Resource[] blanks = new Resource[schema.get_blanks().size()];
+            for (int i = 0; i < blanks.length; i++) {
+                blanks[i] = model.createResource();
+            }
+            
+            RowVisitor visitor = new RdfRowVisitor(model, baseUri, root,blanks,20);
+            
+            filteredRows.accept(project, visitor);
+            StringWriter sw = new StringWriter();
+            model.write(sw,"N3");
+            
+            JSONWriter writer = new JSONWriter(response.getWriter());
+            writer.object();
+            writer.key("v");
+            writer.value(sw.getBuffer().toString());
+            writer.endObject();
+            //respond(response, "{v:" + sw.getBuffer().toString() + "}");
+        }catch (Exception e) {
+            respondException(response, e);
+        }
+    }
+    
+
+    protected static class RdfRowVisitor implements RowVisitor{
+        Model model;
+        URI base;
+        Node root;
+        Resource[] blanks;
+        int limit;
+        
+        int _count;
+        public RdfRowVisitor(Model m,URI base, Node root,Resource[] blanks,int l){
+            this.model = m;
+            this.base = base;
+            this.root = root;
+            this.blanks = blanks;
+            this.limit = l;
+        }
+        public void end(Project project) {
+            // do nothing
+            
+        }
+
+        public void start(Project project) {
+            // do nothing
+            
+        }
+
+        public boolean visit(Project project, int rowIndex, Row row) {
+            if(_count>=limit){
+                return true;
+            }
+            root.createNode(base, model, project, row, rowIndex,blanks);
+            _count +=1;
+            return false;
+        }
+    }
+
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/commands/SaveBaseURI.java

@@ -0,0 +1,32 @@
+package com.google.gridworks.rdf.commands;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.google.gridworks.commands.Command;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.rdf.Util;
+
+public class SaveBaseURI extends Command {
+
+    @Override
+    public void doPost(HttpServletRequest request, HttpServletResponse response)
+            throws ServletException, IOException {
+        try {
+            Project project = getProject(request);
+            String base = request.getParameter("baseURI");
+            
+            Util.getProjectSchema(project).setBaseUri(base);
+            
+            project.getMetadata().updateModified();
+            
+            respond(response,"OK","");
+            
+        }catch(Exception e){
+            respondException(response, e);
+        }
+    }
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/commands/SaveRdfSchemaCommand.java

@@ -0,0 +1,64 @@
+package com.google.gridworks.rdf.commands;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONObject;
+
+import com.google.gridworks.commands.Command;
+import com.google.gridworks.model.AbstractOperation;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.process.Process;
+import com.google.gridworks.rdf.RdfSchema;
+import com.google.gridworks.rdf.operations.SaveRdfSchemaOperation;
+import com.google.gridworks.util.ParsingUtilities;
+
+public class SaveRdfSchemaCommand extends Command{
+
+    @Override
+    public void doPost(HttpServletRequest request, HttpServletResponse response)
+            throws ServletException, IOException {
+        
+        try {
+            Project project = getProject(request);
+            
+            String jsonString = request.getParameter("schema");
+            JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
+            RdfSchema schema = RdfSchema.reconstruct(json);
+            
+            AbstractOperation op = new SaveRdfSchemaOperation(schema);
+            Process process = op.createProcess(project, new Properties());
+            
+            performProcessAndRespond(request, response, project, process);
+            
+            /*project.schema = schema;
+            
+            response.setCharacterEncoding("UTF-8");
+            response.setHeader("Content-Type", "application/json");
+            
+            Writer w = response.getWriter();
+            JSONWriter writer = new JSONWriter(w);
+            writer.object();
+            writer.key("code"); writer.value("ok");
+            writer.key("historyEntry"); 
+            
+            //dummy history for now
+            writer.object();
+            writer.key("op"); writer.value("saveRdfSchema");
+            writer.key("description"); writer.value("Save RDF schema");
+            writer.endObject();
+            
+            writer.endObject();
+            
+            w.flush();
+            w.close();*/
+            
+        } catch (Exception e) {
+            respondException(response, e);
+        }
+    }
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/commands/SuggestTermCommand.java

@@ -0,0 +1,99 @@
+package com.google.gridworks.rdf.commands;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.Jsonizable;
+import com.google.gridworks.commands.Command;
+import com.google.gridworks.rdf.vocab.RDFNode;
+import com.google.gridworks.rdf.vocab.VocabularyManager;
+
+public class SuggestTermCommand extends Command{
+
+    @Override
+    public void doGet(HttpServletRequest request, HttpServletResponse response)
+            throws ServletException, IOException {
+        
+        
+        response.setHeader("Content-Type", "application/json");
+        JSONWriter writer = new JSONWriter(response.getWriter());
+        
+        String type = request.getParameter("type_strict");
+        
+        String prefix = request.getParameter("prefix");
+        try{
+            writer.object();
+            
+            writer.key("prefix");
+            writer.value(prefix);
+            
+            writer.key("result");
+            writer.array();
+            List<RDFNode> nodes;
+            if(type!=null && type.trim().equals("property")){
+                nodes = VocabularyManager.getSingleton(servlet).searchProperties(prefix);
+            }else{
+                nodes = VocabularyManager.getSingleton(servlet).searchClasses(prefix);
+            }
+            for(RDFNode c:nodes){
+                c.writeAsSearchResult(writer);
+            }
+            writer.endArray();
+            writer.endObject();
+        }catch(Exception e){
+            e.printStackTrace();
+            throw new ServletException(e);
+        }
+    }
+
+    
+}
+
+class Result implements Jsonizable{
+
+    private List<String[]> results = new ArrayList<String[]>();
+    private String prefix;
+    
+    Result(String p){
+        this.prefix = p;
+    }
+    void addResult(String id, String name){
+        String[] res = new String[] {id,name};
+        results.add(res);
+    }
+    @Override
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.object();
+        
+        
+        writer.key("prefix");
+        writer.value(prefix);
+        
+        writer.key("result");
+        writer.array();
+        for(String[] res:results){
+            writer.object();
+            
+            writer.key("id");
+            writer.value(res[0]);
+            
+            writer.key("name");
+            writer.value(res[1]);
+            
+            writer.endObject();
+        }
+        writer.endArray();
+        writer.endObject();
+    }
+    
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/exporters/RdfExporter.java

@@ -0,0 +1,106 @@
+package com.google.gridworks.rdf.exporters;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.net.URI;
+import java.util.Properties;
+
+import com.google.gridworks.browsing.Engine;
+import com.google.gridworks.browsing.FilteredRows;
+import com.google.gridworks.browsing.RowVisitor;
+import com.google.gridworks.exporters.Exporter;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+import com.google.gridworks.rdf.Node;
+import com.google.gridworks.rdf.RdfSchema;
+import com.google.gridworks.rdf.Util;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.ModelFactory;
+import com.hp.hpl.jena.rdf.model.Resource;
+
+public class RdfExporter implements Exporter{
+
+    private String format;
+    
+    public RdfExporter(String f){
+        this.format = f;
+    }
+    public void export(Project project, Properties options, Engine engine,
+            OutputStream outputStream) throws IOException {
+        
+        RdfSchema schema = Util.getProjectSchema(project);
+        Model model = ModelFactory.createDefaultModel();
+        URI baseUri = schema.getBaseUri();
+        Node root = schema.getRoot();
+        
+        Resource[] blanks = new Resource[schema.get_blanks().size()];
+        for (int i = 0; i < blanks.length; i++) {
+            blanks[i] = model.createResource();
+        }
+        
+        RowVisitor visitor = new RdfRowVisitor(model, baseUri, root,blanks);
+        FilteredRows filteredRows = engine.getAllFilteredRows();
+        filteredRows.accept(project, visitor);
+        model.write(outputStream);
+    }
+
+    
+    public void export(Project project, Properties options, Engine engine,
+            Writer writer) throws IOException {
+        RdfSchema schema = Util.getProjectSchema(project);
+        Model model = ModelFactory.createDefaultModel();
+        URI baseUri = schema.getBaseUri();
+        Node root = schema.getRoot();
+        
+        Resource[] blanks = new Resource[schema.get_blanks().size()];
+        for (int i = 0; i < blanks.length; i++) {
+            blanks[i] = model.createResource();
+        }
+
+        RowVisitor visitor = new RdfRowVisitor(model, baseUri, root,blanks);
+        FilteredRows filteredRows = engine.getAllFilteredRows();
+        filteredRows.accept(project, visitor);
+        model.write(writer,format);
+    }
+
+    public String getContentType() {
+        if(format.equals("N3")){
+            return "text/rdf+n3";
+        }else{
+            return "application/rdf+xml";
+        }
+    }
+
+    public boolean takeWriter() {
+        return true;
+    }
+    
+    protected static class RdfRowVisitor implements RowVisitor{
+        Model model;
+        URI base;
+        Node root;
+        Resource[] blanks;
+        public RdfRowVisitor(Model m,URI base, Node root,Resource[] blanks){
+            this.model = m;
+            this.base = base;
+            this.root = root;
+            this.blanks = blanks;
+        }
+        public void end(Project project) {
+            // do nothing
+            
+        }
+
+        public void start(Project project) {
+            // do nothing
+            
+        }
+
+        public boolean visit(Project project, int rowIndex, Row row) {
+            root.createNode(base, model, project, row, rowIndex,blanks);
+            return false;
+        }
+    }
+
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/expr/RdfBinder.java

@@ -0,0 +1,23 @@
+package com.google.gridworks.rdf.expr;
+
+import java.util.Properties;
+
+import com.google.gridworks.expr.Binder;
+import com.google.gridworks.model.Cell;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+import com.google.gridworks.rdf.Util;
+
+public class RdfBinder implements Binder {
+
+    @Override
+    public void bind(Properties bindings, Row row, int rowIndex, String columnName, Cell cell) {
+        // nothing to do
+    }
+
+    @Override
+    public void initializeBindings(Properties bindings, Project project) {
+        bindings.put("baseURI", Util.getProjectSchema(project).getBaseUri());
+    }
+
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/expr/functions/strings/Urlify.java

@@ -0,0 +1,51 @@
+package com.google.gridworks.rdf.expr.functions.strings;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.expr.EvalError;
+import com.google.gridworks.gel.ControlFunctionRegistry;
+import com.google.gridworks.gel.Function;
+
+public class Urlify implements Function {
+
+    public Object call(Properties bindings, Object[] args) {
+        if(args.length==1 || args.length==2){
+            String s = args[0].toString();
+            s = s.replaceAll("\\s+", "_");
+            if(args.length==2){
+                String base = args[1].toString();
+                URI base_uri;
+                try {
+                    base_uri = new URI(base);
+                    return base_uri.resolve(s).toString();
+                } catch (URISyntaxException e) {
+                    return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " unable to encode");
+                }
+            }
+            try {
+                return URLEncoder.encode(s, "UTF-8");
+            } catch (UnsupportedEncodingException e) {
+                return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " unable to encode");
+            }
+        }
+        return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 1 string");
+    }
+
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.object();
+        writer.key("description"); writer.value("replaces spaces with underscore");
+        writer.key("params"); writer.value("string s");
+        writer.key("returns"); writer.value("string");
+        writer.endObject();
+        
+    }
+
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/operations/SaveRdfSchemaOperation.java

@@ -0,0 +1,137 @@
+package com.google.gridworks.rdf.operations;
+
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.io.Writer;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.gridworks.history.Change;
+import com.google.gridworks.history.HistoryEntry;
+import com.google.gridworks.model.AbstractOperation;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.operations.OperationRegistry;
+import com.google.gridworks.rdf.RdfSchema;
+import com.google.gridworks.util.ParsingUtilities;
+import com.google.gridworks.util.Pool;
+
+public class SaveRdfSchemaOperation extends AbstractOperation {
+
+    final protected RdfSchema _schema;
+
+    public SaveRdfSchemaOperation(RdfSchema schema) {
+        this._schema = schema;
+    }
+
+    static public AbstractOperation reconstruct(Project project, JSONObject obj)
+            throws Exception {
+        return new SaveRdfSchemaOperation(RdfSchema.reconstruct(obj
+                .getJSONObject("schema")));
+    }
+
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.object();
+        writer.key("op");
+        writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
+        writer.key("description");
+        writer.value("Save RDF schema skeleton");
+        writer.key("schema");
+        _schema.write(writer, options);
+        writer.endObject();
+
+    }
+
+    @Override
+    protected String getBriefDescription(Project project) {
+        return "Save RDF schema skelton";
+    }
+
+    @Override
+    protected HistoryEntry createHistoryEntry(Project project,
+            long historyEntryID) throws Exception {
+        String description = "Save RDF schema skeleton";
+        
+        Change change = new RdfSchemaChange(_schema);
+        
+        return new HistoryEntry(historyEntryID, project, description,
+                SaveRdfSchemaOperation.this, change);
+    }
+
+    static public class RdfSchemaChange implements Change {
+        final protected RdfSchema _newSchema;
+        protected RdfSchema _oldSchema;
+        
+        public RdfSchemaChange(RdfSchema schema) {
+            _newSchema = schema;
+        }
+        
+        public void apply(Project project) {
+            synchronized (project) {
+                _oldSchema = (RdfSchema) project.overlayModels.get("rdfSchema");
+                project.overlayModels.put("rdfSchema", _newSchema);
+            }
+        }
+        
+        public void revert(Project project) {
+            synchronized (project) {
+                if (_oldSchema == null) {
+                    project.overlayModels.remove("rdfSchema");
+                } else {
+                    project.overlayModels.put("rdfSchema", _oldSchema);
+                }
+            }
+        }
+        
+        public void save(Writer writer, Properties options) throws IOException {
+            writer.write("newSchema=");
+            writeRdfSchema(_newSchema, writer);
+            writer.write('\n');
+            writer.write("oldSchema=");
+            writeRdfSchema(_oldSchema, writer);
+            writer.write('\n');
+            writer.write("/ec/\n"); // end of change marker
+        }
+        
+        static public Change load(LineNumberReader reader, Pool pool)
+                throws Exception {
+            RdfSchema oldSchema = null;
+            RdfSchema newSchema = null;
+            
+            String line;
+            while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
+                int equal = line.indexOf('=');
+                CharSequence field = line.subSequence(0, equal);
+                String value = line.substring(equal + 1);
+                
+                if ("oldSchema".equals(field) && value.length() > 0) {
+                    oldSchema = RdfSchema.reconstruct(ParsingUtilities
+                            .evaluateJsonStringToObject(value));
+                } else if ("newSchema".equals(field) && value.length() > 0) {
+                    newSchema = RdfSchema.reconstruct(ParsingUtilities
+                            .evaluateJsonStringToObject(value));
+                }
+            }
+            
+            RdfSchemaChange change = new RdfSchemaChange(newSchema);
+            change._oldSchema = oldSchema;
+            
+            return change;
+        }
+        
+        static protected void writeRdfSchema(RdfSchema s, Writer writer)
+                throws IOException {
+            if (s != null) {
+                JSONWriter jsonWriter = new JSONWriter(writer);
+                try {
+                    s.write(jsonWriter, new Properties());
+                } catch (JSONException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/vocab/RDFNode.java

@@ -0,0 +1,130 @@
+package com.google.gridworks.rdf.vocab;
+
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.Jsonizable;
+
+public abstract class RDFNode implements Jsonizable{
+    private String preferredCURIE;
+    private String description;
+    private String URI;
+    private String label;
+    private String vocabularyPrefix;
+    private String vocabularyUri;
+    
+    public String getVocabularyUri() {
+        return vocabularyUri;
+    }
+    public void setVocabularyUri(String vocabularyUri) {
+        this.vocabularyUri = vocabularyUri;
+    }
+    public String getVocabularyPrefix() {
+        return vocabularyPrefix;
+    }
+    public void setVocabularyPrefix(String vocabularyPrefix) {
+        this.vocabularyPrefix = vocabularyPrefix;
+    }
+    public String getPreferredCURIE() {
+        return preferredCURIE;
+    }
+    public void setPreferredCURIE(String preferredCURIE) {
+        this.preferredCURIE = preferredCURIE;
+    }
+    public String getDescription() {
+        return description;
+    }
+    public void setDescription(String description) {
+        this.description = description;
+    }
+    public String getURI() {
+        return URI;
+    }
+    public void setURI(String uRI) {
+        URI = uRI;
+    }
+    public String getLabel() {
+        return label;
+    }
+    public void setLabel(String label) {
+        this.label = label;
+    }
+    public RDFNode(){
+        
+    }
+    public RDFNode(String description, String uRI,
+            String label,String prefix,String vocabularyUri) {
+        this.description = description;
+        URI = uRI;
+        this.label = label;
+        this.vocabularyPrefix = prefix;
+        this.preferredCURIE = composePreferredCurie();
+        this.vocabularyUri = vocabularyUri;
+    }
+    private String composePreferredCurie(){
+        String qname;
+        if(this.URI==null){
+            return null;
+        }
+        if(this.URI.indexOf("#")!=-1){
+            qname = this.URI.substring(this.URI.indexOf("#")+1);
+        }else{
+            qname = this.URI.substring(this.URI.lastIndexOf("/")+1);
+        }
+        return this.vocabularyPrefix + ":" + qname;
+    }
+    
+    
+    public abstract String getType();
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.object();
+        
+        writer.key("type");
+        writer.value(this.getType());
+        writer.key("prefix");
+        writer.value(vocabularyPrefix);
+        writer.key("preferredCURIE");
+        writer.value(this.preferredCURIE);
+        writer.key("label");
+        writer.value(label);
+        writer.key("description");
+        writer.value(description);
+        writer.key("URI");
+        writer.value(URI);
+        writer.endObject();
+    }
+    
+    public void writeAsSearchResult(JSONWriter writer)throws JSONException {
+        writer.object();
+        
+        writer.key("id");
+        writer.value(URI);
+        
+        writer.key("name");
+        writer.value(preferredCURIE);
+        
+        writer.key("description");
+        writer.value(description);
+        writer.endObject();
+    }
+    @Override
+    public boolean equals(Object obj) {
+        if(!(obj instanceof RDFNode)) return false;
+        RDFNode n = (RDFNode) obj;
+        if(n.getURI()==null || this.URI==null){
+            return false;
+        }
+        return this.URI.equals(n.getURI());
+    }
+    @Override
+    public int hashCode() {
+        return this.URI.hashCode();
+    }
+    
+    
+
+    
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/vocab/RDFSClass.java

@@ -0,0 +1,16 @@
+package com.google.gridworks.rdf.vocab;
+
+
+public class RDFSClass extends RDFNode{
+
+    @Override
+    public String getType() {
+        return "class";
+    }
+
+    public RDFSClass( String uRI,
+            String label,String description,String prefix,String vocabularyUri) {
+        super(description,uRI,label,prefix,vocabularyUri);
+    }
+    
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/vocab/RDFSProperty.java

@@ -0,0 +1,14 @@
+package com.google.gridworks.rdf.vocab;
+
+
+public class RDFSProperty extends RDFNode{
+
+    @Override
+    public String getType() {
+        return "property";
+    }
+    public RDFSProperty(String uRI,
+            String label,String description,String prefix,String vocabularyUri) {
+        super(description,uRI,label,prefix,vocabularyUri);
+    }
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/vocab/Vocabulary.java

@@ -0,0 +1,65 @@
+package com.google.gridworks.rdf.vocab;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.Jsonizable;
+
+public class Vocabulary implements Jsonizable{
+    private final String name;
+    private final String uri;
+    private List<RDFSClass> classes = new ArrayList<RDFSClass>();
+    private List<RDFSProperty> properties = new ArrayList<RDFSProperty>();
+    
+    public Vocabulary(String name,String uri){
+        this.name = name;
+        this.uri = uri;
+    }
+    
+    public void addClass(RDFSClass clazz){
+        this.classes.add(clazz);
+    }
+    
+    public void addProperty(RDFSProperty prop){
+        this.properties.add(prop);
+    }
+    
+    public List<RDFSClass> getClasses() {
+        return classes;
+    }
+
+    public void setClasses(List<RDFSClass> classes) {
+        this.classes = classes;
+    }
+
+    public List<RDFSProperty> getProperties() {
+        return properties;
+    }
+
+    public void setProperties(List<RDFSProperty> properties) {
+        this.properties = properties;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getUri() {
+        return uri;
+    }
+
+    @Override
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.object();
+        
+        writer.key("name"); writer.value(name);
+        writer.key("uri"); writer.value(uri);
+        
+        writer.endObject();
+    }
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/vocab/VocabularyExistException.java

@@ -0,0 +1,10 @@
+package com.google.gridworks.rdf.vocab;
+
+public class VocabularyExistException extends Exception {
+    private static final long serialVersionUID = 1916094460059608851L;
+
+    public VocabularyExistException(String msg){
+        super(msg);
+    }
+
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/vocab/VocabularyImporter.java

@@ -0,0 +1,119 @@
+package com.google.gridworks.rdf.vocab;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.hp.hpl.jena.ontology.OntModelSpec;
+import com.hp.hpl.jena.query.Query;
+import com.hp.hpl.jena.query.QueryExecution;
+import com.hp.hpl.jena.query.QueryExecutionFactory;
+import com.hp.hpl.jena.query.QueryFactory;
+import com.hp.hpl.jena.query.QuerySolution;
+import com.hp.hpl.jena.query.ResultSet;
+import com.hp.hpl.jena.rdf.model.Literal;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.ModelFactory;
+
+public class VocabularyImporter {
+
+    private static final String PREFIXES = "PREFIX rdfs:<http://www.w3.org/2000/01/rdf-schema#> " +
+            "PREFIX rdf:<http://www.w3.org/1999/02/22-rdf-syntax-ns#> " +
+            "PREFIX skos:<http://www.w3.org/2004/02/skos/core#> ";
+    private static final String CLASSES_QUERY_P1 = PREFIXES + 
+    "SELECT ?resource ?label ?en_label ?description ?en_description ?definition ?en_definition " +
+    "WHERE { " +
+        "?resource rdf:type rdfs:Class. " +
+        "OPTIONAL {?resource rdfs:label ?label.} " +
+        "OPTIONAL {?resource rdfs:label ?en_label. FILTER langMatches( lang(?en_label), \"EN\" )  } " +
+        "OPTIONAL {?resource rdfs:comment ?description.} " +
+        "OPTIONAL {?resource rdfs:comment ?en_description. FILTER langMatches( lang(?en_description), \"EN\" )  } " +
+        "OPTIONAL {?resource skos:definition ?definition.} " +
+        "OPTIONAL {?resource skos:definition ?en_definition. FILTER langMatches( lang(?en_definition), \"EN\" )  } " +
+        "FILTER regex(str(?resource), \"^";
+    private static final String CLASSES_QUERY_P2 = "\")}";
+    
+    private static final String PROPERTIES_QUERY_P1 = PREFIXES + 
+    "SELECT ?resource ?label ?en_label ?description ?en_description ?definition ?en_definition " +
+    "WHERE { " +
+        "?resource rdf:type rdf:Property. " +
+        "OPTIONAL {?resource rdfs:label ?label.} " +
+        "OPTIONAL {?resource rdfs:label ?en_label. FILTER langMatches( lang(?en_label), \"EN\" )  } " +
+        "OPTIONAL {?resource rdfs:comment ?description.} " +
+        "OPTIONAL {?resource rdfs:comment ?en_description. FILTER langMatches( lang(?en_description), \"EN\" )  } " +
+        "OPTIONAL {?resource skos:definition ?definition.} " +
+        "OPTIONAL {?resource skos:definition ?en_definition. FILTER langMatches( lang(?en_definition), \"EN\" )  } " +
+        "FILTER regex(str(?resource), \"^";
+    private static final String PROPERTIES_QUERY_P2 = "\")}"; 
+    
+    public Vocabulary getVocabulary(String url, String prefix, String namespace, String format){
+        Model m = getModel(url, format);
+        return getVocabulary(m,namespace,prefix);
+    }
+    
+    private Model getModel(String url,String format){
+        Model model = ModelFactory.createOntologyModel(OntModelSpec.OWL_DL_MEM_RDFS_INF);//ModelFactory.createDefaultModel();
+        model.read(url);
+        return model;
+    }
+    
+
+    private Vocabulary getVocabulary(Model m, String namespace, String prefix){
+        Query query = QueryFactory.create(CLASSES_QUERY_P1 + namespace.trim() + CLASSES_QUERY_P2);
+        QueryExecution qe = QueryExecutionFactory.create(query, m);
+        ResultSet res = qe.execSelect();
+        Set<String> seen = new HashSet<String>();
+        Vocabulary vocab = new Vocabulary(prefix, namespace);
+        while(res.hasNext()){
+            QuerySolution qs = res.nextSolution();
+            String uri = qs.getResource("resource").getURI();
+            if(seen.contains(uri)){
+                continue;
+            }
+            String label = getFirstNotNull(new Literal[]{qs.getLiteral("en_label"),qs.getLiteral("label")});
+            
+            String description = getFirstNotNull(new Literal[]{qs.getLiteral("en_definition"),qs.getLiteral("definition"),
+                    qs.getLiteral("en_description"),qs.getLiteral("description")}) ;
+            RDFSClass clazz = new RDFSClass(uri, label, description,prefix,namespace);
+            vocab.addClass(clazz);
+        }
+        
+        query = QueryFactory.create(PROPERTIES_QUERY_P1 + namespace.trim() + PROPERTIES_QUERY_P2);
+        qe = QueryExecutionFactory.create(query, m);
+        res = qe.execSelect();
+        seen = new HashSet<String>();
+        while(res.hasNext()){
+            QuerySolution qs = res.nextSolution();
+            String uri = qs.getResource("resource").getURI();
+            if(seen.contains(uri)){
+                continue;
+            }
+            String label = getFirstNotNull(new Literal[]{qs.getLiteral("en_label"),qs.getLiteral("label")});
+            
+            String description = getFirstNotNull(new Literal[]{qs.getLiteral("en_definition"),qs.getLiteral("definition"),
+                    qs.getLiteral("en_description"),qs.getLiteral("description")}) ;
+            RDFSProperty prop = new RDFSProperty(uri, label, description,prefix,namespace);
+            vocab.addProperty(prop);
+        }
+        
+        return vocab;
+    }
+    
+    private String getFirstNotNull(Literal[] literals){
+        String s = null;
+        for(int i=0;i<literals.length;i++){
+            s = getString(literals[i]);
+            if(s!=null){
+                break;
+            }
+        }
+        return s;
+    }
+    
+    private String getString(Literal l){
+        if(l!=null){
+            return l.getString();
+        }
+        return null;
+    }
+    
+}

extensions/rdf-exporter/src/com/google/gridworks/rdf/vocab/VocabularyManager.java

@@ -0,0 +1,244 @@
+package com.google.gridworks.rdf.vocab;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.lucene.util.Version;
+
+import com.google.gridworks.GridworksServlet;
+
+/**
+ * @author fadmaa
+ *
+ */
+public class VocabularyManager {
+
+    private static final String CLASS_TYPE = "class";
+    private static final String PROPERTY_TYPE = "property";
+    private static final String VOCABULARY_TYPE = "vocabulary";
+    
+    private IndexWriter writer;
+    private IndexSearcher searcher;
+    
+    private Directory _directory;
+    
+    private static VocabularyManager singleton;
+    
+    private List<Vocabulary> vocabularies = new ArrayList<Vocabulary>();
+    
+    static public VocabularyManager getSingleton(GridworksServlet servlet) {
+        return singleton != null ? singleton : (singleton = new VocabularyManager(servlet));
+    }
+    
+    private VocabularyManager(GridworksServlet servlet) {
+        try{
+            synchronized (this) {
+                File dir = servlet.getCacheDir("rdfImporter");
+                _directory = new SimpleFSDirectory(new File(dir, "luceneIndex"));
+                writer = new IndexWriter(_directory, new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.LIMITED);
+                searcher = new IndexSearcher(_directory);
+                updateVocabulariesList();                
+            }
+        } catch (CorruptIndexException e) {
+            throw new RuntimeException("Failed initialize vocabulary search",e);
+        } catch (IOException e) {
+            throw new RuntimeException("Failed initialize vocabulary search",e);
+        }
+    }
+    
+    
+    /**
+     * @param url where to get the vocabulary description from
+     * @param prefix preferred prefix for vocabulary e.g. foaf, dc, skos, dcat
+     * @param namespace the base URI of the vocabulary. usually but not alway the same as url
+     * @param format the format of the RDF description of the vocabulary at the end of url (default to RDF/XML)
+     * @throws IOException 
+     * @throws CorruptIndexException 
+     * @throws VocabularyExistException 
+     */
+    public void addVocabulary(String url, String prefix, String namespace, String format) throws CorruptIndexException, IOException, VocabularyExistException{
+        if (defined(namespace)){
+            throw new VocabularyExistException(namespace + " already exists!");
+        }
+        VocabularyImporter importer = new VocabularyImporter();
+        Vocabulary vocabulary = importer.getVocabulary(url,prefix,namespace,format);
+        indexVocabulary(vocabulary);
+        updateSearcher();    
+//        updateVocabulariesList();
+        this.vocabularies.add(vocabulary);
+    }
+    
+    public void addVocabulary(String url, String prefix, String namespace) throws CorruptIndexException, IOException, VocabularyExistException{
+        addVocabulary(url, prefix, namespace,"RDF/XML");
+    }
+    private void indexVocabulary(Vocabulary v) throws CorruptIndexException, IOException{
+        Document doc = new Document();
+        doc.add(new Field("type",VOCABULARY_TYPE,Field.Store.YES,Field.Index.NOT_ANALYZED));
+        doc.add(new Field("name",v.getName(),Field.Store.YES,Field.Index.NO));
+        doc.add(new Field("uri",v.getUri(),Field.Store.YES,Field.Index.NOT_ANALYZED));
+        writer.addDocument(doc);
+        
+        for(RDFSClass c:v.getClasses()){
+            indexRdfNode(c, CLASS_TYPE);
+        }
+        for(RDFSProperty p:v.getProperties()){
+            indexRdfNode(p, PROPERTY_TYPE);
+        }
+        
+        writer.commit();
+    }
+    
+    public List<RDFNode> searchClasses(String str)throws ParseException, IOException{
+        List<RDFNode> res = new ArrayList<RDFNode>();
+        org.apache.lucene.search.Query query = prepareQuery(str, "class");
+        TopDocs docs = searcher.search(query, 1000);
+        for(int i=0;i<docs.totalHits;i++){
+            Document doc= searcher.doc(docs.scoreDocs[i].doc);
+            String uri = doc.get("uri");
+            String label = doc.get("label");
+            String description = doc.get("description");
+            String namespace = doc.get("namespace"); 
+            String prefix = doc.get("prefix");
+            RDFSClass node = new RDFSClass(uri,label,description,prefix,namespace);
+            res.add(node);
+        }
+        
+        return res;
+    }
+    
+    public List<RDFNode> searchProperties(String str)throws ParseException, IOException{
+        List<RDFNode> res = new ArrayList<RDFNode>();
+        org.apache.lucene.search.Query query = prepareQuery(str, "property");
+        TopDocs docs = searcher.search(query, 1000);
+        for(int i=0;i<docs.totalHits;i++){
+            Document doc= searcher.doc(docs.scoreDocs[i].doc);
+            String uri = doc.get("uri");
+            String label = doc.get("label");
+            String description = doc.get("description");
+            String namespace = doc.get("namespace"); 
+            String prefix = doc.get("prefix");
+            RDFNode node = new RDFSProperty(uri,label,description,prefix,namespace);
+            res.add(node);
+        }
+        
+        return res;
+    }
+    
+    public List<Vocabulary> getVocabularies(){
+        return vocabularies;
+    }
+    
+    public void deleteVocabulary(String uri) throws IOException{
+        Vocabulary vocab = null;
+        for(Vocabulary v:vocabularies){
+            if(v.getUri().equals(uri)){
+                vocab = v;
+                break;
+            }
+        }
+        if(vocab==null){
+            throw new RuntimeException("Vocabulary " + uri + " not found");
+        }
+        vocabularies.remove(vocab);
+        Term t = new Term("uri",uri);
+        writer.deleteDocuments(t);
+        t = new Term("namespace",uri);
+        writer.deleteDocuments(t);
+        
+        writer.commit();
+        updateSearcher();
+    }
+    
+    private org.apache.lucene.search.Query prepareQuery(String s,String type)throws ParseException{
+        QueryParser parser = new QueryParser(Version.LUCENE_30,"description",new StandardAnalyzer(Version.LUCENE_30));
+        String queryStr = "type:" + type ;
+        if(s!=null && s.trim().length()>0){
+            s =s.trim();
+            if(s.indexOf("*")==-1){
+                s += "*";
+            }
+            if(s.indexOf(":")==-1){
+                queryStr += " AND (curie:" + s ;
+                queryStr += " OR description:" + s ;
+                queryStr += " OR label:" + s + ")";
+            }else{
+                String p1 = s.substring(0,s.indexOf(":"));
+                String p2 = s.substring(s.lastIndexOf(":")+1);
+                queryStr += " AND prefix:" + p1;
+                if(p2.length()>1){
+                    //queryStr += " AND (description:" + p2;
+                    queryStr += " AND label:" + p2;
+                }
+            }
+            
+        }
+        return parser.parse(queryStr);
+    }
+    
+    private void indexRdfNode(RDFNode node, String type) throws CorruptIndexException, IOException{
+        //TODO weight fields... setBoost
+        Document doc = new Document();
+        doc.add(new Field("type",type,Field.Store.YES,Field.Index.NOT_ANALYZED));
+        doc.add(new Field("prefix",node.getVocabularyPrefix(),Field.Store.YES,Field.Index.NOT_ANALYZED));
+        String l = node.getLabel()==null?"":node.getLabel();
+        Field labelField = new Field("label",l,Field.Store.NO,Field.Index.ANALYZED);
+        doc.add(labelField);
+        String d = node.getDescription()==null?"":node.getDescription();
+        Field descriptionField = new Field("description",d,Field.Store.YES,Field.Index.ANALYZED);
+        doc.add(descriptionField);
+        doc.add(new Field("uri", node.getURI(),Field.Store.YES,Field.Index.NO));
+        Field curieField = new Field("curie", node.getPreferredCURIE(),Field.Store.YES,Field.Index.ANALYZED);
+        doc.add(curieField);
+        Field namespaceField = new Field("namespace", node.getVocabularyUri(),Field.Store.YES,Field.Index.NOT_ANALYZED);
+        doc.add(namespaceField);
+        writer.addDocument(doc);
+    }
+    
+    private void updateVocabulariesList() throws IOException{
+        Term typeTerm = new Term("type", VOCABULARY_TYPE);
+        Query query = new TermQuery(typeTerm);
+        //TODO 1000 :O
+        TopDocs vocabDocs = searcher.search(query, 1000);
+        for(int i=0;i<vocabDocs.totalHits;i++){
+            Document doc = searcher.doc(vocabDocs.scoreDocs[i].doc);
+            String name = doc.get("name");
+            String uri = doc.get("uri");
+            Vocabulary vocab = new Vocabulary(name, uri);
+            this.vocabularies.add(vocab);
+        }
+        
+    }
+    
+    private void updateSearcher() throws IOException{
+        this.searcher = new IndexSearcher(_directory);
+    }
+    
+    private boolean defined(String namespace){
+        if(namespace==null){
+            return false;
+        }
+        namespace = namespace.trim();
+        for(Vocabulary v:vocabularies){
+            if(v.getUri().equals(namespace)){
+                return true;
+            }
+        }
+        return false;
+    }
+}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/CellBlankNode.java

@@ -1,39 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-import java.net.URI;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.Resource;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-
-public class CellBlankNode extends ResourceNode{
-
-    final public int columnIndex;
-    final public String columnName;
-    
-    public CellBlankNode(int i,String columnName){
-        this.columnIndex = i;
-        this.columnName = columnName;
-    }
-    
-    @Override
-    public Resource createResource(URI baseUri, Model model, Project project,
-            Row row, int rowIndex,Resource[] blanks) {
-        return model.createResource();
-    }
-
-    @Override
-    protected void writeNode(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.key("nodeType");writer.value("cell-as-blank");
-        writer.key("columnIndex");writer.value(columnIndex);
-        writer.key("columnName");writer.value(columnName);
-    }
-
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/CellLiteralNode.java

@@ -1,78 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-import java.net.URI;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.hp.hpl.jena.rdf.model.Literal;
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.RDFNode;
-import com.hp.hpl.jena.rdf.model.Resource;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-
-public class CellLiteralNode extends CellNode{
-
-    private String valueType;
-    private String lang;
-    public String getValueType() {
-        return valueType;
-    }
-    public void setValueType(String valueType) {
-        this.valueType = valueType;
-    }
-    public String getLang() {
-        return lang;
-    }
-    public void setLang(String lang) {
-        this.lang = lang;
-    }
-    public void write(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.object();
-        writer.key("nodeType");writer.value("cell-as-literal");
-        writer.key("columnIndex");writer.value(columnIndex);
-        writer.key("columnName");writer.value(columnName);
-        if(valueType!=null){
-            writer.key("valueType");writer.value(valueType);
-        }
-        if(lang!=null){
-            writer.key("lang");writer.value(lang);
-        }
-        writer.endObject();
-    }
-    
-    public CellLiteralNode(int index,String columnName){
-        super(index,columnName);
-    }
-    
-    public CellLiteralNode(int index,String columnName,String valueType,String lang){
-        this(index,columnName);
-        this.lang = lang;
-        this.valueType = valueType;
-    }
-    public RDFNode createNode(URI baseUri, Model model, Project project,
-            Row row, int rowIndex,Resource[] blanks) {
-        String val;
-        try{
-            val= row.getCell(this.columnIndex).value.toString();
-        }catch(NullPointerException ne){
-            return null;
-        }
-        if(val!=null && val.length()>0){
-            //TODO language and datatype
-            Literal l;
-            if(this.valueType!=null){
-                l = model.createTypedLiteral(val, valueType);
-            }else{
-                l = model.createLiteral(val);
-            }
-            return l;
-        }else{
-            return null;
-        }
-    }
-
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/CellNode.java

@@ -1,12 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-public abstract class CellNode implements Node{
-    final public int columnIndex;
-    final public String columnName;
-    
-    public CellNode(int i,String columnName){
-        this.columnIndex = i;
-        this.columnName = columnName; 
-    }
-
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/CellResourceNode.java

@@ -1,80 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-import java.net.URI;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.Resource;
-import com.metaweb.gridworks.expr.Evaluable;
-import com.metaweb.gridworks.expr.ExpressionUtils;
-import com.metaweb.gridworks.expr.MetaParser;
-import com.metaweb.gridworks.model.Cell;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-
-
-public class CellResourceNode extends ResourceNode{
-
-    private String uriExpression;
-    final public String columnName;
-    public String getUriExpression() {
-        return uriExpression;
-    }
-
-    
-    final public int columnIndex;
-    
-    public CellResourceNode(int i,String columnName){
-        this.columnIndex = i;
-        this.columnName = columnName;
-    }
-    
-    public CellResourceNode(int columnIndex,String columnName,String exp) {
-        this(columnIndex,columnName);
-        this.uriExpression = exp;
-    }
-
-    @Override
-    public Resource createResource(URI baseUri,Model model,Project project,Row row,int rowIndex,Resource[] blanks) {
-        try{
-            Properties bindings = ExpressionUtils.createBindings(project);
-            Evaluable eval = MetaParser.parse(uriExpression);
-            Cell cell = row.getCell(this.columnIndex);
-            String colName = this.columnIndex>-1?project.columnModel.getColumnByCellIndex(this.columnIndex).getName():"";
-            ExpressionUtils.bind(bindings, row, rowIndex,colName , cell);
-            Object result = eval.evaluate(bindings);
-            if(result.toString().length()>0){
-                String uri = Util.getUri(baseUri, result.toString());
-                Resource r =  model.createResource(uri);
-                return r;
-            }else{
-                return null;
-            }
-        }catch(Exception e){
-//            e.printStackTrace();
-            //an empty cell might result in an exception out of evaluating URI expression... so it is intended to eat the exception
-            return null;
-        }
-        
-    }
-
-    @Override
-    protected void writeNode(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.key("nodeType"); writer.value("cell-as-resource");
-        writer.key("uriExpression"); writer.value(uriExpression);
-        writer.key("columnIndex"); writer.value(columnIndex);
-        if(columnIndex==-1){
-            //Row number
-            writer.key("isRowNumberCell"); writer.value(true);
-        } else {
-            writer.key("columnName"); writer.value(columnName);
-        }
-    }
-    
-    
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/ConstantBlankNode.java

@@ -1,33 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-import java.net.URI;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.Resource;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-
-public class ConstantBlankNode extends ResourceNode{
-
-    private int _id;
-    ConstantBlankNode(int id){
-        this._id = id;
-    }
-    
-    @Override
-    public Resource createResource(URI baseUri, Model model, Project project,
-            Row row, int rowIndex,Resource[] blanks) {
-        return blanks[this._id];
-    }
-
-    @Override
-    protected void writeNode(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.key("nodeType");writer.value("blank");
-    }
-
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/ConstantLiteralNode.java

@@ -1,89 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-import java.net.URI;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.hp.hpl.jena.rdf.model.Literal;
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.RDFNode;
-import com.hp.hpl.jena.rdf.model.Resource;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-
-public class ConstantLiteralNode implements Node{
-
-    private String valueType;
-    private String lang;
-    private String value;
-    
-    
-    public ConstantLiteralNode(String val,String type,String l){
-        this.value = val;
-        this.valueType = type;
-        this.lang = l;
-    }
-    public String getValueType() {
-        return valueType;
-    }
-
-
-    public void setValueType(String valueType) {
-        this.valueType = valueType;
-    }
-
-
-    public String getLang() {
-        return lang;
-    }
-
-
-    public void setLang(String lang) {
-        this.lang = lang;
-    }
-
-
-    public String getValue() {
-        return value;
-    }
-
-
-    public void setValue(String value) {
-        this.value = value;
-    }
-
-
-    public void write(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.object();
-        writer.key("nodeType"); writer.value("literal");
-        writer.key("value"); writer.value(value);
-        if(valueType!=null){
-            writer.key("valueType");
-            writer.value(valueType);
-        }
-        if(lang!=null){
-            writer.key("lang");
-            writer.value(lang);
-        }
-        writer.endObject();
-    }
-    public RDFNode createNode(URI baseUri, Model model, Project project,
-            Row row, int rowIndex,Resource[] blanks) {
-        if(this.value!=null && this.value.length()>0){
-            
-            Literal l ;
-            if(this.valueType!=null){
-                l = model.createTypedLiteral(this.value, valueType);
-            }else{
-                l = model.createLiteral(this.value);
-            }
-            return l;
-        }else{
-            return null;
-        }
-    }
-
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/ConstantResourceNode.java

@@ -1,60 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.Resource;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-
-public class ConstantResourceNode extends ResourceNode{
-
-    private String uri;
-
-
-    public String getUri() {
-        return uri;
-    }
-
-    public void setUri(String uri) {
-        this.uri = uri;
-    }
-    
-
-    public ConstantResourceNode(String uri){
-        this.uri = uri;
-    }
-
-    @Override
-    public Resource createResource(URI baseUri, Model model, Project project,
-            Row row, int rowIndex,Resource[] blanks) {
-        if(this.uri!=null & this.uri.length()>0){
-            String tmp;
-            try {
-                tmp = Util.getUri(baseUri, this.uri);
-            } catch (URISyntaxException e) {
-                // TODO Auto-generated catch block
-                e.printStackTrace();
-                return null;
-            }
-            Resource r =  model.createResource(tmp);
-            return r;
-        }else{
-            return null;
-        }
-    }
-
-    @Override
-    protected void writeNode(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.key("nodeType"); writer.value("resource");
-        writer.key("uri"); writer.value(uri);
-    }
-    
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/Link.java

@@ -1,36 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.metaweb.gridworks.Jsonizable;
-
-public class Link implements Jsonizable {
-
-    public final String propertyUri;
-    public final String curie;
-    public final Node target;
-
-    public Link(String uri,String curie,Node t){
-        this.propertyUri = uri;
-        this.target = t;
-        this.curie = curie;
-    }
-    
-    public void write(JSONWriter writer, Properties options)
-            throws JSONException {
-
-        writer.object();
-        writer.key("uri");
-        writer.value(propertyUri);
-        writer.key("curie");
-        writer.value(curie);
-        if (target != null) {
-            writer.key("target");
-            target.write(writer, options);
-        }
-        writer.endObject();
-    }
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/Node.java

@@ -1,15 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-import java.net.URI;
-
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.RDFNode;
-import com.hp.hpl.jena.rdf.model.Resource;
-import com.metaweb.gridworks.Jsonizable;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-
-public interface Node extends Jsonizable{
-
-    RDFNode createNode(URI baseUri,Model model,Project project,Row row,int rowIndex,Resource[] blanks);
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/RdfSchema.java

@@ -1,180 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.metaweb.gridworks.model.OverlayModel;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.rdf.ResourceNode.RdfType;
-
-public class RdfSchema implements OverlayModel {
-
-    final protected List<Node> _rootNodes = new ArrayList<Node>();
-    final protected List<ConstantBlankNode> _blanks = new ArrayList<ConstantBlankNode>();
-
-    public List<ConstantBlankNode> get_blanks() {
-        return _blanks;
-    }
-
-    protected URI baseUri;
-
-    public void setBaseUri(URI baseUri) {
-        this.baseUri = baseUri;
-    }
-
-    public void setBaseUri(String base) throws URISyntaxException {
-        this.baseUri = new URI(base);
-    }
-
-    public RdfSchema() {
-        // FIXME
-        try {
-            this.baseUri = new URI("http://localhost:3333/");
-        } catch (URISyntaxException ue) {
-        }
-    }
-
-    public URI getBaseUri() {
-        return baseUri;
-    }
-
-    public Node getRoot() {
-        return _rootNodes.get(0);
-    }
-
-    static public RdfSchema reconstruct(JSONObject o) throws JSONException {
-        RdfSchema s = new RdfSchema();
-        // TODO
-        try {
-            s.baseUri = new URI(o.getString("baseUri"));
-        } catch (URISyntaxException me) {
-            me.printStackTrace();
-        }
-
-        JSONArray rootNodes = o.getJSONArray("rootNodes");
-        int count = rootNodes.length();
-
-        for (int i = 0; i < count; i++) {
-            JSONObject o2 = rootNodes.getJSONObject(i);
-            Node node = reconstructNode(o2, s);
-            if (node != null) {
-                s._rootNodes.add(node);
-            }
-        }
-
-        return s;
-    }
-
-    static protected Node reconstructNode(JSONObject o, RdfSchema s)
-            throws JSONException {
-        Node node = null;
-        int blanksCount = 0;
-        String nodeType = o.getString("nodeType");
-        if (nodeType.startsWith("cell-as-")) {
-            int columnIndex = o.getInt("columnIndex");
-            String columnName = null;
-            if(columnIndex!=-1){
-                columnName = o.getString("columnName");
-            }
-
-            if ("cell-as-resource".equals(nodeType)) {
-                String exp = o.getString("uriExpression");
-                node = new CellResourceNode(columnIndex, columnName, exp);
-                if (o.has("rdfTypes")) {
-                    List<RdfType> types = reconstructTypes(o
-                            .getJSONArray("rdfTypes"));
-                    ((CellResourceNode) node).setTypes(types);
-                }
-            } else if ("cell-as-literal".equals(nodeType)) {
-                String valueType = o.has("valueType")?Util.getDataType(o.getString("valueType")):null;
-                String lang = o.has("lang") ? o.getString("lang"):null;
-                node = new CellLiteralNode(columnIndex, columnName, valueType, lang);
-            } else if ("cell-as-blank".equals(nodeType)) {
-                node = new CellBlankNode(columnIndex,columnName);
-                if (o.has("rdfTypes")) {
-                    List<RdfType> types = reconstructTypes(o
-                            .getJSONArray("rdfTypes"));
-                    ((CellBlankNode) node).setTypes(types);
-                }
-            }
-        } else if ("resource".equals(nodeType)) {
-            node = new ConstantResourceNode(o.getString("uri"));
-            if (o.has("rdfTypes")) {
-                List<RdfType> types = reconstructTypes(o
-                        .getJSONArray("rdfTypes"));
-                ((ConstantResourceNode) node).setTypes(types);
-            }
-        } else if ("literal".equals(nodeType)) {
-            String valueType = o.has("valueType")?Util.getDataType(o.getString("valueType")):null;
-            String lang = o.has("lang") ? o.getString("lang"):null;
-            node = new ConstantLiteralNode(o.getString("value"), valueType,lang);
-        } else if ("blank".equals(nodeType)) {
-            node = new ConstantBlankNode(blanksCount);
-            blanksCount += 1;
-            s._blanks.add((ConstantBlankNode) node);
-            if (o.has("rdfTypes")) {
-                List<RdfType> types = reconstructTypes(o
-                        .getJSONArray("rdfTypes"));
-                ((ConstantBlankNode) node).setTypes(types);
-            }
-        }
-
-        if (node != null && node instanceof ResourceNode && o.has("links")) {
-            ResourceNode node2 = (ResourceNode) node;
-
-            JSONArray links = o.getJSONArray("links");
-            int linkCount = links.length();
-
-            for (int j = 0; j < linkCount; j++) {
-                JSONObject oLink = links.getJSONObject(j);
-
-                node2.addLink(new Link(oLink.getString("uri"), oLink.getString("curie"),oLink
-                        .has("target")
-                        && !oLink.isNull("target") ? reconstructNode(oLink
-                        .getJSONObject("target"), s) : null));
-            }
-        }
-
-        return node;
-    }
-
-    static private List<RdfType> reconstructTypes(JSONArray arr)
-            throws JSONException {
-        List<RdfType> lst = new ArrayList<RdfType>();
-        for (int i = 0; i < arr.length(); i++) {
-            String uri = arr.getJSONObject(i).getString("uri");
-            String curie = arr.getJSONObject(i).getString("curie");
-            lst.add(new RdfType(uri, curie));
-        }
-        return lst;
-    }
-
-    @Override
-    public void write(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.object();
-        writer.key("baseUri"); writer.value(baseUri);
-        
-        writer.key("rootNodes");
-        writer.array();
-        
-        for (Node node : _rootNodes) {
-            node.write(writer, options);
-        }
-        
-        writer.endArray();
-        writer.endObject();
-    }
-    
-    static public RdfSchema load(Project project, JSONObject obj) throws Exception {
-        return reconstruct(obj);
-    }
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/ResourceNode.java

@@ -1,133 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.Property;
-import com.hp.hpl.jena.rdf.model.RDFNode;
-import com.hp.hpl.jena.rdf.model.Resource;
-import com.hp.hpl.jena.vocabulary.RDF;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-
-
-abstract public class ResourceNode implements Node {
-
-    private List<Link> links = new ArrayList<Link>();
-    
-    private List<RdfType> rdfTypes = new ArrayList<RdfType>();
-    
-    public void addLink(Link link) {
-        this.links.add(link);
-    }
-
-    public void addType(RdfType type) {
-        this.rdfTypes.add(type);
-    }
-
-    public Link getLink(int index) {
-        return this.links.get(index);
-    }
-
-    public int getLinkCount() {
-        return this.links.size();
-    }
-
-    public List<RdfType> getTypes() {
-        return this.rdfTypes;
-    }
-
-    protected abstract void writeNode(JSONWriter writer, Properties options) throws JSONException;
-    public void write(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.object();
-        //writer node
-        writeNode(writer,options);
-        //write types
-        writer.key("rdfTypes");
-        writer.array();
-        for(RdfType type:this.getTypes()){
-            writer.object();
-            writer.key("uri");writer.value(type.uri);
-            writer.key("curie");writer.value(type.curie);
-            writer.endObject();
-        }
-        writer.endArray();
-        //write links
-        writer.key("links");
-        writer.array();
-        for(int i=0;i<getLinkCount();i++){
-            Link l = getLink(i);
-            /*writer.object();
-            writer.key("uri");writer.value(l.propertyUri);
-            writer.key("target");
-            l.target.write(writer, options);
-            writer.endObject();*/
-            
-            l.write(writer, options);
-        }
-        writer.endArray();
-        
-        writer.endObject();
-        
-    }
-
-    
-    protected void addTypes(Resource r,Model model){
-        for(RdfType type:this.getTypes()){
-            r.addProperty(RDF.type, model.createResource(type.uri));
-        }
-    }
-    
-    protected Resource addLinks(Resource r,URI baseUri,Model model,Project project,Row row,int rowIndex,Resource[] blanks){
-        for(int i=0;i<getLinkCount();i++){
-            Link l = getLink(i);
-            String propertyUri;
-            try {
-                propertyUri = Util.getUri(baseUri, l.propertyUri);
-                Property p = model.createProperty(propertyUri);
-                RDFNode o = l.target.createNode(baseUri, model, project, row, rowIndex,blanks);
-                if(o!=null){
-                    r.addProperty(p, o);
-                }
-            } catch (URISyntaxException e) {
-                // TODO Auto-generated catch block
-                e.printStackTrace();
-                return null;
-            }
-        }
-        return r;
-    }
-
-    public void setTypes(List<RdfType> types) {
-        this.rdfTypes = types;
-    }
-    
-    public RDFNode createNode(URI baseUri,Model model,Project project,Row row,int rowIndex,Resource[] blanks) {
-        Resource r = createResource(baseUri, model, project, row, rowIndex,blanks);
-        if(r==null){
-            return null;
-        }
-        addTypes(r, model);
-        return addLinks(r,baseUri,model,project,row,rowIndex,blanks);
-    }
-    
-    public abstract Resource createResource(URI baseUri,Model model,Project project,Row row,int rowIndex,Resource[] blanks) ;
-    
-    public static class RdfType{
-        String uri;
-        String curie;
-        public RdfType(String uri,String curie){
-            this.uri = uri;
-            this.curie = curie;
-            
-        }
-    }
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/Util.java

@@ -1,47 +0,0 @@
-package com.metaweb.gridworks.rdf;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import com.metaweb.gridworks.model.Project;
-
-public class Util {
-
-    private static final String XSD_INT_URI ="http://www.w3.org/2001/XMLSchema#int";
-    private static final String XSD_DOUBLE_URI ="http://www.w3.org/2001/XMLSchema#double";
-    private static final String XSD_DATE_URI ="http://www.w3.org/2001/XMLSchema#date";
-    
-    public static String getUri(URI base,String rel)throws URISyntaxException{
-        return base.resolve(rel).toString();
-    }
-    
-    public static String getDataType(String s){
-        if(s==null){
-            return null;
-        }
-        if(s.equals(XSD_INT_URI)){
-            return XSD_INT_URI;
-        }
-        if(s.equals(XSD_DOUBLE_URI)){
-            return XSD_DOUBLE_URI;
-        }
-        if(s.equals(XSD_DATE_URI)){
-            return XSD_DATE_URI;
-        }
-        return null;
-    }
-    
-    public static RdfSchema getProjectSchema(Project project) {
-        synchronized (project) {
-            RdfSchema rdfSchema = (RdfSchema) project.overlayModels.get("rdfSchema");
-            if (rdfSchema == null) {
-                rdfSchema = new RdfSchema();
-                
-                project.overlayModels.put("rdfSchema", rdfSchema);
-                project.getMetadata().updateModified();
-            }
-            
-            return rdfSchema;
-        }
-    }
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/commands/DeleteVocabularyCommand.java

@@ -1,42 +0,0 @@
-package com.metaweb.gridworks.rdf.commands;
-
-import java.io.IOException;
-import java.util.Properties;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.metaweb.gridworks.Jsonizable;
-import com.metaweb.gridworks.commands.Command;
-import com.metaweb.gridworks.rdf.vocab.VocabularyManager;
-
-public class DeleteVocabularyCommand extends Command{
-
-    @Override
-    public void doPost(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        String uri = request.getParameter("uri");
-        try {    
-            VocabularyManager.getSingleton(servlet).deleteVocabulary(uri);
-            respondJSON(response, new Jsonizable() {
-                
-                @Override
-                public void write(JSONWriter writer, Properties options)
-                        throws JSONException {
-                    writer.object();
-                    writer.key("code"); writer.value("ok");
-                    writer.endObject();
-                }
-            });
-        } catch (JSONException e) {
-            respondException(response, e);
-        } catch (Exception e){
-            respondException(response, e);
-        }
-    }
-
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/commands/ImportVocabularyCommand.java

@@ -1,48 +0,0 @@
-package com.metaweb.gridworks.rdf.commands;
-
-import java.io.IOException;
-import java.util.Properties;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.metaweb.gridworks.Jsonizable;
-import com.metaweb.gridworks.commands.Command;
-import com.metaweb.gridworks.rdf.vocab.VocabularyExistException;
-import com.metaweb.gridworks.rdf.vocab.VocabularyManager;
-
-public class ImportVocabularyCommand extends Command{
-
-    @Override
-    public void doGet(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        String prefix = request.getParameter("prefix");
-        String url = request.getParameter("url");
-        String namespace = request.getParameter("namespace");
-        try {    
-            VocabularyManager.getSingleton(servlet).addVocabulary(url, prefix, namespace);
-            respondJSON(response, new Jsonizable() {
-                
-                @Override
-                public void write(JSONWriter writer, Properties options)
-                        throws JSONException {
-                    writer.object();
-                    writer.key("code"); writer.value("ok");
-                    writer.endObject();
-                }
-            });
-        } catch (JSONException e) {
-            respondException(response, e);
-        } catch (VocabularyExistException e) {
-            respondException(response, e);
-        } catch (Exception e){
-            respondException(response, e);
-        }
-    }
-
-    
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/commands/ListVocabulariesCommand.java

@@ -1,37 +0,0 @@
-package com.metaweb.gridworks.rdf.commands;
-
-import java.io.IOException;
-import java.util.Properties;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.json.JSONWriter;
-
-import com.metaweb.gridworks.commands.Command;
-import com.metaweb.gridworks.rdf.vocab.Vocabulary;
-import com.metaweb.gridworks.rdf.vocab.VocabularyManager;
-
-public class ListVocabulariesCommand extends Command{
-    @Override
-    public void doGet(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        response.setCharacterEncoding("UTF-8");
-        response.setHeader("Content-Type", "application/json");
-        try{
-            JSONWriter writer = new JSONWriter(response.getWriter());
-            writer.object();
-            writer.key("vocabularies");
-            writer.array();
-            Properties p = new Properties();
-            for(Vocabulary v:VocabularyManager.getSingleton(servlet).getVocabularies()){
-                v.write(writer, p);
-            }
-            writer.endArray();
-            writer.endObject();
-        } catch (Exception e) {
-            respondException(response, e);
-        }
-    }
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/commands/PreviewRdfCommand.java

@@ -1,106 +0,0 @@
-package com.metaweb.gridworks.rdf.commands;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.net.URI;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.ModelFactory;
-import com.hp.hpl.jena.rdf.model.Resource;
-import com.metaweb.gridworks.browsing.Engine;
-import com.metaweb.gridworks.browsing.FilteredRows;
-import com.metaweb.gridworks.browsing.RowVisitor;
-import com.metaweb.gridworks.commands.Command;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-import com.metaweb.gridworks.rdf.Node;
-import com.metaweb.gridworks.rdf.RdfSchema;
-import com.metaweb.gridworks.util.ParsingUtilities;
-
-public class PreviewRdfCommand extends Command {
-
-    @Override
-    public void doPost(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        try {
-            Project project = getProject(request);
-            Engine engine = getEngine(request, project);
-            FilteredRows filteredRows = engine.getAllFilteredRows();
-            
-            response.setCharacterEncoding("UTF-8");
-            response.setHeader("Content-Type", "application/json");
-            
-            String jsonString = request.getParameter("schema");
-            JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
-            RdfSchema schema = RdfSchema.reconstruct(json);
-            
-            Model model = ModelFactory.createDefaultModel();
-            URI baseUri = schema.getBaseUri();
-            Node root = schema.getRoot();
-            
-            Resource[] blanks = new Resource[schema.get_blanks().size()];
-            for (int i = 0; i < blanks.length; i++) {
-                blanks[i] = model.createResource();
-            }
-            
-            RowVisitor visitor = new RdfRowVisitor(model, baseUri, root,blanks,20);
-            
-            filteredRows.accept(project, visitor);
-            StringWriter sw = new StringWriter();
-            model.write(sw,"N3");
-            
-            JSONWriter writer = new JSONWriter(response.getWriter());
-            writer.object();
-            writer.key("v");
-            writer.value(sw.getBuffer().toString());
-            writer.endObject();
-            //respond(response, "{v:" + sw.getBuffer().toString() + "}");
-        }catch (Exception e) {
-            respondException(response, e);
-        }
-    }
-    
-
-    protected static class RdfRowVisitor implements RowVisitor{
-        Model model;
-        URI base;
-        Node root;
-        Resource[] blanks;
-        int limit;
-        
-        int _count;
-        public RdfRowVisitor(Model m,URI base, Node root,Resource[] blanks,int l){
-            this.model = m;
-            this.base = base;
-            this.root = root;
-            this.blanks = blanks;
-            this.limit = l;
-        }
-        public void end(Project project) {
-            // do nothing
-            
-        }
-
-        public void start(Project project) {
-            // do nothing
-            
-        }
-
-        public boolean visit(Project project, int rowIndex, Row row) {
-            if(_count>=limit){
-                return true;
-            }
-            root.createNode(base, model, project, row, rowIndex,blanks);
-            _count +=1;
-            return false;
-        }
-    }
-
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/commands/SaveBaseURI.java

@@ -1,32 +0,0 @@
-package com.metaweb.gridworks.rdf.commands;
-
-import java.io.IOException;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import com.metaweb.gridworks.commands.Command;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.rdf.Util;
-
-public class SaveBaseURI extends Command {
-
-    @Override
-    public void doPost(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        try {
-            Project project = getProject(request);
-            String base = request.getParameter("baseURI");
-            
-            Util.getProjectSchema(project).setBaseUri(base);
-            
-            project.getMetadata().updateModified();
-            
-            respond(response,"OK","");
-            
-        }catch(Exception e){
-            respondException(response, e);
-        }
-    }
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/commands/SaveRdfSchemaCommand.java

@@ -1,64 +0,0 @@
-package com.metaweb.gridworks.rdf.commands;
-
-import java.io.IOException;
-import java.util.Properties;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.json.JSONObject;
-
-import com.metaweb.gridworks.commands.Command;
-import com.metaweb.gridworks.model.AbstractOperation;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.process.Process;
-import com.metaweb.gridworks.rdf.RdfSchema;
-import com.metaweb.gridworks.rdf.operations.SaveRdfSchemaOperation;
-import com.metaweb.gridworks.util.ParsingUtilities;
-
-public class SaveRdfSchemaCommand extends Command{
-
-    @Override
-    public void doPost(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        
-        try {
-            Project project = getProject(request);
-            
-            String jsonString = request.getParameter("schema");
-            JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
-            RdfSchema schema = RdfSchema.reconstruct(json);
-            
-            AbstractOperation op = new SaveRdfSchemaOperation(schema);
-            Process process = op.createProcess(project, new Properties());
-            
-            performProcessAndRespond(request, response, project, process);
-            
-            /*project.schema = schema;
-            
-            response.setCharacterEncoding("UTF-8");
-            response.setHeader("Content-Type", "application/json");
-            
-            Writer w = response.getWriter();
-            JSONWriter writer = new JSONWriter(w);
-            writer.object();
-            writer.key("code"); writer.value("ok");
-            writer.key("historyEntry"); 
-            
-            //dummy history for now
-            writer.object();
-            writer.key("op"); writer.value("saveRdfSchema");
-            writer.key("description"); writer.value("Save RDF schema");
-            writer.endObject();
-            
-            writer.endObject();
-            
-            w.flush();
-            w.close();*/
-            
-        } catch (Exception e) {
-            respondException(response, e);
-        }
-    }
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/commands/SuggestTermCommand.java

@@ -1,99 +0,0 @@
-package com.metaweb.gridworks.rdf.commands;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.metaweb.gridworks.Jsonizable;
-import com.metaweb.gridworks.commands.Command;
-import com.metaweb.gridworks.rdf.vocab.RDFNode;
-import com.metaweb.gridworks.rdf.vocab.VocabularyManager;
-
-public class SuggestTermCommand extends Command{
-
-    @Override
-    public void doGet(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        
-        
-        response.setHeader("Content-Type", "application/json");
-        JSONWriter writer = new JSONWriter(response.getWriter());
-        
-        String type = request.getParameter("type_strict");
-        
-        String prefix = request.getParameter("prefix");
-        try{
-            writer.object();
-            
-            writer.key("prefix");
-            writer.value(prefix);
-            
-            writer.key("result");
-            writer.array();
-            List<RDFNode> nodes;
-            if(type!=null && type.trim().equals("property")){
-                nodes = VocabularyManager.getSingleton(servlet).searchProperties(prefix);
-            }else{
-                nodes = VocabularyManager.getSingleton(servlet).searchClasses(prefix);
-            }
-            for(RDFNode c:nodes){
-                c.writeAsSearchResult(writer);
-            }
-            writer.endArray();
-            writer.endObject();
-        }catch(Exception e){
-            e.printStackTrace();
-            throw new ServletException(e);
-        }
-    }
-
-    
-}
-
-class Result implements Jsonizable{
-
-    private List<String[]> results = new ArrayList<String[]>();
-    private String prefix;
-    
-    Result(String p){
-        this.prefix = p;
-    }
-    void addResult(String id, String name){
-        String[] res = new String[] {id,name};
-        results.add(res);
-    }
-    @Override
-    public void write(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.object();
-        
-        
-        writer.key("prefix");
-        writer.value(prefix);
-        
-        writer.key("result");
-        writer.array();
-        for(String[] res:results){
-            writer.object();
-            
-            writer.key("id");
-            writer.value(res[0]);
-            
-            writer.key("name");
-            writer.value(res[1]);
-            
-            writer.endObject();
-        }
-        writer.endArray();
-        writer.endObject();
-    }
-    
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/exporters/RdfExporter.java

@@ -1,106 +0,0 @@
-package com.metaweb.gridworks.rdf.exporters;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.Writer;
-import java.net.URI;
-import java.util.Properties;
-
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.ModelFactory;
-import com.hp.hpl.jena.rdf.model.Resource;
-import com.metaweb.gridworks.browsing.Engine;
-import com.metaweb.gridworks.browsing.FilteredRows;
-import com.metaweb.gridworks.browsing.RowVisitor;
-import com.metaweb.gridworks.exporters.Exporter;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-import com.metaweb.gridworks.rdf.Node;
-import com.metaweb.gridworks.rdf.RdfSchema;
-import com.metaweb.gridworks.rdf.Util;
-
-public class RdfExporter implements Exporter{
-
-    private String format;
-    
-    public RdfExporter(String f){
-        this.format = f;
-    }
-    public void export(Project project, Properties options, Engine engine,
-            OutputStream outputStream) throws IOException {
-        
-        RdfSchema schema = Util.getProjectSchema(project);
-        Model model = ModelFactory.createDefaultModel();
-        URI baseUri = schema.getBaseUri();
-        Node root = schema.getRoot();
-        
-        Resource[] blanks = new Resource[schema.get_blanks().size()];
-        for (int i = 0; i < blanks.length; i++) {
-            blanks[i] = model.createResource();
-        }
-        
-        RowVisitor visitor = new RdfRowVisitor(model, baseUri, root,blanks);
-        FilteredRows filteredRows = engine.getAllFilteredRows();
-        filteredRows.accept(project, visitor);
-        model.write(outputStream);
-    }
-
-    
-    public void export(Project project, Properties options, Engine engine,
-            Writer writer) throws IOException {
-        RdfSchema schema = Util.getProjectSchema(project);
-        Model model = ModelFactory.createDefaultModel();
-        URI baseUri = schema.getBaseUri();
-        Node root = schema.getRoot();
-        
-        Resource[] blanks = new Resource[schema.get_blanks().size()];
-        for (int i = 0; i < blanks.length; i++) {
-            blanks[i] = model.createResource();
-        }
-
-        RowVisitor visitor = new RdfRowVisitor(model, baseUri, root,blanks);
-        FilteredRows filteredRows = engine.getAllFilteredRows();
-        filteredRows.accept(project, visitor);
-        model.write(writer,format);
-    }
-
-    public String getContentType() {
-        if(format.equals("N3")){
-            return "text/rdf+n3";
-        }else{
-            return "application/rdf+xml";
-        }
-    }
-
-    public boolean takeWriter() {
-        return true;
-    }
-    
-    protected static class RdfRowVisitor implements RowVisitor{
-        Model model;
-        URI base;
-        Node root;
-        Resource[] blanks;
-        public RdfRowVisitor(Model m,URI base, Node root,Resource[] blanks){
-            this.model = m;
-            this.base = base;
-            this.root = root;
-            this.blanks = blanks;
-        }
-        public void end(Project project) {
-            // do nothing
-            
-        }
-
-        public void start(Project project) {
-            // do nothing
-            
-        }
-
-        public boolean visit(Project project, int rowIndex, Row row) {
-            root.createNode(base, model, project, row, rowIndex,blanks);
-            return false;
-        }
-    }
-
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/expr/RdfBinder.java

@@ -1,23 +0,0 @@
-package com.metaweb.gridworks.rdf.expr;
-
-import java.util.Properties;
-
-import com.metaweb.gridworks.expr.Binder;
-import com.metaweb.gridworks.model.Cell;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-import com.metaweb.gridworks.rdf.Util;
-
-public class RdfBinder implements Binder {
-
-    @Override
-    public void bind(Properties bindings, Row row, int rowIndex, String columnName, Cell cell) {
-        // nothing to do
-    }
-
-    @Override
-    public void initializeBindings(Properties bindings, Project project) {
-        bindings.put("baseURI", Util.getProjectSchema(project).getBaseUri());
-    }
-
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/expr/functions/strings/Urlify.java

@@ -1,51 +0,0 @@
-package com.metaweb.gridworks.rdf.expr.functions.strings;
-
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.metaweb.gridworks.expr.EvalError;
-import com.metaweb.gridworks.gel.ControlFunctionRegistry;
-import com.metaweb.gridworks.gel.Function;
-
-public class Urlify implements Function {
-
-    public Object call(Properties bindings, Object[] args) {
-        if(args.length==1 || args.length==2){
-            String s = args[0].toString();
-            s = s.replaceAll("\\s+", "_");
-            if(args.length==2){
-                String base = args[1].toString();
-                URI base_uri;
-                try {
-                    base_uri = new URI(base);
-                    return base_uri.resolve(s).toString();
-                } catch (URISyntaxException e) {
-                    return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " unable to encode");
-                }
-            }
-            try {
-                return URLEncoder.encode(s, "UTF-8");
-            } catch (UnsupportedEncodingException e) {
-                return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " unable to encode");
-            }
-        }
-        return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 1 string");
-    }
-
-    public void write(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.object();
-        writer.key("description"); writer.value("replaces spaces with underscore");
-        writer.key("params"); writer.value("string s");
-        writer.key("returns"); writer.value("string");
-        writer.endObject();
-        
-    }
-
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/operations/SaveRdfSchemaOperation.java

@@ -1,137 +0,0 @@
-package com.metaweb.gridworks.rdf.operations;
-
-import java.io.IOException;
-import java.io.LineNumberReader;
-import java.io.Writer;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONWriter;
-
-import com.metaweb.gridworks.history.Change;
-import com.metaweb.gridworks.history.HistoryEntry;
-import com.metaweb.gridworks.model.AbstractOperation;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.operations.OperationRegistry;
-import com.metaweb.gridworks.rdf.RdfSchema;
-import com.metaweb.gridworks.util.ParsingUtilities;
-import com.metaweb.gridworks.util.Pool;
-
-public class SaveRdfSchemaOperation extends AbstractOperation {
-
-    final protected RdfSchema _schema;
-
-    public SaveRdfSchemaOperation(RdfSchema schema) {
-        this._schema = schema;
-    }
-
-    static public AbstractOperation reconstruct(Project project, JSONObject obj)
-            throws Exception {
-        return new SaveRdfSchemaOperation(RdfSchema.reconstruct(obj
-                .getJSONObject("schema")));
-    }
-
-    public void write(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.object();
-        writer.key("op");
-        writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
-        writer.key("description");
-        writer.value("Save RDF schema skeleton");
-        writer.key("schema");
-        _schema.write(writer, options);
-        writer.endObject();
-
-    }
-
-    @Override
-    protected String getBriefDescription(Project project) {
-        return "Save RDF schema skelton";
-    }
-
-    @Override
-    protected HistoryEntry createHistoryEntry(Project project,
-            long historyEntryID) throws Exception {
-        String description = "Save RDF schema skeleton";
-        
-        Change change = new RdfSchemaChange(_schema);
-        
-        return new HistoryEntry(historyEntryID, project, description,
-                SaveRdfSchemaOperation.this, change);
-    }
-
-    static public class RdfSchemaChange implements Change {
-        final protected RdfSchema _newSchema;
-        protected RdfSchema _oldSchema;
-        
-        public RdfSchemaChange(RdfSchema schema) {
-            _newSchema = schema;
-        }
-        
-        public void apply(Project project) {
-            synchronized (project) {
-                _oldSchema = (RdfSchema) project.overlayModels.get("rdfSchema");
-                project.overlayModels.put("rdfSchema", _newSchema);
-            }
-        }
-        
-        public void revert(Project project) {
-            synchronized (project) {
-                if (_oldSchema == null) {
-                    project.overlayModels.remove("rdfSchema");
-                } else {
-                    project.overlayModels.put("rdfSchema", _oldSchema);
-                }
-            }
-        }
-        
-        public void save(Writer writer, Properties options) throws IOException {
-            writer.write("newSchema=");
-            writeRdfSchema(_newSchema, writer);
-            writer.write('\n');
-            writer.write("oldSchema=");
-            writeRdfSchema(_oldSchema, writer);
-            writer.write('\n');
-            writer.write("/ec/\n"); // end of change marker
-        }
-        
-        static public Change load(LineNumberReader reader, Pool pool)
-                throws Exception {
-            RdfSchema oldSchema = null;
-            RdfSchema newSchema = null;
-            
-            String line;
-            while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
-                int equal = line.indexOf('=');
-                CharSequence field = line.subSequence(0, equal);
-                String value = line.substring(equal + 1);
-                
-                if ("oldSchema".equals(field) && value.length() > 0) {
-                    oldSchema = RdfSchema.reconstruct(ParsingUtilities
-                            .evaluateJsonStringToObject(value));
-                } else if ("newSchema".equals(field) && value.length() > 0) {
-                    newSchema = RdfSchema.reconstruct(ParsingUtilities
-                            .evaluateJsonStringToObject(value));
-                }
-            }
-            
-            RdfSchemaChange change = new RdfSchemaChange(newSchema);
-            change._oldSchema = oldSchema;
-            
-            return change;
-        }
-        
-        static protected void writeRdfSchema(RdfSchema s, Writer writer)
-                throws IOException {
-            if (s != null) {
-                JSONWriter jsonWriter = new JSONWriter(writer);
-                try {
-                    s.write(jsonWriter, new Properties());
-                } catch (JSONException e) {
-                    e.printStackTrace();
-                }
-            }
-        }
-    }
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/vocab/RDFNode.java

@@ -1,130 +0,0 @@
-package com.metaweb.gridworks.rdf.vocab;
-
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.metaweb.gridworks.Jsonizable;
-
-public abstract class RDFNode implements Jsonizable{
-    private String preferredCURIE;
-    private String description;
-    private String URI;
-    private String label;
-    private String vocabularyPrefix;
-    private String vocabularyUri;
-    
-    public String getVocabularyUri() {
-        return vocabularyUri;
-    }
-    public void setVocabularyUri(String vocabularyUri) {
-        this.vocabularyUri = vocabularyUri;
-    }
-    public String getVocabularyPrefix() {
-        return vocabularyPrefix;
-    }
-    public void setVocabularyPrefix(String vocabularyPrefix) {
-        this.vocabularyPrefix = vocabularyPrefix;
-    }
-    public String getPreferredCURIE() {
-        return preferredCURIE;
-    }
-    public void setPreferredCURIE(String preferredCURIE) {
-        this.preferredCURIE = preferredCURIE;
-    }
-    public String getDescription() {
-        return description;
-    }
-    public void setDescription(String description) {
-        this.description = description;
-    }
-    public String getURI() {
-        return URI;
-    }
-    public void setURI(String uRI) {
-        URI = uRI;
-    }
-    public String getLabel() {
-        return label;
-    }
-    public void setLabel(String label) {
-        this.label = label;
-    }
-    public RDFNode(){
-        
-    }
-    public RDFNode(String description, String uRI,
-            String label,String prefix,String vocabularyUri) {
-        this.description = description;
-        URI = uRI;
-        this.label = label;
-        this.vocabularyPrefix = prefix;
-        this.preferredCURIE = composePreferredCurie();
-        this.vocabularyUri = vocabularyUri;
-    }
-    private String composePreferredCurie(){
-        String qname;
-        if(this.URI==null){
-            return null;
-        }
-        if(this.URI.indexOf("#")!=-1){
-            qname = this.URI.substring(this.URI.indexOf("#")+1);
-        }else{
-            qname = this.URI.substring(this.URI.lastIndexOf("/")+1);
-        }
-        return this.vocabularyPrefix + ":" + qname;
-    }
-    
-    
-    public abstract String getType();
-    public void write(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.object();
-        
-        writer.key("type");
-        writer.value(this.getType());
-        writer.key("prefix");
-        writer.value(vocabularyPrefix);
-        writer.key("preferredCURIE");
-        writer.value(this.preferredCURIE);
-        writer.key("label");
-        writer.value(label);
-        writer.key("description");
-        writer.value(description);
-        writer.key("URI");
-        writer.value(URI);
-        writer.endObject();
-    }
-    
-    public void writeAsSearchResult(JSONWriter writer)throws JSONException {
-        writer.object();
-        
-        writer.key("id");
-        writer.value(URI);
-        
-        writer.key("name");
-        writer.value(preferredCURIE);
-        
-        writer.key("description");
-        writer.value(description);
-        writer.endObject();
-    }
-    @Override
-    public boolean equals(Object obj) {
-        if(!(obj instanceof RDFNode)) return false;
-        RDFNode n = (RDFNode) obj;
-        if(n.getURI()==null || this.URI==null){
-            return false;
-        }
-        return this.URI.equals(n.getURI());
-    }
-    @Override
-    public int hashCode() {
-        return this.URI.hashCode();
-    }
-    
-    
-
-    
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/vocab/RDFSClass.java

@@ -1,16 +0,0 @@
-package com.metaweb.gridworks.rdf.vocab;
-
-
-public class RDFSClass extends RDFNode{
-
-    @Override
-    public String getType() {
-        return "class";
-    }
-
-    public RDFSClass( String uRI,
-            String label,String description,String prefix,String vocabularyUri) {
-        super(description,uRI,label,prefix,vocabularyUri);
-    }
-    
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/vocab/RDFSProperty.java

@@ -1,14 +0,0 @@
-package com.metaweb.gridworks.rdf.vocab;
-
-
-public class RDFSProperty extends RDFNode{
-
-    @Override
-    public String getType() {
-        return "property";
-    }
-    public RDFSProperty(String uRI,
-            String label,String description,String prefix,String vocabularyUri) {
-        super(description,uRI,label,prefix,vocabularyUri);
-    }
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/vocab/Vocabulary.java

@@ -1,65 +0,0 @@
-package com.metaweb.gridworks.rdf.vocab;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
-import org.json.JSONException;
-import org.json.JSONWriter;
-
-import com.metaweb.gridworks.Jsonizable;
-
-public class Vocabulary implements Jsonizable{
-    private final String name;
-    private final String uri;
-    private List<RDFSClass> classes = new ArrayList<RDFSClass>();
-    private List<RDFSProperty> properties = new ArrayList<RDFSProperty>();
-    
-    public Vocabulary(String name,String uri){
-        this.name = name;
-        this.uri = uri;
-    }
-    
-    public void addClass(RDFSClass clazz){
-        this.classes.add(clazz);
-    }
-    
-    public void addProperty(RDFSProperty prop){
-        this.properties.add(prop);
-    }
-    
-    public List<RDFSClass> getClasses() {
-        return classes;
-    }
-
-    public void setClasses(List<RDFSClass> classes) {
-        this.classes = classes;
-    }
-
-    public List<RDFSProperty> getProperties() {
-        return properties;
-    }
-
-    public void setProperties(List<RDFSProperty> properties) {
-        this.properties = properties;
-    }
-
-    public String getName() {
-        return name;
-    }
-
-    public String getUri() {
-        return uri;
-    }
-
-    @Override
-    public void write(JSONWriter writer, Properties options)
-            throws JSONException {
-        writer.object();
-        
-        writer.key("name"); writer.value(name);
-        writer.key("uri"); writer.value(uri);
-        
-        writer.endObject();
-    }
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/vocab/VocabularyExistException.java

@@ -1,10 +0,0 @@
-package com.metaweb.gridworks.rdf.vocab;
-
-public class VocabularyExistException extends Exception {
-    private static final long serialVersionUID = 1916094460059608851L;
-
-    public VocabularyExistException(String msg){
-        super(msg);
-    }
-
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/vocab/VocabularyImporter.java

@@ -1,119 +0,0 @@
-package com.metaweb.gridworks.rdf.vocab;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import com.hp.hpl.jena.ontology.OntModelSpec;
-import com.hp.hpl.jena.query.Query;
-import com.hp.hpl.jena.query.QueryExecution;
-import com.hp.hpl.jena.query.QueryExecutionFactory;
-import com.hp.hpl.jena.query.QueryFactory;
-import com.hp.hpl.jena.query.QuerySolution;
-import com.hp.hpl.jena.query.ResultSet;
-import com.hp.hpl.jena.rdf.model.Literal;
-import com.hp.hpl.jena.rdf.model.Model;
-import com.hp.hpl.jena.rdf.model.ModelFactory;
-
-public class VocabularyImporter {
-
-    private static final String PREFIXES = "PREFIX rdfs:<http://www.w3.org/2000/01/rdf-schema#> " +
-            "PREFIX rdf:<http://www.w3.org/1999/02/22-rdf-syntax-ns#> " +
-            "PREFIX skos:<http://www.w3.org/2004/02/skos/core#> ";
-    private static final String CLASSES_QUERY_P1 = PREFIXES + 
-    "SELECT ?resource ?label ?en_label ?description ?en_description ?definition ?en_definition " +
-    "WHERE { " +
-        "?resource rdf:type rdfs:Class. " +
-        "OPTIONAL {?resource rdfs:label ?label.} " +
-        "OPTIONAL {?resource rdfs:label ?en_label. FILTER langMatches( lang(?en_label), \"EN\" )  } " +
-        "OPTIONAL {?resource rdfs:comment ?description.} " +
-        "OPTIONAL {?resource rdfs:comment ?en_description. FILTER langMatches( lang(?en_description), \"EN\" )  } " +
-        "OPTIONAL {?resource skos:definition ?definition.} " +
-        "OPTIONAL {?resource skos:definition ?en_definition. FILTER langMatches( lang(?en_definition), \"EN\" )  } " +
-        "FILTER regex(str(?resource), \"^";
-    private static final String CLASSES_QUERY_P2 = "\")}";
-    
-    private static final String PROPERTIES_QUERY_P1 = PREFIXES + 
-    "SELECT ?resource ?label ?en_label ?description ?en_description ?definition ?en_definition " +
-    "WHERE { " +
-        "?resource rdf:type rdf:Property. " +
-        "OPTIONAL {?resource rdfs:label ?label.} " +
-        "OPTIONAL {?resource rdfs:label ?en_label. FILTER langMatches( lang(?en_label), \"EN\" )  } " +
-        "OPTIONAL {?resource rdfs:comment ?description.} " +
-        "OPTIONAL {?resource rdfs:comment ?en_description. FILTER langMatches( lang(?en_description), \"EN\" )  } " +
-        "OPTIONAL {?resource skos:definition ?definition.} " +
-        "OPTIONAL {?resource skos:definition ?en_definition. FILTER langMatches( lang(?en_definition), \"EN\" )  } " +
-        "FILTER regex(str(?resource), \"^";
-    private static final String PROPERTIES_QUERY_P2 = "\")}"; 
-    
-    public Vocabulary getVocabulary(String url, String prefix, String namespace, String format){
-        Model m = getModel(url, format);
-        return getVocabulary(m,namespace,prefix);
-    }
-    
-    private Model getModel(String url,String format){
-        Model model = ModelFactory.createOntologyModel(OntModelSpec.OWL_DL_MEM_RDFS_INF);//ModelFactory.createDefaultModel();
-        model.read(url);
-        return model;
-    }
-    
-
-    private Vocabulary getVocabulary(Model m, String namespace, String prefix){
-        Query query = QueryFactory.create(CLASSES_QUERY_P1 + namespace.trim() + CLASSES_QUERY_P2);
-        QueryExecution qe = QueryExecutionFactory.create(query, m);
-        ResultSet res = qe.execSelect();
-        Set<String> seen = new HashSet<String>();
-        Vocabulary vocab = new Vocabulary(prefix, namespace);
-        while(res.hasNext()){
-            QuerySolution qs = res.nextSolution();
-            String uri = qs.getResource("resource").getURI();
-            if(seen.contains(uri)){
-                continue;
-            }
-            String label = getFirstNotNull(new Literal[]{qs.getLiteral("en_label"),qs.getLiteral("label")});
-            
-            String description = getFirstNotNull(new Literal[]{qs.getLiteral("en_definition"),qs.getLiteral("definition"),
-                    qs.getLiteral("en_description"),qs.getLiteral("description")}) ;
-            RDFSClass clazz = new RDFSClass(uri, label, description,prefix,namespace);
-            vocab.addClass(clazz);
-        }
-        
-        query = QueryFactory.create(PROPERTIES_QUERY_P1 + namespace.trim() + PROPERTIES_QUERY_P2);
-        qe = QueryExecutionFactory.create(query, m);
-        res = qe.execSelect();
-        seen = new HashSet<String>();
-        while(res.hasNext()){
-            QuerySolution qs = res.nextSolution();
-            String uri = qs.getResource("resource").getURI();
-            if(seen.contains(uri)){
-                continue;
-            }
-            String label = getFirstNotNull(new Literal[]{qs.getLiteral("en_label"),qs.getLiteral("label")});
-            
-            String description = getFirstNotNull(new Literal[]{qs.getLiteral("en_definition"),qs.getLiteral("definition"),
-                    qs.getLiteral("en_description"),qs.getLiteral("description")}) ;
-            RDFSProperty prop = new RDFSProperty(uri, label, description,prefix,namespace);
-            vocab.addProperty(prop);
-        }
-        
-        return vocab;
-    }
-    
-    private String getFirstNotNull(Literal[] literals){
-        String s = null;
-        for(int i=0;i<literals.length;i++){
-            s = getString(literals[i]);
-            if(s!=null){
-                break;
-            }
-        }
-        return s;
-    }
-    
-    private String getString(Literal l){
-        if(l!=null){
-            return l.getString();
-        }
-        return null;
-    }
-    
-}

extensions/rdf-exporter/src/com/metaweb/gridworks/rdf/vocab/VocabularyManager.java

@@ -1,244 +0,0 @@
-package com.metaweb.gridworks.rdf.vocab;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.SimpleFSDirectory;
-import org.apache.lucene.util.Version;
-
-import com.metaweb.gridworks.GridworksServlet;
-
-/**
- * @author fadmaa
- *
- */
-public class VocabularyManager {
-
-    private static final String CLASS_TYPE = "class";
-    private static final String PROPERTY_TYPE = "property";
-    private static final String VOCABULARY_TYPE = "vocabulary";
-    
-    private IndexWriter writer;
-    private IndexSearcher searcher;
-    
-    private Directory _directory;
-    
-    private static VocabularyManager singleton;
-    
-    private List<Vocabulary> vocabularies = new ArrayList<Vocabulary>();
-    
-    static public VocabularyManager getSingleton(GridworksServlet servlet) {
-        return singleton != null ? singleton : (singleton = new VocabularyManager(servlet));
-    }
-    
-    private VocabularyManager(GridworksServlet servlet) {
-        try{
-            synchronized (this) {
-                File dir = servlet.getCacheDir("rdfImporter");
-                _directory = new SimpleFSDirectory(new File(dir, "luceneIndex"));
-                writer = new IndexWriter(_directory, new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.LIMITED);
-                searcher = new IndexSearcher(_directory);
-                updateVocabulariesList();                
-            }
-        } catch (CorruptIndexException e) {
-            throw new RuntimeException("Failed initialize vocabulary search",e);
-        } catch (IOException e) {
-            throw new RuntimeException("Failed initialize vocabulary search",e);
-        }
-    }
-    
-    
-    /**
-     * @param url where to get the vocabulary description from
-     * @param prefix preferred prefix for vocabulary e.g. foaf, dc, skos, dcat
-     * @param namespace the base URI of the vocabulary. usually but not alway the same as url
-     * @param format the format of the RDF description of the vocabulary at the end of url (default to RDF/XML)
-     * @throws IOException 
-     * @throws CorruptIndexException 
-     * @throws VocabularyExistException 
-     */
-    public void addVocabulary(String url, String prefix, String namespace, String format) throws CorruptIndexException, IOException, VocabularyExistException{
-        if (defined(namespace)){
-            throw new VocabularyExistException(namespace + " already exists!");
-        }
-        VocabularyImporter importer = new VocabularyImporter();
-        Vocabulary vocabulary = importer.getVocabulary(url,prefix,namespace,format);
-        indexVocabulary(vocabulary);
-        updateSearcher();    
-//        updateVocabulariesList();
-        this.vocabularies.add(vocabulary);
-    }
-    
-    public void addVocabulary(String url, String prefix, String namespace) throws CorruptIndexException, IOException, VocabularyExistException{
-        addVocabulary(url, prefix, namespace,"RDF/XML");
-    }
-    private void indexVocabulary(Vocabulary v) throws CorruptIndexException, IOException{
-        Document doc = new Document();
-        doc.add(new Field("type",VOCABULARY_TYPE,Field.Store.YES,Field.Index.NOT_ANALYZED));
-        doc.add(new Field("name",v.getName(),Field.Store.YES,Field.Index.NO));
-        doc.add(new Field("uri",v.getUri(),Field.Store.YES,Field.Index.NOT_ANALYZED));
-        writer.addDocument(doc);
-        
-        for(RDFSClass c:v.getClasses()){
-            indexRdfNode(c, CLASS_TYPE);
-        }
-        for(RDFSProperty p:v.getProperties()){
-            indexRdfNode(p, PROPERTY_TYPE);
-        }
-        
-        writer.commit();
-    }
-    
-    public List<RDFNode> searchClasses(String str)throws ParseException, IOException{
-        List<RDFNode> res = new ArrayList<RDFNode>();
-        org.apache.lucene.search.Query query = prepareQuery(str, "class");
-        TopDocs docs = searcher.search(query, 1000);
-        for(int i=0;i<docs.totalHits;i++){
-            Document doc= searcher.doc(docs.scoreDocs[i].doc);
-            String uri = doc.get("uri");
-            String label = doc.get("label");
-            String description = doc.get("description");
-            String namespace = doc.get("namespace"); 
-            String prefix = doc.get("prefix");
-            RDFSClass node = new RDFSClass(uri,label,description,prefix,namespace);
-            res.add(node);
-        }
-        
-        return res;
-    }
-    
-    public List<RDFNode> searchProperties(String str)throws ParseException, IOException{
-        List<RDFNode> res = new ArrayList<RDFNode>();
-        org.apache.lucene.search.Query query = prepareQuery(str, "property");
-        TopDocs docs = searcher.search(query, 1000);
-        for(int i=0;i<docs.totalHits;i++){
-            Document doc= searcher.doc(docs.scoreDocs[i].doc);
-            String uri = doc.get("uri");
-            String label = doc.get("label");
-            String description = doc.get("description");
-            String namespace = doc.get("namespace"); 
-            String prefix = doc.get("prefix");
-            RDFNode node = new RDFSProperty(uri,label,description,prefix,namespace);
-            res.add(node);
-        }
-        
-        return res;
-    }
-    
-    public List<Vocabulary> getVocabularies(){
-        return vocabularies;
-    }
-    
-    public void deleteVocabulary(String uri) throws IOException{
-        Vocabulary vocab = null;
-        for(Vocabulary v:vocabularies){
-            if(v.getUri().equals(uri)){
-                vocab = v;
-                break;
-            }
-        }
-        if(vocab==null){
-            throw new RuntimeException("Vocabulary " + uri + " not found");
-        }
-        vocabularies.remove(vocab);
-        Term t = new Term("uri",uri);
-        writer.deleteDocuments(t);
-        t = new Term("namespace",uri);
-        writer.deleteDocuments(t);
-        
-        writer.commit();
-        updateSearcher();
-    }
-    
-    private org.apache.lucene.search.Query prepareQuery(String s,String type)throws ParseException{
-        QueryParser parser = new QueryParser(Version.LUCENE_30,"description",new StandardAnalyzer(Version.LUCENE_30));
-        String queryStr = "type:" + type ;
-        if(s!=null && s.trim().length()>0){
-            s =s.trim();
-            if(s.indexOf("*")==-1){
-                s += "*";
-            }
-            if(s.indexOf(":")==-1){
-                queryStr += " AND (curie:" + s ;
-                queryStr += " OR description:" + s ;
-                queryStr += " OR label:" + s + ")";
-            }else{
-                String p1 = s.substring(0,s.indexOf(":"));
-                String p2 = s.substring(s.lastIndexOf(":")+1);
-                queryStr += " AND prefix:" + p1;
-                if(p2.length()>1){
-                    //queryStr += " AND (description:" + p2;
-                    queryStr += " AND label:" + p2;
-                }
-            }
-            
-        }
-        return parser.parse(queryStr);
-    }
-    
-    private void indexRdfNode(RDFNode node, String type) throws CorruptIndexException, IOException{
-        //TODO weight fields... setBoost
-        Document doc = new Document();
-        doc.add(new Field("type",type,Field.Store.YES,Field.Index.NOT_ANALYZED));
-        doc.add(new Field("prefix",node.getVocabularyPrefix(),Field.Store.YES,Field.Index.NOT_ANALYZED));
-        String l = node.getLabel()==null?"":node.getLabel();
-        Field labelField = new Field("label",l,Field.Store.NO,Field.Index.ANALYZED);
-        doc.add(labelField);
-        String d = node.getDescription()==null?"":node.getDescription();
-        Field descriptionField = new Field("description",d,Field.Store.YES,Field.Index.ANALYZED);
-        doc.add(descriptionField);
-        doc.add(new Field("uri", node.getURI(),Field.Store.YES,Field.Index.NO));
-        Field curieField = new Field("curie", node.getPreferredCURIE(),Field.Store.YES,Field.Index.ANALYZED);
-        doc.add(curieField);
-        Field namespaceField = new Field("namespace", node.getVocabularyUri(),Field.Store.YES,Field.Index.NOT_ANALYZED);
-        doc.add(namespaceField);
-        writer.addDocument(doc);
-    }
-    
-    private void updateVocabulariesList() throws IOException{
-        Term typeTerm = new Term("type", VOCABULARY_TYPE);
-        Query query = new TermQuery(typeTerm);
-        //TODO 1000 :O
-        TopDocs vocabDocs = searcher.search(query, 1000);
-        for(int i=0;i<vocabDocs.totalHits;i++){
-            Document doc = searcher.doc(vocabDocs.scoreDocs[i].doc);
-            String name = doc.get("name");
-            String uri = doc.get("uri");
-            Vocabulary vocab = new Vocabulary(name, uri);
-            this.vocabularies.add(vocab);
-        }
-        
-    }
-    
-    private void updateSearcher() throws IOException{
-        this.searcher = new IndexSearcher(_directory);
-    }
-    
-    private boolean defined(String namespace){
-        if(namespace==null){
-            return false;
-        }
-        namespace = namespace.trim();
-        for(Vocabulary v:vocabularies){
-            if(v.getUri().equals(namespace)){
-                return true;
-            }
-        }
-        return false;
-    }
-}

extensions/sample-extension/module/MOD-INF/controller.js

@@ -1,6 +1,6 @@
 var html = "text/html";
 var encoding = "UTF-8";
-var ClientSideResourceManager = Packages.com.metaweb.gridworks.ClientSideResourceManager;
+var ClientSideResourceManager = Packages.com.google.gridworks.ClientSideResourceManager;
 
 /*
  * Function invoked to initialize the extension.
@@ -39,7 +39,7 @@ function process(path, request, response) {
         // here's how to pass things into the .vt templates
         context.someList = ["Superior","Michigan","Huron","Erie","Ontario"];
         context.someString = "foo";
-        context.someInt = Packages.com.metaweb.gridworks.sampleExtension.SampleUtil.stringArrayLength(context.someList);
+        context.someInt = Packages.com.google.gridworks.sampleExtension.SampleUtil.stringArrayLength(context.someList);
         
         send(request, response, "index.vt", context);
     }

extensions/sample-extension/src/com/google/gridworks/sampleExtension/SampleUtil.java

@@ -0,0 +1,8 @@
+package com.google.gridworks.sampleExtension;
+
+
+public class SampleUtil {
+    static public int stringArrayLength(String[] a) {
+        return a.length;
+    }
+}

extensions/sample-extension/src/com/metaweb/gridworks/sampleExtension/SampleUtil.java

@@ -1,8 +0,0 @@
-package com.metaweb.gridworks.sampleExtension;
-
-
-public class SampleUtil {
-    static public int stringArrayLength(String[] a) {
-        return a.length;
-    }
-}

main/src/com/google/gridworks/ClientSideResourceManager.java

@@ -0,0 +1,91 @@
+package com.google.gridworks;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import edu.mit.simile.butterfly.ButterflyModule;
+import edu.mit.simile.butterfly.MountPoint;
+
+
+public class ClientSideResourceManager {
+    final static Logger logger = LoggerFactory.getLogger("gridworks_clientSideResourceManager");
+    
+    static public class QualifiedPath {
+        public ButterflyModule  module;
+        public String           path;
+        public String           fullPath;
+    }
+    static public class ClientSideResourceBundle {
+        final protected Set<String>     _pathSet = new HashSet<String>();
+        final protected List<QualifiedPath>      _pathList = new ArrayList<QualifiedPath>();
+    }
+    
+    final static protected Map<String, ClientSideResourceBundle> s_bundles
+        = new HashMap<String, ClientSideResourceBundle>();
+    
+    static public void addPaths(
+        String bundleName, 
+        ButterflyModule module, 
+        String[] paths) {
+        
+        ClientSideResourceBundle bundle = s_bundles.get(bundleName);
+        if (bundle == null) {
+            bundle = new ClientSideResourceBundle();
+            s_bundles.put(bundleName, bundle);
+        }
+        
+        for (String path : paths) {
+            String fullPath = resolve(module, path);
+            if (fullPath == null) {
+                logger.error("Failed to add paths to unmounted module " + module.getName());
+                break;
+            }
+            if (!bundle._pathSet.contains(fullPath)) {
+                QualifiedPath qualifiedPath = new QualifiedPath();
+                qualifiedPath.module = module;
+                qualifiedPath.path = path;
+                qualifiedPath.fullPath = fullPath;
+                
+                bundle._pathSet.add(fullPath);
+                bundle._pathList.add(qualifiedPath);
+            }
+        }
+    }
+    
+    static public QualifiedPath[] getPaths(String bundleName) {
+        ClientSideResourceBundle bundle = s_bundles.get(bundleName);
+        if (bundle == null) {
+            return new QualifiedPath[] {};
+        } else {
+            QualifiedPath[] paths = new QualifiedPath[bundle._pathList.size()];
+            bundle._pathList.toArray(paths);
+            return paths;
+        }
+    }
+    
+    static protected String resolve(ButterflyModule module, String path) {
+        MountPoint mountPoint = module.getMountPoint();
+        if (mountPoint != null) {
+            String mountPointPath = mountPoint.getMountPoint();
+            if (mountPointPath != null) {
+                StringBuffer sb = new StringBuffer();
+                
+                boolean slashed = path.startsWith("/");
+                char[] mountPointChars = mountPointPath.toCharArray();
+                
+                sb.append(mountPointChars, 0, slashed ? mountPointChars.length - 1 : mountPointChars.length);
+                sb.append(path);
+                
+                return sb.toString();
+            }
+        }
+        return null;
+    }
+}

main/src/com/google/gridworks/GridworksServlet.java

@@ -0,0 +1,346 @@
+package com.google.gridworks;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gridworks.commands.Command;
+import com.google.gridworks.io.FileProjectManager;
+
+import edu.mit.simile.butterfly.Butterfly;
+
+public class GridworksServlet extends Butterfly {
+
+    static private final String VERSION = "1.0";
+
+    private static final long serialVersionUID = 2386057901503517403L;
+
+    private static final String JAVAX_SERVLET_CONTEXT_TEMPDIR = "javax.servlet.context.tempdir";
+
+    static private GridworksServlet s_singleton;
+    static private File             s_dataDir;
+    
+    static final private Map<String, Command> commands = new HashMap<String, Command>();
+
+    // timer for periodically saving projects
+    static private Timer _timer;
+
+    final static Logger logger = LoggerFactory.getLogger("gridworks");
+
+    // TODO: This belongs in an external config file somewhere
+    private static final String[][] commandNames = {
+        {"create-project-from-upload", "com.google.gridworks.commands.project.CreateProjectCommand"},
+        {"import-project", "com.google.gridworks.commands.project.ImportProjectCommand"},
+        {"export-project", "com.google.gridworks.commands.project.ExportProjectCommand"},
+        {"export-rows", "com.google.gridworks.commands.project.ExportRowsCommand"},
+
+        {"get-project-metadata", "com.google.gridworks.commands.project.GetProjectMetadataCommand"},
+        {"get-all-project-metadata", "com.google.gridworks.commands.workspace.GetAllProjectMetadataCommand"},
+
+        {"delete-project", "com.google.gridworks.commands.project.DeleteProjectCommand"},
+        {"rename-project", "com.google.gridworks.commands.project.RenameProjectCommand"},
+
+        {"get-models", "com.google.gridworks.commands.project.GetModelsCommand"},
+        {"get-rows", "com.google.gridworks.commands.row.GetRowsCommand"},
+        {"get-processes", "com.google.gridworks.commands.history.GetProcessesCommand"},
+        {"get-history", "com.google.gridworks.commands.history.GetHistoryCommand"},
+        {"get-operations", "com.google.gridworks.commands.history.GetOperationsCommand"},
+        {"get-columns-info", "com.google.gridworks.commands.column.GetColumnsInfoCommand"},
+        {"get-scatterplot", "com.google.gridworks.commands.browsing.GetScatterplotCommand"},
+
+        {"undo-redo", "com.google.gridworks.commands.history.UndoRedoCommand"},
+        {"apply-operations", "com.google.gridworks.commands.history.ApplyOperationsCommand"},
+        {"cancel-processes", "com.google.gridworks.commands.history.CancelProcessesCommand"},
+
+        {"compute-facets", "com.google.gridworks.commands.browsing.ComputeFacetsCommand"},
+        {"compute-clusters", "com.google.gridworks.commands.browsing.ComputeClustersCommand"},
+
+        {"edit-one-cell", "com.google.gridworks.commands.cell.EditOneCellCommand"},
+        {"text-transform", "com.google.gridworks.commands.cell.TextTransformCommand"},
+        {"mass-edit", "com.google.gridworks.commands.cell.MassEditCommand"},
+        {"join-multi-value-cells", "com.google.gridworks.commands.cell.JoinMultiValueCellsCommand"},
+        {"split-multi-value-cells", "com.google.gridworks.commands.cell.SplitMultiValueCellsCommand"},
+        {"transpose-columns-into-rows", "com.google.gridworks.commands.cell.TransposeColumnsIntoRowsCommand"},
+
+        {"add-column", "com.google.gridworks.commands.column.AddColumnCommand"},
+        {"remove-column", "com.google.gridworks.commands.column.RemoveColumnCommand"},
+        {"rename-column", "com.google.gridworks.commands.column.RenameColumnCommand"},
+        {"split-column", "com.google.gridworks.commands.column.SplitColumnCommand"},
+        {"extend-data", "com.google.gridworks.commands.column.ExtendDataCommand"},
+
+        {"denormalize", "com.google.gridworks.commands.row.DenormalizeCommand"},
+
+        {"reconcile", "com.google.gridworks.commands.recon.ReconcileCommand"},
+        {"recon-match-best-candidates", "com.google.gridworks.commands.recon.ReconMatchBestCandidatesCommand"},
+        {"recon-mark-new-topics", "com.google.gridworks.commands.recon.ReconMarkNewTopicsCommand"},
+        {"recon-discard-judgments", "com.google.gridworks.commands.recon.ReconDiscardJudgmentsCommand"},
+        {"recon-match-specific-topic-to-cells", "com.google.gridworks.commands.recon.ReconMatchSpecificTopicCommand"},
+        {"recon-judge-one-cell", "com.google.gridworks.commands.recon.ReconJudgeOneCellCommand"},
+        {"recon-judge-similar-cells", "com.google.gridworks.commands.recon.ReconJudgeSimilarCellsCommand"},
+
+        {"annotate-one-row", "com.google.gridworks.commands.row.AnnotateOneRowCommand"},
+        {"annotate-rows", "com.google.gridworks.commands.row.AnnotateRowsCommand"},
+        {"remove-rows", "com.google.gridworks.commands.row.RemoveRowsCommand"},
+        {"reorder-rows", "com.google.gridworks.commands.row.ReorderRowsCommand"},
+
+        {"save-protograph", "com.google.gridworks.commands.freebase.SaveProtographCommand"},
+
+        {"get-expression-language-info", "com.google.gridworks.commands.expr.GetExpressionLanguageInfoCommand"},
+        {"get-expression-history", "com.google.gridworks.commands.expr.GetExpressionHistoryCommand"},
+        {"log-expression", "com.google.gridworks.commands.expr.LogExpressionCommand"},
+
+        {"preview-expression", "com.google.gridworks.commands.expr.PreviewExpressionCommand"},
+        {"preview-extend-data", "com.google.gridworks.commands.column.PreviewExtendDataCommand"},
+        {"preview-protograph", "com.google.gridworks.commands.freebase.PreviewProtographCommand"},
+
+        {"guess-types-of-column", "com.google.gridworks.commands.freebase.GuessTypesOfColumnCommand"},
+
+        {"check-authorization", "com.google.gridworks.commands.auth.CheckAuthorizationCommand"},
+        {"authorize", "com.google.gridworks.commands.auth.AuthorizeCommand"},
+        {"deauthorize", "com.google.gridworks.commands.auth.DeAuthorizeCommand"},
+        {"user-badges", "com.google.gridworks.commands.auth.GetUserBadgesCommand"},
+
+        {"upload-data", "com.google.gridworks.commands.freebase.UploadDataCommand"},
+        {"mqlread", "com.google.gridworks.commands.freebase.MQLReadCommand"},
+        {"mqlwrite", "com.google.gridworks.commands.freebase.MQLWriteCommand"},
+        
+        {"get-preference", "com.google.gridworks.commands.GetPreferenceCommand"},
+        {"get-all-preferences", "com.google.gridworks.commands.GetAllPreferencesCommand"},
+        {"set-preference", "com.google.gridworks.commands.SetPreferenceCommand"},
+    };
+
+    public static String getVersion() {
+        return VERSION;
+    }
+
+    final static protected long s_autoSavePeriod = 1000 * 60 * 5; // 5 minutes
+
+    static protected class AutoSaveTimerTask extends TimerTask {
+        public void run() {
+            try {
+                ProjectManager.singleton.save(false); // quick, potentially incomplete save
+            } finally {
+                _timer.schedule(new AutoSaveTimerTask(), s_autoSavePeriod);
+                // we don't use scheduleAtFixedRate because that might result in
+                // bunched up events when the computer is put in sleep mode
+            }
+        }
+    }
+
+    protected ServletConfig config;
+
+    @Override
+    public void init() throws ServletException {
+        super.init();
+        
+        s_singleton = this;
+
+        logger.trace("> initialize");
+
+        String data = getInitParameter("gridworks.data");
+
+        if (data == null) {
+            throw new ServletException("can't find servlet init config 'gridworks.data', I have to give up initializing");
+        }
+
+        registerCommands(commandNames);
+
+        s_dataDir = new File(data);
+        FileProjectManager.initialize(s_dataDir);
+
+        if (_timer == null) {
+            _timer = new Timer("autosave");
+            _timer.schedule(new AutoSaveTimerTask(), s_autoSavePeriod);
+        }
+
+        logger.trace("< initialize");
+    }
+
+    @Override
+    public void destroy() {
+        logger.trace("> destroy");
+
+        // cancel automatic periodic saving and force a complete save.
+        if (_timer != null) {
+            _timer.cancel();
+            _timer = null;
+        }
+        if (ProjectManager.singleton != null) {
+            ProjectManager.singleton.save(true); // complete save
+            ProjectManager.singleton = null;
+        }
+
+        this.config = null;
+
+        logger.trace("< destroy");
+
+        super.destroy();
+    }
+
+    @Override
+    public void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+        if (request.getPathInfo().startsWith("/command")) {
+            String commandName = getCommandName(request);
+            Command command = commands.get(commandName);
+            if (command != null) {
+                if (request.getMethod().equals("GET")) {
+                    logger.trace("> GET {}", commandName);
+                    command.doGet(request, response);
+                    logger.trace("< GET {}", commandName);
+                } else if (request.getMethod().equals("POST")) {
+                    logger.trace("> POST {}", commandName);
+                    command.doPost(request, response);
+                    logger.trace("< POST {}", commandName);
+                } else {
+                    response.sendError(405);
+                }
+            } else {
+                response.sendError(404);
+            }
+        } else {
+            super.service(request, response);
+        }
+    }
+
+    protected String getCommandName(HttpServletRequest request) {
+        // Remove extraneous path segments that might be there for other purposes,
+        // e.g., for /export-rows/filename.ext, export-rows is the command while
+        // filename.ext is only for the browser to prompt a convenient filename.
+        String commandName = request.getPathInfo().substring("/command/".length());
+        int slash = commandName.indexOf('/');
+        return slash > 0 ? commandName.substring(0, slash) : commandName;
+    }
+
+    private File tempDir = null;
+
+    public File getTempDir() {
+        if (tempDir == null) {
+            File tempDir = (File) this.config.getServletContext().getAttribute(JAVAX_SERVLET_CONTEXT_TEMPDIR);
+            if (tempDir == null) {
+                throw new RuntimeException("This app server doesn't support temp directories");
+            }
+        }
+        return tempDir;
+    }
+
+    public File getTempFile(String name) {
+        return new File(getTempDir(), name);
+    }
+    
+    public File getCacheDir(String name) {
+        File dir = new File(new File(s_dataDir, "cache"), name);
+        dir.mkdirs();
+        
+        return dir;
+    }
+
+    public String getConfiguration(String name, String def) {
+        return null;
+    }
+
+    /**
+     * Register an array of commands
+     *
+     * @param commands
+     *            An array of arrays containing pairs of strings with the
+     *            command name in the first element of the tuple and the fully
+     *            qualified class name of the class implementing the command in
+     *            the second.
+     * @return false if any commands failed to load
+     */
+    private boolean registerCommands(String[][] commands) {
+        boolean status = true;
+        for (String[] command : commandNames) {
+            String commandName = command[0];
+            String className = command[1];
+            status |= registerOneCommand(commandName, className);
+        }
+        return status;
+    }
+    
+    /**
+     * Register a single command given its class name.
+     *
+     * @param name
+     *            command verb for command
+     * @param className
+     *            class name of command class
+     * @return true if command was loaded and registered successfully
+     */
+    protected boolean registerOneCommand(String commandName, String className) {
+        logger.debug("Loading command " + commandName + " class: " + className);
+        Command cmd;
+        try {
+            cmd = (Command) this.getClass().getClassLoader().loadClass(className).newInstance();
+            
+            return registerOneCommand(commandName, cmd);
+        } catch (InstantiationException e) {
+            logger.error("Failed to load command class " + className, e);
+            return false;
+        } catch (IllegalAccessException e) {
+            logger.error("Failed to load command class " + className, e);
+            return false;
+        } catch (ClassNotFoundException e) {
+            logger.error("Failed to load command class " + className, e);
+            return false;
+        }
+    }
+    
+    /**
+     * Register a single command.
+     *
+     * @param name
+     *            command verb for command
+     * @param commandObject
+     *            object implementing the command
+     * @return true if command was loaded and registered successfully
+     */
+    protected boolean registerOneCommand(String name, Command commandObject) {
+        if (commands.containsKey(name)) {
+            return false;
+        }
+        
+        commandObject.init(this);
+        commands.put(name, commandObject);
+        
+        return true;
+    }
+
+    // Currently only for test purposes
+    protected boolean unregisterCommand(String verb) {
+        return commands.remove(verb) != null;
+    }
+    
+    /**
+     * Register a single command. Used by extensions.
+     *
+     * @param name
+     *            command verb for command
+     * @param commandObject
+     *            object implementing the command
+     *            
+     * @return true if command was loaded and registered successfully
+     */
+    static public boolean registerCommand(String commandName, Command commandObject) {
+        return s_singleton.registerOneCommand(commandName, commandObject);
+    }
+    
+    static public Class<?> getClass(String className) throws ClassNotFoundException {
+        if (className.startsWith("com.metaweb.")) {
+            className = "com.google." + className.substring("com.metaweb.".length());
+        }
+        return Class.forName(className);
+    }
+}
+

main/src/com/google/gridworks/InterProjectModel.java

@@ -0,0 +1,128 @@
+package com.google.gridworks;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.google.gridworks.expr.ExpressionUtils;
+import com.google.gridworks.expr.HasFieldsListImpl;
+import com.google.gridworks.expr.WrappedRow;
+import com.google.gridworks.model.Column;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+
+public class InterProjectModel {
+    static public class ProjectJoin {
+        final public long   fromProjectID;
+        final public String fromProjectColumnName;
+        final public long   toProjectID;
+        final public String toProjectColumnName;
+        
+        final public Map<Object, List<Integer>> valueToRowIndices = 
+            new HashMap<Object, List<Integer>>();
+        
+        ProjectJoin(
+            long   fromProjectID,
+            String fromProjectColumnName,
+            long   toProjectID,
+            String toProjectColumnName
+        ) {
+            this.fromProjectID = fromProjectID;
+            this.fromProjectColumnName = fromProjectColumnName;
+            this.toProjectID = toProjectID;
+            this.toProjectColumnName = toProjectColumnName;
+        }
+        
+        public HasFieldsListImpl getRows(Object value) {
+            if (ExpressionUtils.isNonBlankData(value) && valueToRowIndices.containsKey(value)) {
+                Project toProject = ProjectManager.singleton.getProject(toProjectID);
+                if (toProject != null) {
+                    HasFieldsListImpl rows = new HasFieldsListImpl();
+                    for (Integer r : valueToRowIndices.get(value)) {
+                        Row row = toProject.rows.get(r);
+                        rows.add(new WrappedRow(toProject, r, row));
+                    }
+                    
+                    return rows;
+                }
+            }
+            return null;
+        }
+    }
+    
+    protected Map<String, ProjectJoin> _joins = new HashMap<String, ProjectJoin>();
+    
+    public ProjectJoin getJoin(String fromProject, String fromColumn, String toProject, String toColumn) {
+        String key = fromProject + ";" + fromColumn + ";" + toProject + ";" + toColumn;
+        if (!_joins.containsKey(key)) {
+            ProjectJoin join = new ProjectJoin(
+                ProjectManager.singleton.getProjectID(fromProject), 
+                fromColumn, 
+                ProjectManager.singleton.getProjectID(toProject), 
+                toColumn
+            );
+            
+            computeJoin(join);
+            
+            _joins.put(key, join);
+        }
+        
+        return _joins.get(key);
+    }
+    
+    public void flushJoinsInvolvingProject(long projectID) {
+        for (Entry<String, ProjectJoin> entry : _joins.entrySet()) {
+            ProjectJoin join = entry.getValue();
+            if (join.fromProjectID == projectID || join.toProjectID == projectID) {
+                _joins.remove(entry.getKey());
+            }
+        }
+    }
+
+    public void flushJoinsInvolvingProjectColumn(long projectID, String columnName) {
+        for (Entry<String, ProjectJoin> entry : _joins.entrySet()) {
+            ProjectJoin join = entry.getValue();
+            if (join.fromProjectID == projectID && join.fromProjectColumnName.equals(columnName) || 
+                join.toProjectID == projectID && join.toProjectColumnName.equals(columnName)) {
+                _joins.remove(entry.getKey());
+            }
+        }
+    }
+
+    protected void computeJoin(ProjectJoin join) {
+        if (join.fromProjectID < 0 || join.toProjectID < 0) {
+            return;
+        }
+        
+        Project fromProject = ProjectManager.singleton.getProject(join.fromProjectID);
+        Project toProject = ProjectManager.singleton.getProject(join.toProjectID);
+        if (fromProject == null || toProject == null) {
+            return;
+        }
+        
+        Column fromColumn = fromProject.columnModel.getColumnByName(join.fromProjectColumnName);
+        Column toColumn = toProject.columnModel.getColumnByName(join.toProjectColumnName);
+        if (fromColumn == null || toColumn == null) {
+            return;
+        }
+        
+        for (Row fromRow : fromProject.rows) {
+            Object value = fromRow.getCellValue(fromColumn.getCellIndex());
+            if (ExpressionUtils.isNonBlankData(value) && !join.valueToRowIndices.containsKey(value)) {
+                join.valueToRowIndices.put(value, new ArrayList<Integer>());
+            }
+        }
+        
+        int count = toProject.rows.size();
+        for (int r = 0; r < count; r++) {
+            Row toRow = toProject.rows.get(r);
+            
+            Object value = toRow.getCellValue(toColumn.getCellIndex());
+            if (ExpressionUtils.isNonBlankData(value) && join.valueToRowIndices.containsKey(value)) {
+                join.valueToRowIndices.get(value).add(r);
+            }
+        }
+    }
+}

main/src/com/google/gridworks/Jsonizable.java

@@ -0,0 +1,16 @@
+package com.google.gridworks;
+
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+/**
+ * Interface for streaming out JSON, either into HTTP responses or 
+ * serialization files.
+ * 
+ * @author dfhuynh
+ */
+public interface Jsonizable {
+    public void write(JSONWriter writer, Properties options) throws JSONException;
+}

main/src/com/google/gridworks/ProjectManager.java

@@ -0,0 +1,421 @@
+package com.google.gridworks;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.tools.tar.TarOutputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gridworks.history.HistoryEntryManager;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.preference.PreferenceStore;
+import com.google.gridworks.preference.TopList;
+
+/**
+ * ProjectManager is responsible for loading and saving the workspace and projects.
+ *
+ *
+ */
+public abstract class ProjectManager {
+    // last n expressions used across all projects
+    static protected final int s_expressionHistoryMax = 100;
+
+    protected Map<Long, ProjectMetadata> _projectsMetadata;
+    protected PreferenceStore            _preferenceStore;
+
+    final static Logger logger = LoggerFactory.getLogger("project_manager");
+
+    /**
+     *  What caches the joins between projects.
+     */
+    transient protected InterProjectModel _interProjectModel = new InterProjectModel();
+
+    /**
+     *  Flags
+     */
+    transient protected int _busy = 0; // heavy operations like creating or importing projects are going on
+
+    /**
+     *  While each project's metadata is loaded completely at start-up, each project's raw data
+     *  is loaded only when the project is accessed by the user. This is because project
+     *  metadata is tiny compared to raw project data. This hash map from project ID to project
+     *  is more like a last accessed-last out cache.
+     */
+    transient protected Map<Long, Project> _projects;
+
+    static public ProjectManager singleton;
+
+    protected ProjectManager(){
+        _projectsMetadata = new HashMap<Long, ProjectMetadata>();
+        _preferenceStore = new PreferenceStore();
+        _projects = new HashMap<Long, Project>();
+
+        preparePreferenceStore(_preferenceStore);
+    }
+
+    /**
+     * Registers the project in the memory of the current session
+     * @param project
+     * @param projectMetadata
+     */
+    public void registerProject(Project project, ProjectMetadata projectMetadata) {
+        synchronized (this) {
+            _projects.put(project.id, project);
+            _projectsMetadata.put(project.id, projectMetadata);
+        }
+    }
+ //----------Load from data store to memory----------------
+
+    /**
+     * Load project metadata from data storage
+     * @param projectID
+     * @return
+     */
+    public abstract boolean loadProjectMetadata(long projectID);
+
+    /**
+     * Loads a project from the data store into memory
+     * @param id
+     * @return
+     */
+    protected abstract Project loadProject(long id);
+
+    //------------Import and Export from Gridworks archive-----------------
+    /**
+     * Import project from a Gridworks archive
+     * @param projectID
+     * @param inputStream
+     * @param gziped
+     * @throws IOException
+     */
+    public abstract void importProject(long projectID, InputStream inputStream, boolean gziped) throws IOException;
+
+    /**
+     * Export project to a Gridworks archive
+     * @param projectId
+     * @param tos
+     * @throws IOException
+     */
+    public abstract void exportProject(long projectId, TarOutputStream tos) throws IOException;
+
+
+ //------------Save to record store------------
+    /**
+     * Saves a project and its metadata to the data store
+     * @param id
+     */
+    public void ensureProjectSaved(long id) {
+        synchronized(this){
+            ProjectMetadata metadata = this.getProjectMetadata(id);
+            if (metadata != null) {
+                try {
+                    saveMetadata(metadata, id);
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }//FIXME what should be the behaviour if metadata is null? i.e. not found
+
+            Project project = getProject(id);
+            if (project != null && metadata != null && metadata.getModified().after(project.getLastSave())) {
+                try {
+                    saveProject(project);
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }//FIXME what should be the behaviour if project is null? i.e. not found or loaded.
+            //FIXME what should happen if the metadata is found, but not the project? or vice versa?
+        }
+
+    }
+
+    /**
+     * Save project metadata to the data store
+     * @param metadata
+     * @param projectId
+     * @throws Exception
+     */
+    protected abstract void saveMetadata(ProjectMetadata metadata, long projectId) throws Exception;
+
+    /**
+     * Save project to the data store
+     * @param project
+     */
+    protected abstract void saveProject(Project project);
+
+    /**
+     * Save workspace and all projects to data store
+     * @param allModified
+     */
+    public void save(boolean allModified) {
+        if (allModified || _busy == 0) {
+            saveProjects(allModified);
+            saveWorkspace();
+        }
+    }
+
+    /**
+     * Saves the workspace to the data store
+     */
+    protected abstract void saveWorkspace();
+
+    /**
+     * A utility class to prioritize projects for saving, depending on how long ago
+     * they have been changed but have not been saved.
+     */
+    static protected class SaveRecord {
+        final Project project;
+        final long overdue;
+
+        SaveRecord(Project project, long overdue) {
+            this.project = project;
+            this.overdue = overdue;
+        }
+    }
+
+    static protected final int s_projectFlushDelay = 1000 * 60 * 60; // 1 hour
+    static protected final int s_quickSaveTimeout = 1000 * 30; // 30 secs
+
+    /**
+     * Saves all projects to the data store
+     * @param allModified
+     */
+    protected void saveProjects(boolean allModified) {
+        List<SaveRecord> records = new ArrayList<SaveRecord>();
+        Date startTimeOfSave = new Date();
+
+        synchronized (this) {
+            for (long id : _projectsMetadata.keySet()) {
+                ProjectMetadata metadata = getProjectMetadata(id);
+                Project project = _projects.get(id); // don't call getProject() as that will load the project.
+
+                if (project != null) {
+                    boolean hasUnsavedChanges =
+                        metadata.getModified().getTime() > project.getLastSave().getTime();
+
+                    if (hasUnsavedChanges) {
+                        long msecsOverdue = startTimeOfSave.getTime() - project.getLastSave().getTime();
+
+                        records.add(new SaveRecord(project, msecsOverdue));
+
+                    } else if (startTimeOfSave.getTime() - project.getLastSave().getTime() > s_projectFlushDelay) {
+                        /*
+                         *  It's been a while since the project was last saved and it hasn't been
+                         *  modified. We can safely remove it from the cache to save some memory.
+                         */
+                        _projects.remove(id);
+                    }
+                }
+            }
+        }
+
+        if (records.size() > 0) {
+            Collections.sort(records, new Comparator<SaveRecord>() {
+                public int compare(SaveRecord o1, SaveRecord o2) {
+                    if (o1.overdue < o2.overdue) {
+                        return 1;
+                    } else if (o1.overdue > o2.overdue) {
+                        return -1;
+                    } else {
+                        return 0;
+                    }
+                }
+            });
+
+            logger.info(allModified ?
+                "Saving all modified projects ..." :
+                "Saving some modified projects ..."
+            );
+
+            for (int i = 0;
+                 i < records.size() &&
+                    (allModified || (new Date().getTime() - startTimeOfSave.getTime() < s_quickSaveTimeout));
+                 i++) {
+
+                try {
+                    saveProject(records.get(i).project);
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+
+    //--------------Get from memory--------------
+    /**
+     * Gets the InterProjectModel from memory
+     */
+    public InterProjectModel getInterProjectModel() {
+        return _interProjectModel;
+    }
+
+
+    /**
+     * Gets the project metadata from memory
+     * Requires that the metadata has already been loaded from the data store
+     * @param id
+     * @return
+     */
+    public ProjectMetadata getProjectMetadata(long id) {
+        return _projectsMetadata.get(id);
+    }
+
+    /**
+     * Gets the project metadata from memory
+     * Requires that the metadata has already been loaded from the data store
+     * @param name
+     * @return
+     */
+    public ProjectMetadata getProjectMetadata(String name) {
+        for (ProjectMetadata pm : _projectsMetadata.values()) {
+            if (pm.getName().equals(name)) {
+                return pm;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Tries to find the project id when given a project name
+     * Requires that all project metadata exists has been loaded to memory from the data store
+     * @param name
+     *     The name of the project
+     * @return
+     *     The id of the project, or -1 if it cannot be found
+     */
+    public long getProjectID(String name) {
+        for (Entry<Long, ProjectMetadata> entry : _projectsMetadata.entrySet()) {
+            if (entry.getValue().getName().equals(name)) {
+                return entry.getKey();
+            }
+        }
+        return -1;
+    }
+
+
+    /**
+     * Gets all the project Metadata currently held in memory
+     * @return
+     */
+    public Map<Long, ProjectMetadata> getAllProjectMetadata() {
+        return _projectsMetadata;
+    }
+
+    /**
+     * Gets the required project from the data store
+     * If project does not already exist in memory, it is loaded from the data store
+     * @param id
+     *     the id of the project
+     * @return
+     *     the project with the matching id, or null if it can't be found
+     */
+    public Project getProject(long id) {
+        synchronized (this) {
+            if (_projects.containsKey(id)) {
+                return _projects.get(id);
+            } else {
+                Project project = loadProject(id);
+
+                _projects.put(id, project);
+
+                return project;
+            }
+        }
+    }
+
+    /**
+     * Gets the preference store
+     * @return
+     */
+    public PreferenceStore getPreferenceStore() {
+        return _preferenceStore;
+    }
+
+    /**
+     * Gets all expressions from the preference store
+     * @return
+     */
+    public List<String> getExpressions() {
+        return ((TopList) _preferenceStore.get("expressions")).getList();
+    }
+
+    /**
+     * The history entry manager deals with changes
+     * @return manager for handling history
+     */
+    public abstract HistoryEntryManager getHistoryEntryManager();
+
+    //-------------remove project-----------
+
+    /**
+     * Remove the project from the data store
+     * @param project
+     */
+    public void deleteProject(Project project) {
+        deleteProject(project.id);
+    }
+
+    /**
+     * Remove project from data store
+     * @param projectID
+     */
+    public abstract void deleteProject(long projectID);
+
+    /**
+     * Removes project from memory
+     * @param projectID
+     */
+    protected void removeProject(long projectID){
+        if (_projectsMetadata.containsKey(projectID)) {
+            _projectsMetadata.remove(projectID);
+        }
+        if (_projects.containsKey(projectID)) {
+            _projects.remove(projectID);
+        }
+    }
+
+    //--------------Miscellaneous-----------
+
+    /**
+     * Sets the flag for long running operations
+     * @param busy
+     */
+    public void setBusy(boolean busy) {
+        synchronized (this) {
+            if (busy) {
+                _busy++;
+            } else {
+                _busy--;
+            }
+        }
+    }
+
+
+
+    /**
+     * Add the latest expression to the preference store
+     * @param s
+     */
+    public void addLatestExpression(String s) {
+        synchronized (this) {
+            ((TopList) _preferenceStore.get("expressions")).add(s);
+        }
+    }
+
+
+    /**
+    *
+    * @param ps
+    */
+   static protected void preparePreferenceStore(PreferenceStore ps) {
+       ps.put("scripting.expressions", new TopList(s_expressionHistoryMax));
+   }
+}

main/src/com/google/gridworks/ProjectMetadata.java

@@ -0,0 +1,156 @@
+package com.google.gridworks;
+
+import java.util.Date;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gridworks.preference.PreferenceStore;
+import com.google.gridworks.preference.TopList;
+import com.google.gridworks.util.JSONUtilities;
+import com.google.gridworks.util.ParsingUtilities;
+
+public class ProjectMetadata implements Jsonizable {
+    private final Date     _created;
+    private Date           _modified;
+    private String         _name;
+    private String         _password;
+
+    private String         _encoding;
+    private int            _encodingConfidence;
+    
+    private PreferenceStore _preferenceStore = new PreferenceStore();
+
+    final Logger logger = LoggerFactory.getLogger("project_metadata");
+
+    protected ProjectMetadata(Date date) {
+        _created = date;
+        preparePreferenceStore(_preferenceStore);
+    }
+
+    public ProjectMetadata() {
+        _created = new Date();
+        _modified = _created;
+        preparePreferenceStore(_preferenceStore);
+    }
+
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+
+        writer.object();
+        writer.key("name"); writer.value(_name);
+        writer.key("created"); writer.value(ParsingUtilities.dateToString(_created));
+        writer.key("modified"); writer.value(ParsingUtilities.dateToString(_modified));
+
+        if ("save".equals(options.getProperty("mode"))) {
+            writer.key("password"); writer.value(_password);
+
+            writer.key("encoding"); writer.value(_encoding);
+            writer.key("encodingConfidence"); writer.value(_encodingConfidence);
+            writer.key("preferences"); _preferenceStore.write(writer, options);
+        }
+        writer.endObject();
+    }
+
+
+
+    public void write(JSONWriter jsonWriter) throws Exception {
+        Properties options = new Properties();
+        options.setProperty("mode", "save");
+
+        write(jsonWriter, options);
+    }
+    
+    static public ProjectMetadata loadFromJSON(JSONObject obj) {
+        ProjectMetadata pm = new ProjectMetadata(JSONUtilities.getDate(obj, "modified", new Date()));
+
+        pm._modified = JSONUtilities.getDate(obj, "modified", new Date());
+        pm._name = JSONUtilities.getString(obj, "name", "<Error recovering project name>");
+        pm._password = JSONUtilities.getString(obj, "password", "");
+
+        pm._encoding = JSONUtilities.getString(obj, "encoding", "");
+        pm._encodingConfidence = JSONUtilities.getInt(obj, "encodingConfidence", 0);
+
+        if (obj.has("preferences") && !obj.isNull("preferences")) {
+            try {
+                pm._preferenceStore.load(obj.getJSONObject("preferences"));
+            } catch (JSONException e) {
+                // ignore
+            }
+        }
+        
+        if (obj.has("expressions") && !obj.isNull("expressions")) {
+            try {
+                ((TopList) pm._preferenceStore.get("expressions"))
+                    .load(obj.getJSONArray("expressions"));
+            } catch (JSONException e) {
+                // ignore
+            }
+        }
+        
+        return pm;
+    }
+    
+    static protected void preparePreferenceStore(PreferenceStore ps) {
+        ProjectManager.preparePreferenceStore(ps);
+        // Any project specific preferences?
+    }
+
+    public Date getCreated() {
+        return _created;
+    }
+
+    public void setName(String name) {
+        this._name = name;
+    }
+
+    public String getName() {
+        return _name;
+    }
+
+    public void setEncoding(String encoding) {
+        this._encoding = encoding;
+    }
+
+    public String getEncoding() {
+        return _encoding;
+    }
+
+    public void setEncodingConfidence(int confidence) {
+        this._encodingConfidence = confidence;
+    }
+
+    public void setEncodingConfidence(String confidence) {
+        if (confidence != null) {
+            this.setEncodingConfidence(Integer.parseInt(confidence));
+        }
+    }
+
+    public int getEncodingConfidence() {
+        return _encodingConfidence;
+    }
+
+    public void setPassword(String password) {
+        this._password = password;
+    }
+
+    public String getPassword() {
+        return _password;
+    }
+
+    public Date getModified() {
+        return _modified;
+    }
+
+    public void updateModified() {
+        _modified = new Date();
+    }
+
+    public PreferenceStore getPreferenceStore() {
+        return _preferenceStore;
+    }
+}

main/src/com/google/gridworks/browsing/DecoratedValue.java

@@ -0,0 +1,33 @@
+package com.google.gridworks.browsing;
+
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.Jsonizable;
+
+/**
+ * Store a value and its text label, in case the value is not a string itself.
+ * For instance, if a value is a date, then its label can be one particular
+ * rendering of that date.
+ * 
+ * Facet choices that are presented to the user as text are stored as decorated values.
+ */
+public class DecoratedValue implements Jsonizable {
+    final public Object value;
+    final public String label;
+    
+    public DecoratedValue(Object value, String label) {
+        this.value = value;
+        this.label = label;
+    }
+    
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        writer.object();
+        writer.key("v"); writer.value(value);
+        writer.key("l"); writer.value(label);
+        writer.endObject();
+    }
+}

main/src/com/google/gridworks/browsing/Engine.java

@@ -0,0 +1,211 @@
+package com.google.gridworks.browsing;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.gridworks.Jsonizable;
+import com.google.gridworks.browsing.facets.Facet;
+import com.google.gridworks.browsing.facets.ListFacet;
+import com.google.gridworks.browsing.facets.RangeFacet;
+import com.google.gridworks.browsing.facets.ScatterplotFacet;
+import com.google.gridworks.browsing.facets.TextSearchFacet;
+import com.google.gridworks.browsing.util.ConjunctiveFilteredRecords;
+import com.google.gridworks.browsing.util.ConjunctiveFilteredRows;
+import com.google.gridworks.browsing.util.FilteredRecordsAsFilteredRows;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.Row;
+
+/**
+ * Faceted browsing engine.
+ */
+public class Engine implements Jsonizable {
+	static public enum Mode {
+		RowBased,
+		RecordBased
+	}
+	
+    public final static String INCLUDE_DEPENDENT = "includeDependent";
+    public final static String MODE = "mode";
+    public final static String MODE_ROW_BASED = "row-based";
+    public final static String MODE_RECORD_BASED = "record-based";
+    
+    protected Project         _project;
+    protected List<Facet>     _facets = new LinkedList<Facet>();
+    protected Mode			  _mode = Mode.RowBased;
+    
+    static public String modeToString(Mode mode) {
+    	return mode == Mode.RowBased ? MODE_ROW_BASED : MODE_RECORD_BASED;
+    }
+    static public Mode stringToMode(String s) {
+    	return MODE_ROW_BASED.equals(s) ? Mode.RowBased : Mode.RecordBased;
+    }
+    
+    public Engine(Project project) {
+        _project  = project;
+    }
+    
+    public Mode getMode() {
+    	return _mode;
+    }
+    public void setMode(Mode mode) {
+    	_mode = mode;
+    }
+    
+    public FilteredRows getAllRows() {
+        return new FilteredRows() {
+			@Override
+			public void accept(Project project, RowVisitor visitor) {
+		    	try {
+		    		visitor.start(project);
+		    		
+			        int c = project.rows.size();
+			        for (int rowIndex = 0; rowIndex < c; rowIndex++) {
+			            Row row = project.rows.get(rowIndex);
+			            visitor.visit(project, rowIndex, row);
+			        }
+		    	} finally {
+		    		visitor.end(project);
+		    	}
+			}
+        };
+    }
+    
+    public FilteredRows getAllFilteredRows() {
+        return getFilteredRows(null);
+    }
+
+    public FilteredRows getFilteredRows(Facet except) {
+    	if (_mode == Mode.RecordBased) {
+    		return new FilteredRecordsAsFilteredRows(getFilteredRecords(except));
+    	} else if (_mode == Mode.RowBased) {
+	        ConjunctiveFilteredRows cfr = new ConjunctiveFilteredRows();
+	        for (Facet facet : _facets) {
+	            if (facet != except) {
+	                RowFilter rowFilter = facet.getRowFilter(_project);
+	                if (rowFilter != null) {
+	                    cfr.add(rowFilter);
+	                }
+	            }
+	        }
+	        return cfr;
+    	}
+    	throw new InternalError("Unknown mode.");
+    }
+    
+    public FilteredRecords getAllRecords() {
+        return new FilteredRecords() {
+			@Override
+			public void accept(Project project, RecordVisitor visitor) {
+		    	try {
+		    		visitor.start(project);
+		    		
+			        int c = project.recordModel.getRecordCount();
+			        for (int r = 0; r < c; r++) {
+			            visitor.visit(project, project.recordModel.getRecord(r));
+			        }
+		    	} finally {
+		    		visitor.end(project);
+		    	}
+			}
+        };
+    }
+    
+    public FilteredRecords getFilteredRecords() {
+    	return getFilteredRecords(null);
+    }
+    
+    public FilteredRecords getFilteredRecords(Facet except) {
+    	if (_mode == Mode.RecordBased) {
+    		ConjunctiveFilteredRecords cfr = new ConjunctiveFilteredRecords();
+            for (Facet facet : _facets) {
+                if (facet != except) {
+                    RecordFilter recordFilter = facet.getRecordFilter(_project);
+                    if (recordFilter != null) {
+                        cfr.add(recordFilter);
+                    }
+                }
+            }
+            return cfr;
+    	}
+    	throw new InternalError("This method should not be called when the engine is not in record mode.");
+    }
+    
+    public void initializeFromJSON(JSONObject o) throws Exception {
+        if (o == null) {
+            return;
+        }
+        
+        if (o.has("facets") && !o.isNull("facets")) {
+            JSONArray a = o.getJSONArray("facets");
+            int length = a.length();
+            
+            for (int i = 0; i < length; i++) {
+                JSONObject fo = a.getJSONObject(i);
+                String type = fo.has("type") ? fo.getString("type") : "list";
+                
+                Facet facet = null;
+                if ("list".equals(type)) {
+                    facet = new ListFacet();
+                } else if ("range".equals(type)) {
+                    facet = new RangeFacet();
+                } else if ("scatterplot".equals(type)) {
+                    facet = new ScatterplotFacet();
+                } else if ("text".equals(type)) {
+                    facet = new TextSearchFacet();
+                }
+                
+                if (facet != null) {
+                    facet.initializeFromJSON(_project, fo);
+                    _facets.add(facet);
+                }
+            }
+        }
+        
+        // for backward compatibility
+        if (o.has(INCLUDE_DEPENDENT) && !o.isNull(INCLUDE_DEPENDENT)) {
+            _mode = o.getBoolean(INCLUDE_DEPENDENT) ? Mode.RecordBased : Mode.RowBased;
+        }
+        
+        if (o.has(MODE) && !o.isNull(MODE)) {
+        	_mode = MODE_ROW_BASED.equals(o.getString(MODE)) ? Mode.RowBased : Mode.RecordBased;
+        }
+    }
+        
+    public void computeFacets() throws JSONException {
+    	if (_mode == Mode.RowBased) {
+	        for (Facet facet : _facets) {
+	            FilteredRows filteredRows = getFilteredRows(facet);
+	            
+	            facet.computeChoices(_project, filteredRows);
+	        }
+    	} else if (_mode == Mode.RecordBased) {
+	        for (Facet facet : _facets) {
+	            FilteredRecords filteredRecords = getFilteredRecords(facet);
+	            
+	            facet.computeChoices(_project, filteredRecords);
+	        }
+    	} else {
+        	throw new InternalError("Unknown mode.");
+    	}
+    }
+    
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        
+        writer.object();
+        writer.key("facets");
+            writer.array();
+            for (Facet facet : _facets) {
+                facet.write(writer, options);
+            }
+            writer.endArray();
+        writer.key(MODE); writer.value(_mode == Mode.RowBased ? MODE_ROW_BASED : MODE_RECORD_BASED);
+        writer.endObject();
+    }
+}

main/src/com/google/gridworks/browsing/FilteredRecords.java

@@ -0,0 +1,18 @@
+package com.google.gridworks.browsing;
+
+import com.google.gridworks.model.Project;
+
+/**
+ * Interface for anything that can decide which records match and which don't
+ * based on some particular criteria.
+ */
+public interface FilteredRecords {
+    /**
+     * Go through the records of the given project, determine which match and which don't,
+     * and call visitor.visit() on those that match
+     * 
+     * @param project
+     * @param visitor
+     */
+    public void accept(Project project, RecordVisitor visitor);
+}