diff --git a/broker/appengine/.classpath b/broker/appengine/.classpath new file mode 100644 index 000000000..14955e135 --- /dev/null +++ b/broker/appengine/.classpath @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/broker/appengine/.externalToolBuilders/com.google.appengine.eclipse.core.projectValidator.launch b/broker/appengine/.externalToolBuilders/com.google.appengine.eclipse.core.projectValidator.launch new file mode 100644 index 000000000..3a3b1b194 --- /dev/null +++ b/broker/appengine/.externalToolBuilders/com.google.appengine.eclipse.core.projectValidator.launch @@ -0,0 +1,7 @@ + + + + + + + diff --git a/broker/appengine/.externalToolBuilders/com.google.gdt.eclipse.core.webAppProjectValidator.launch b/broker/appengine/.externalToolBuilders/com.google.gdt.eclipse.core.webAppProjectValidator.launch new file mode 100644 index 000000000..eb4252719 --- /dev/null +++ b/broker/appengine/.externalToolBuilders/com.google.gdt.eclipse.core.webAppProjectValidator.launch @@ -0,0 +1,7 @@ + + + + + + + diff --git a/broker/appengine/.project b/broker/appengine/.project new file mode 100644 index 000000000..31d51910d --- /dev/null +++ b/broker/appengine/.project @@ -0,0 +1,43 @@ + + + gridworks-appengine-broker + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.ui.externaltools.ExternalToolBuilder + full,incremental, + + + LaunchConfigHandle + <project>/.externalToolBuilders/com.google.gdt.eclipse.core.webAppProjectValidator.launch + + + + + com.google.appengine.eclipse.core.enhancerbuilder + + + + + org.eclipse.ui.externaltools.ExternalToolBuilder + full,incremental, + + + LaunchConfigHandle + <project>/.externalToolBuilders/com.google.appengine.eclipse.core.projectValidator.launch + + + + + + org.eclipse.jdt.core.javanature + com.google.appengine.eclipse.core.gaeNature + + diff --git a/broker/appengine/.settings/com.google.appengine.eclipse.core.prefs b/broker/appengine/.settings/com.google.appengine.eclipse.core.prefs new file mode 100644 index 000000000..b727dd4d9 --- /dev/null +++ b/broker/appengine/.settings/com.google.appengine.eclipse.core.prefs @@ -0,0 +1,3 @@ +#Wed May 26 15:13:15 PDT 2010 +eclipse.preferences.version=1 +validationExclusions=src/com/metaweb/gridworks/appengine/*ClientConnection*.java diff --git a/broker/appengine/.settings/com.google.gdt.eclipse.core.prefs b/broker/appengine/.settings/com.google.gdt.eclipse.core.prefs new file mode 100644 index 000000000..a060470b7 --- /dev/null +++ b/broker/appengine/.settings/com.google.gdt.eclipse.core.prefs @@ -0,0 +1,5 @@ +#Wed May 26 15:11:38 PDT 2010 +eclipse.preferences.version=1 +jarsExcludedFromWebInfLib= +warSrcDir= +warSrcDirIsOutput=true diff --git a/broker/appengine/WEB-INF/appengine-web.xml b/broker/appengine/WEB-INF/appengine-web.xml new file mode 100644 index 000000000..818840528 --- /dev/null +++ b/broker/appengine/WEB-INF/appengine-web.xml @@ -0,0 +1,9 @@ + + + $APPID + $VERSION + + + + + \ No newline at end of file diff --git a/broker/appengine/WEB-INF/butterfly.properties b/broker/appengine/WEB-INF/butterfly.properties new file mode 100644 index 000000000..ab7bf072f --- /dev/null +++ b/broker/appengine/WEB-INF/butterfly.properties @@ -0,0 +1,29 @@ +# +# Butterfly Configuration +# +# NOTE: properties passed to the JVM using '-Dkey=value' from the command line +# override the settings in this file. + +# indicates the URL path where butterfly is available in the proxy URL space +# as there is no way of knowing otherwise as this information is not +# transferred thru the HTTP protocol or otherwise (different story if +# the appserver is connected thru a different protocol such as AJP) + +butterfly.url = / + +# ---------- Miscellaneous ---------- + +#butterfly.locale.language = en +#butterfly.locale.country = US +#butterfly.timeZone = GMT+09:00 + +# ---------- Module ------ + +butterfly.modules.path = modules + +butterfly.modules.wirings = WEB-INF/modules.properties + +# ---------- Clustering ---- + +#butterfly.routing.cookie.maxage = -1 + diff --git a/broker/appengine/WEB-INF/jdoconfig.xml b/broker/appengine/WEB-INF/jdoconfig.xml new file mode 100644 index 000000000..fcb4f0b61 --- /dev/null +++ b/broker/appengine/WEB-INF/jdoconfig.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/broker/appengine/WEB-INF/lib/slf4j-jdk14-1.5.6.jar b/broker/appengine/WEB-INF/lib/slf4j-jdk14-1.5.6.jar new file mode 100644 index 000000000..1ce0a28e7 Binary files /dev/null and b/broker/appengine/WEB-INF/lib/slf4j-jdk14-1.5.6.jar differ diff --git a/broker/appengine/WEB-INF/logging.properties b/broker/appengine/WEB-INF/logging.properties new file mode 100644 index 000000000..7714de9fe --- /dev/null +++ b/broker/appengine/WEB-INF/logging.properties @@ -0,0 +1,13 @@ +# A default java.util.logging configuration. +# (All App Engine logging is through java.util.logging by default). +# +# To use this configuration, copy it into your application's WEB-INF +# folder and add the following to your appengine-web.xml: +# +# +# +# +# + +# Set the default logging level for all loggers +.level = WARN diff --git a/broker/appengine/WEB-INF/modules.properties b/broker/appengine/WEB-INF/modules.properties new file mode 100644 index 000000000..3b978d362 --- /dev/null +++ b/broker/appengine/WEB-INF/modules.properties @@ -0,0 +1,5 @@ +# +# Butterfly Modules Configuration +# + +appengine-broker = / diff --git a/broker/appengine/WEB-INF/web.xml b/broker/appengine/WEB-INF/web.xml new file mode 100644 index 000000000..63b0c075a --- /dev/null +++ b/broker/appengine/WEB-INF/web.xml @@ -0,0 +1,20 @@ + + + + + + + + Butterfly + edu.mit.simile.butterfly.Butterfly + 1 + + + + Butterfly + /* + + + diff --git a/broker/appengine/module/MOD-INF/module.properties b/broker/appengine/module/MOD-INF/module.properties new file mode 100644 index 000000000..951693540 --- /dev/null +++ b/broker/appengine/module/MOD-INF/module.properties @@ -0,0 +1,5 @@ +name = appengine-broker +extends = broker +description = Google App Engine implementation of Gridworks Broker Module +module-impl = com.metaweb.gridworks.broker.AppEngineGridworksBroker +templating = false diff --git a/broker/appengine/src/com/metaweb/gridworks/appengine/AppEngineClientConnection.java b/broker/appengine/src/com/metaweb/gridworks/appengine/AppEngineClientConnection.java new file mode 100644 index 000000000..9f0effa08 --- /dev/null +++ b/broker/appengine/src/com/metaweb/gridworks/appengine/AppEngineClientConnection.java @@ -0,0 +1,243 @@ +package com.metaweb.gridworks.appengine; + +import static com.google.appengine.api.urlfetch.FetchOptions.Builder.allowTruncate; + +import java.io.ByteArrayOutputStream; +import java.net.InetAddress; +import java.net.URL; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLSession; + +import org.apache.http.Header; +import org.apache.http.HttpConnectionMetrics; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.ProtocolVersion; +import org.apache.http.conn.ManagedClientConnection; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.message.BasicHttpResponse; +import org.apache.http.params.HttpParams; +import org.apache.http.protocol.HttpContext; + +import com.google.appengine.api.urlfetch.HTTPHeader; +import com.google.appengine.api.urlfetch.HTTPMethod; +import com.google.appengine.api.urlfetch.HTTPRequest; +import com.google.appengine.api.urlfetch.HTTPResponse; +import com.google.appengine.api.urlfetch.URLFetchService; +import com.google.appengine.api.urlfetch.URLFetchServiceFactory; + +class AppEngineClientConnection implements ManagedClientConnection { + // Managed is the composition of ConnectionReleaseTrigger, + // HttpClientConnection, HttpConnection, HttpInetConnection + + private HttpRoute _route; + private Object _state; + private boolean _reuseable; + + public AppEngineClientConnection(HttpRoute route, Object state) { + _route = route; + _state = state; + } + + // ManagedClientConnection methods + + public HttpRoute getRoute() { + return _route; + } + + public Object getState() { + return _state; + } + + public SSLSession getSSLSession() { + return null; + } + + public boolean isSecure() { + // XXX maybe parse the url to see if it's https? + return false; + } + + public boolean isMarkedReusable() { + return _reuseable; + } + + public void markReusable() { + _reuseable = true; + } + + public void layerProtocol(HttpContext context, HttpParams params) { + return; + } + + public void open(HttpRoute route, HttpContext context, HttpParams params) { + return; + } + + public void setIdleDuration(long duration, TimeUnit unit) { + return; + } + + public void setState(Object state) { + _state = state; + } + + public void tunnelProxy(HttpHost next, boolean secure, HttpParams params) { + return; + } + + public void tunnelTarget(boolean secure, HttpParams params) { + return; + } + + public void unmarkReusable() { + _reuseable = false; + } + + + // ConnectionReleaseTrigger methods + + public void releaseConnection() { + return; + } + + public void abortConnection() { + return; + } + + // HttpClientConnection methods + + private HTTPRequest _appengine_hrequest; + private HTTPResponse _appengine_hresponse; + + public void flush() { + return; + } + + public boolean isResponseAvailable(int timeout) { + // XXX possibly use Async fetcher + return true; + } + + public void receiveResponseEntity(org.apache.http.HttpResponse apache_response) { + byte[] data = _appengine_hresponse.getContent(); + + if (data != null) { + apache_response.setEntity(new ByteArrayEntity(data)); + } + } + + public HttpResponse receiveResponseHeader() { + URLFetchService ufs = URLFetchServiceFactory.getURLFetchService(); + try { + _appengine_hresponse = ufs.fetch(_appengine_hrequest); + } catch (java.io.IOException e) { + throw new RuntimeException(e); + } + + org.apache.http.HttpResponse apache_response = + new BasicHttpResponse(new ProtocolVersion("HTTP", 1, 0), + _appengine_hresponse.getResponseCode(), + null); + + for (HTTPHeader h : _appengine_hresponse.getHeaders()) { + apache_response.addHeader(h.getName(), h.getValue()); + } + + return apache_response; + } + + public void sendRequestEntity(org.apache.http.HttpEntityEnclosingRequest request) { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + + org.apache.http.HttpEntity ent = request.getEntity(); + if (ent != null) { + try { + ent.writeTo(os); + } catch (java.io.IOException e) { + throw new RuntimeException(e); + } + } + + _appengine_hrequest.setPayload(os.toByteArray()); + } + + public void sendRequestHeader(org.apache.http.HttpRequest apache_request) { + URL request_url; + + HttpHost host = _route.getTargetHost(); + + String protocol = host.getSchemeName(); + String addr = host.getHostName(); + int port = host.getPort(); + + String path = apache_request.getRequestLine().getUri(); + + try { + request_url = new URL(protocol, addr, port, path); + } catch (java.net.MalformedURLException e) { + throw new RuntimeException(e); + } + + HTTPMethod method = HTTPMethod.valueOf(apache_request.getRequestLine().getMethod()); + _appengine_hrequest = new HTTPRequest(request_url, method, allowTruncate() + .doNotFollowRedirects()); + + Header[] apache_headers = apache_request.getAllHeaders(); + for (int i = 0; i < apache_headers.length; i++) { + Header h = apache_headers[i]; + _appengine_hrequest + .setHeader(new HTTPHeader(h.getName(), h.getValue())); + } + } + + // HttpConnection methods + + public void close() { + return; + } + + public HttpConnectionMetrics getMetrics() { + return null; + } + + public int getSocketTimeout() { + return -1; + } + + public boolean isOpen() { + return true; + } + + public boolean isStale() { + return false; + } + + public void setSocketTimeout(int timeout) { + return; + } + + public void shutdown() { + return; + } + + // HttpInetConnection methods + + public InetAddress getLocalAddress() { + return null; + } + + public int getLocalPort() { + return -1; + } + + public InetAddress getRemoteAddress() { + return null; + } + + public int getRemotePort() { + return -1; + } +} \ No newline at end of file diff --git a/broker/appengine/src/com/metaweb/gridworks/appengine/AppEngineClientConnectionManager.java b/broker/appengine/src/com/metaweb/gridworks/appengine/AppEngineClientConnectionManager.java new file mode 100644 index 000000000..c0a9f7225 --- /dev/null +++ b/broker/appengine/src/com/metaweb/gridworks/appengine/AppEngineClientConnectionManager.java @@ -0,0 +1,76 @@ +package com.metaweb.gridworks.appengine; + +import java.net.InetAddress; +import java.net.Socket; +import java.util.concurrent.TimeUnit; + +import org.apache.http.conn.ClientConnectionManager; +import org.apache.http.conn.ClientConnectionRequest; +import org.apache.http.conn.ManagedClientConnection; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.conn.scheme.Scheme; +import org.apache.http.conn.scheme.SchemeRegistry; +import org.apache.http.conn.scheme.SocketFactory; +import org.apache.http.params.HttpParams; + +public class AppEngineClientConnectionManager implements ClientConnectionManager { + + private SchemeRegistry schemes; + + class NoopSocketFactory implements SocketFactory { + public Socket connectSocket(Socket sock, String host, int port, InetAddress addr, int lport, HttpParams params) { + return null; + } + + public Socket createSocket() { + return null; + } + + public boolean isSecure(Socket sock) { + return false; + } + } + + public AppEngineClientConnectionManager() { + SocketFactory noop_sf = new NoopSocketFactory(); + schemes = new SchemeRegistry(); + schemes.register(new Scheme("http", noop_sf, 80)); + schemes.register(new Scheme("https", noop_sf, 443)); + } + + public void closeExpiredConnections() { + return; + } + + public void closeIdleConnections(long idletime, TimeUnit tunit) { + return; + } + + public ManagedClientConnection getConnection(HttpRoute route, Object state) { + return new AppEngineClientConnection(route, state); + } + + public SchemeRegistry getSchemeRegistry() { + return schemes; + } + + public void releaseConnection(ManagedClientConnection conn, long valid, TimeUnit tuint) { + return; + } + + public ClientConnectionRequest requestConnection(final HttpRoute route, final Object state) { + return new ClientConnectionRequest() { + public void abortRequest() { + return; + } + + public ManagedClientConnection getConnection(long idletime, TimeUnit tunit) { + return AppEngineClientConnectionManager.this.getConnection(route, state); + } + }; + } + + public void shutdown() { + return; + } +} diff --git a/broker/appengine/src/com/metaweb/gridworks/broker/AppEngineGridworksBroker.java b/broker/appengine/src/com/metaweb/gridworks/broker/AppEngineGridworksBroker.java new file mode 100644 index 000000000..7f6350e2d --- /dev/null +++ b/broker/appengine/src/com/metaweb/gridworks/broker/AppEngineGridworksBroker.java @@ -0,0 +1,322 @@ +package com.metaweb.gridworks.broker; + +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; + +import javax.jdo.JDOHelper; +import javax.jdo.PersistenceManager; +import javax.jdo.PersistenceManagerFactory; +import javax.jdo.Transaction; +import javax.jdo.annotations.IdGeneratorStrategy; +import javax.jdo.annotations.PersistenceCapable; +import javax.jdo.annotations.Persistent; +import javax.jdo.annotations.PrimaryKey; +import javax.servlet.ServletConfig; +import javax.servlet.http.HttpServletResponse; + +import org.apache.http.client.HttpClient; +import org.apache.http.conn.ClientConnectionManager; +import org.apache.http.impl.client.DefaultHttpClient; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.appengine.api.datastore.Text; +import com.metaweb.gridworks.appengine.AppEngineClientConnectionManager; + +public class AppEngineGridworksBroker extends GridworksBroker { + + protected static final Logger logger = LoggerFactory.getLogger("gridworks.broker.appengine"); + + PersistenceManagerFactory pmfInstance; + + @Override + public void init(ServletConfig config) throws Exception { + super.init(config); + + pmfInstance = JDOHelper.getPersistenceManagerFactory("transactional"); + } + + @Override + public void destroy() throws Exception { + } + + // --------------------------------------------------------------------------------- + + protected HttpClient getHttpClient() { + ClientConnectionManager cm = new AppEngineClientConnectionManager(); + return new DefaultHttpClient(cm, null); + } + + // --------------------------------------------------------------------------------- + + protected void getLock(HttpServletResponse response, String pid) throws Exception { + PersistenceManager pm = pmfInstance.getPersistenceManager(); + + try { + respond(response, lockToJSON(getLock(pm,pid))); + } finally { + pm.close(); + } + } + + protected void obtainLock(HttpServletResponse response, String pid, String uid) throws Exception { + PersistenceManager pm = pmfInstance.getPersistenceManager(); + + try { + Lock lock = getLock(pm, pid); + if (lock == null) { + Transaction tx = pm.currentTransaction(); + + try { + tx.begin(); + lock = new Lock(Long.toHexString(tx.hashCode()), pid, uid); + pm.makePersistent(lock); + tx.commit(); + } finally { + if (tx.isActive()) { + tx.rollback(); + } + } + } + + respond(response, lockToJSON(lock)); + + } finally { + pm.close(); + } + } + + protected void releaseLock(HttpServletResponse response, String pid, String uid, String lid) throws Exception { + + PersistenceManager pm = pmfInstance.getPersistenceManager(); + + try { + Lock lock = getLock(pm, pid); + if (lock != null) { + if (!lock.id.equals(lid)) { + throw new RuntimeException("Lock id doesn't match, can't release the lock"); + } + if (!lock.uid.equals(uid)) { + throw new RuntimeException("User id doesn't match the lock owner, can't release the lock"); + } + + Transaction tx = pm.currentTransaction(); + + try { + tx.begin(); + pm.deletePersistent(lock); + tx.commit(); + } finally { + if (tx.isActive()) { + tx.rollback(); + } + } + } + + respond(response, OK); + + } finally { + pm.close(); + } + } + + // ---------------------------------------------------------------------------------------------------- + + protected void startProject(HttpServletResponse response, String pid, String uid, String lid, String data) throws Exception { + PersistenceManager pm = pmfInstance.getPersistenceManager(); + + try { + checkLock(pm, pid, uid, lid); + + Project project = getProject(pm, pid); + + if (project != null) { + throw new RuntimeException("Project '" + pid + "' already exists"); + } + + Transaction tx = pm.currentTransaction(); + + try { + tx.begin(); + project = new Project(pid, data); + pm.makePersistent(project); + tx.commit(); + } finally { + if (tx.isActive()) { + tx.rollback(); + } + } + + respond(response, OK); + } finally { + pm.close(); + } + } + + protected void addTransformations(HttpServletResponse response, String pid, String uid, String lid, List transformations) throws Exception { + PersistenceManager pm = pmfInstance.getPersistenceManager(); + + try { + checkLock(pm, pid, uid, lid); + + Project project = getProject(pm, pid); + + if (project == null) { + throw new RuntimeException("Project '" + pid + "' not found"); + } + + Transaction tx = pm.currentTransaction(); + + try { + for (String s : transformations) { + project.transformations.add(new Text(s)); + } + tx.commit(); + } finally { + if (tx.isActive()) { + tx.rollback(); + } + } + + respond(response, OK); + } finally { + pm.close(); + } + } + + // --------------------------------------------------------------------------------- + + protected void getProject(HttpServletResponse response, String pid) throws Exception { + PersistenceManager pm = pmfInstance.getPersistenceManager(); + + try { + Project project = getProject(pm, pid); + + Writer w = response.getWriter(); + JSONWriter writer = new JSONWriter(w); + writer.object(); + writer.key("data"); writer.value(project.data.toString()); + writer.key("transformations"); + writer.array(); + for (Text s : project.transformations) { + writer.value(s.toString()); + } + writer.endArray(); + writer.endObject(); + w.flush(); + w.close(); + } finally { + pm.close(); + } + } + + protected void getHistory(HttpServletResponse response, String pid, int tindex) throws Exception { + PersistenceManager pm = pmfInstance.getPersistenceManager(); + + try { + Project project = getProject(pm, pid); + + Writer w = response.getWriter(); + JSONWriter writer = new JSONWriter(w); + writer.object(); + writer.key("transformations"); + writer.array(); + int size = project.transformations.size(); + for (int i = tindex; i < size; i++) { + writer.value(project.transformations.get(i).toString()); + } + writer.endArray(); + writer.endObject(); + w.flush(); + w.close(); + } finally { + pm.close(); + } + } + + // --------------------------------------------------------------------------------- + + Project getProject(PersistenceManager pm, String pid) { + Project project = pm.getObjectById(Project.class, pid); + if (project == null) { + throw new RuntimeException("Project '" + pid + "' is not managed by this broker"); + } + return project; + } + + @PersistenceCapable + static class Project { + + @PrimaryKey + @Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY) + String pid; + + @Persistent + List transformations = new ArrayList(); + + @Persistent + Text data; + + Project(String pid, String data) { + this.pid = pid; + this.data = new Text(data); + } + } + + // --------------------------------------------------------------------------------- + + Lock getLock(PersistenceManager pm, String pid) { + return pm.getObjectById(Lock.class, pid); + } + + void checkLock(PersistenceManager pm, String pid, String uid, String lid) { + Lock lock = getLock(pm, pid); + + if (lock == null) { + throw new RuntimeException("No lock was found with the given Lock id '" + lid + "', you have to have a valid lock on a project in order to start it"); + } + + if (!lock.pid.equals(pid)) { + throw new RuntimeException("Lock '" + lid + "' is for another project: " + pid); + } + + if (!lock.uid.equals(uid)) { + throw new RuntimeException("Lock '" + lid + "' is owned by another user: " + uid); + } + } + + JSONObject lockToJSON(Lock lock) throws JSONException { + JSONObject o = new JSONObject(); + if (lock != null) { + o.put("lock_id", lock.id); + o.put("project_id", lock.pid); + o.put("user_id", lock.uid); + } + return o; + } + + @PersistenceCapable + static class Lock { + + @Persistent + String id; + + @PrimaryKey + @Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY) + String pid; + + @Persistent + String uid; + + Lock(String id, String pid, String uid) { + this.id = id; + this.pid = pid; + this.uid = uid; + } + } + +} diff --git a/broker/core/.classpath b/broker/core/.classpath new file mode 100644 index 000000000..691aebe5c --- /dev/null +++ b/broker/core/.classpath @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/broker/core/.project b/broker/core/.project new file mode 100644 index 000000000..95bbf2e63 --- /dev/null +++ b/broker/core/.project @@ -0,0 +1,17 @@ + + + gridworks-broker + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/broker/core/module/MOD-INF/module.properties b/broker/core/module/MOD-INF/module.properties new file mode 100644 index 000000000..215955193 --- /dev/null +++ b/broker/core/module/MOD-INF/module.properties @@ -0,0 +1,4 @@ +name = broker +description = Gridworks Broker +module-impl = com.metaweb.gridworks.broker.GridworksBroker +templating = false diff --git a/broker/core/src/com/metaweb/gridworks/broker/GridworksBroker.java b/broker/core/src/com/metaweb/gridworks/broker/GridworksBroker.java new file mode 100644 index 000000000..8032c2a7c --- /dev/null +++ b/broker/core/src/com/metaweb/gridworks/broker/GridworksBroker.java @@ -0,0 +1,265 @@ + +package com.metaweb.gridworks.broker; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.http.NameValuePair; +import org.apache.http.client.HttpClient; +import org.apache.http.client.ResponseHandler; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.BasicResponseHandler; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.params.CoreProtocolPNames; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import edu.mit.simile.butterfly.ButterflyModuleImpl; + +/** + * This class contains all the code shared by various implementations of a Gridworks Broker. + * + * A broker is a server used by multiple Gridworks installations to enable collaborative + * development over the same project. + * + * Broker implementations differ in how they store their state but all of them are required + * to extend this abstract class and implement the services that are called via HTTP. + * + */ +public abstract class GridworksBroker extends ButterflyModuleImpl { + + protected static final Logger logger = LoggerFactory.getLogger("gridworks.broker"); + + static final protected String USER_INFO_URL = "http://www.freebase.com/api/service/user_info"; + static final protected String DELEGATED_OAUTH_HEADER = "X-Freebase-Credentials"; + static final protected String OAUTH_HEADER = "Authorization"; + + static protected String OK; + + static { + try { + JSONObject o = new JSONObject(); + o.put("status","ok"); + OK = o.toString(); + } catch (JSONException e) { + // not going to happen; + } + } + + protected HttpClient httpclient; + + @Override + public void init(ServletConfig config) throws Exception { + super.init(config); + httpclient = getHttpClient(); + } + + @Override + public void destroy() throws Exception { + httpclient.getConnectionManager().shutdown(); + } + + @Override + public boolean process(String path, HttpServletRequest request, HttpServletResponse response) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("> process {}", path); + } else { + logger.info("process {}", path); + } + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + try { + String uid = getUserId(request); + logger.debug("uid: {}", path); + String pid = getParameter(request, "pid"); + logger.debug("pid: {}", path); + + // NOTE: conditionals should be ordered by call frequency estimate to (slightly) improve performance + // we could be using a hashtable and some classes that implement the commands, but the complexity overhead + // doesn't seem to justify the marginal benefit. + + if ("get_lock".equals(path)) { + getLock(response, pid); + } else if ("obtain_lock".equals(path)) { + obtainLock(response, pid, uid); + } else if ("release_lock".equals(path)) { + releaseLock(response, pid, uid, getParameter(request, "lock")); + } else if ("history".equals(path)) { + getHistory(response, pid, getInteger(request, "tindex")); + } else if ("transform".equals(path)) { + addTransformations(response, pid, uid, getParameter(request, "lock"), getList(request, "transformations")); + } else if ("start".equals(path)) { + startProject(response, pid, uid, getParameter(request, "lock"), getParameter(request, "data")); + } else if ("get".equals(path)) { + getProject(response, pid); + } + + } catch (RuntimeException e) { + logger.error("runtime error", e); + respondError(response, e.getMessage()); + } catch (Exception e) { + logger.error("internal error", e); + respondException(response, e); + } + + if (logger.isDebugEnabled()) logger.debug("< process {}", path); + + return super.process(path, request, response); + } + + // ---------------------------------------------------------------------------------------- + + protected abstract HttpClient getHttpClient(); + + protected abstract void getLock(HttpServletResponse response, String pid) throws Exception; + + protected abstract void obtainLock(HttpServletResponse response, String pid, String uid) throws Exception; + + protected abstract void releaseLock(HttpServletResponse response, String pid, String uid, String lock) throws Exception; + + protected abstract void startProject(HttpServletResponse response, String pid, String uid, String lock, String data) throws Exception; + + protected abstract void addTransformations(HttpServletResponse response, String pid, String uid, String lock, List transformations) throws Exception; + + protected abstract void getProject(HttpServletResponse response, String pid) throws Exception; + + protected abstract void getHistory(HttpServletResponse response, String pid, int tindex) throws Exception; + + // ---------------------------------------------------------------------------------------- + + @SuppressWarnings("unchecked") + protected String getUserId(HttpServletRequest request) throws Exception { + + String oauth = request.getHeader(DELEGATED_OAUTH_HEADER); + if (oauth == null) { + throw new RuntimeException("The request needs to contain the '" + DELEGATED_OAUTH_HEADER + "' header set to obtain user identity via Freebase."); + } + + List formparams = new ArrayList(); + Map params = (Map) request.getParameterMap(); + for (Entry e : params.entrySet()) { + formparams.add(new BasicNameValuePair((String) e.getKey(), (String) e.getValue())); + } + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); + + HttpPost httpRequest = new HttpPost(USER_INFO_URL); + httpRequest.setHeader(OAUTH_HEADER, oauth); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks Broker"); + httpRequest.setEntity(entity); + + ResponseHandler responseHandler = new BasicResponseHandler(); + String responseBody = httpclient.execute(httpRequest, responseHandler); + JSONObject o = new JSONObject(responseBody); + + return o.getString("username"); + } + + // ---------------------------------------------------------------------------------------- + + static protected String getParameter(HttpServletRequest request, String name) throws ServletException { + String param = request.getParameter(name); + if (param == null) { + throw new ServletException("request must come with a '" + name + "' parameter"); + } + return param; + } + + static protected List getList(HttpServletRequest request, String name) throws ServletException, JSONException { + String param = getParameter(request, name); + JSONArray a = new JSONArray(param); + List result = new ArrayList(a.length()); + for (int i = 0; i < a.length(); i++) { + result.add(a.getString(i)); + } + return result; + } + + static protected int getInteger(HttpServletRequest request, String name) throws ServletException, JSONException { + return Integer.parseInt(getParameter(request, name)); + } + + static protected void respondError(HttpServletResponse response, String error) throws IOException, ServletException { + + if (response == null) { + throw new ServletException("Response object can't be null"); + } + + try { + JSONObject o = new JSONObject(); + o.put("code", "error"); + o.put("message", error); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + respond(response, o.toString()); + } catch (JSONException e) { + e.printStackTrace(response.getWriter()); + } + } + + static protected void respondException(HttpServletResponse response, Exception e) throws IOException, ServletException { + + if (response == null) { + throw new ServletException("Response object can't be null"); + } + + try { + JSONObject o = new JSONObject(); + o.put("code", "error"); + o.put("message", e.getMessage()); + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + e.printStackTrace(pw); + pw.flush(); + sw.flush(); + + o.put("stack", sw.toString()); + + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + respond(response, o.toString()); + } catch (JSONException e1) { + e.printStackTrace(response.getWriter()); + } + } + + static protected void respond(HttpServletResponse response, JSONObject content) throws IOException, ServletException { + if (content == null) { + throw new ServletException("Content object can't be null"); + } + + JSONObject o = new JSONObject(); + respond(response, o.toString()); + } + + static protected void respond(HttpServletResponse response, String content) throws IOException, ServletException { + if (response == null) { + throw new ServletException("Response object can't be null"); + } + + Writer w = response.getWriter(); + if (w != null) { + w.write(content); + w.flush(); + w.close(); + } else { + throw new ServletException("response returned a null writer"); + } + } +} diff --git a/broker/local/.classpath b/broker/local/.classpath new file mode 100644 index 000000000..d25c3081d --- /dev/null +++ b/broker/local/.classpath @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/broker/local/.project b/broker/local/.project new file mode 100644 index 000000000..3c6a98419 --- /dev/null +++ b/broker/local/.project @@ -0,0 +1,17 @@ + + + gridworks-local-broker + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/broker/local/WEB-INF/butterfly.properties b/broker/local/WEB-INF/butterfly.properties new file mode 100644 index 000000000..ab7bf072f --- /dev/null +++ b/broker/local/WEB-INF/butterfly.properties @@ -0,0 +1,29 @@ +# +# Butterfly Configuration +# +# NOTE: properties passed to the JVM using '-Dkey=value' from the command line +# override the settings in this file. + +# indicates the URL path where butterfly is available in the proxy URL space +# as there is no way of knowing otherwise as this information is not +# transferred thru the HTTP protocol or otherwise (different story if +# the appserver is connected thru a different protocol such as AJP) + +butterfly.url = / + +# ---------- Miscellaneous ---------- + +#butterfly.locale.language = en +#butterfly.locale.country = US +#butterfly.timeZone = GMT+09:00 + +# ---------- Module ------ + +butterfly.modules.path = modules + +butterfly.modules.wirings = WEB-INF/modules.properties + +# ---------- Clustering ---- + +#butterfly.routing.cookie.maxage = -1 + diff --git a/broker/local/WEB-INF/modules.properties b/broker/local/WEB-INF/modules.properties new file mode 100644 index 000000000..b74d3bfca --- /dev/null +++ b/broker/local/WEB-INF/modules.properties @@ -0,0 +1,5 @@ +# +# Butterfly Modules Configuration +# + +local-broker = / diff --git a/broker/local/WEB-INF/web.xml b/broker/local/WEB-INF/web.xml new file mode 100644 index 000000000..63b0c075a --- /dev/null +++ b/broker/local/WEB-INF/web.xml @@ -0,0 +1,20 @@ + + + + + + + + Butterfly + edu.mit.simile.butterfly.Butterfly + 1 + + + + Butterfly + /* + + + diff --git a/broker/local/licenses/je.license.txt b/broker/local/licenses/je.license.txt new file mode 100644 index 000000000..2d35190c0 --- /dev/null +++ b/broker/local/licenses/je.license.txt @@ -0,0 +1,67 @@ +=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +/* + * Copyright (c) 2002-2010 Oracle. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Redistributions in any form must be accompanied by information on + * how to obtain complete source code for the Oracle Berkeley DB + * Java Edition software and any accompanying software that uses the + * Oracle Berkeley DB Java Edition software. The source code must + * either be included in the distribution or be available for no + * more than the cost of distribution plus a nominal fee, and must be + * freely redistributable under reasonable conditions. For an + * executable file, complete source code means the source code for all + * modules it contains. It does not include source code for modules or + * files that typically accompany the major components of the operating + * system on which the executable file runs. + * + * THIS SOFTWARE IS PROVIDED BY ORACLE ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR + * NON-INFRINGEMENT, ARE DISCLAIMED. IN NO EVENT SHALL ORACLE BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +/*** + * ASM: a very small and fast Java bytecode manipulation framework + * Copyright (c) 2000-2005 INRIA, France Telecom + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + diff --git a/broker/local/module/MOD-INF/lib-src/bdb-je-4.0.103-sources.jar b/broker/local/module/MOD-INF/lib-src/bdb-je-4.0.103-sources.jar new file mode 100644 index 000000000..b031a8d0e Binary files /dev/null and b/broker/local/module/MOD-INF/lib-src/bdb-je-4.0.103-sources.jar differ diff --git a/broker/local/module/MOD-INF/lib/bdb-je-4.0.103.jar b/broker/local/module/MOD-INF/lib/bdb-je-4.0.103.jar new file mode 100644 index 000000000..a821080b8 Binary files /dev/null and b/broker/local/module/MOD-INF/lib/bdb-je-4.0.103.jar differ diff --git a/broker/local/module/MOD-INF/module.properties b/broker/local/module/MOD-INF/module.properties new file mode 100644 index 000000000..ba86c3eb5 --- /dev/null +++ b/broker/local/module/MOD-INF/module.properties @@ -0,0 +1,5 @@ +name = local-broker +extends = broker +description = Local implementation of Gridworks Broker Module +module-impl = com.metaweb.gridworks.broker.LocalDBGridworksBroker +templating = false diff --git a/broker/local/src/com/metaweb/gridworks/broker/LocalGridworksBroker.java b/broker/local/src/com/metaweb/gridworks/broker/LocalGridworksBroker.java new file mode 100644 index 000000000..77dce4702 --- /dev/null +++ b/broker/local/src/com/metaweb/gridworks/broker/LocalGridworksBroker.java @@ -0,0 +1,309 @@ +package com.metaweb.gridworks.broker; + +import java.io.File; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; + +import javax.servlet.ServletConfig; +import javax.servlet.http.HttpServletResponse; + +import org.apache.http.client.HttpClient; +import org.apache.http.impl.client.DefaultHttpClient; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.sleepycat.je.Environment; +import com.sleepycat.je.EnvironmentConfig; +import com.sleepycat.je.Transaction; +import com.sleepycat.persist.EntityStore; +import com.sleepycat.persist.PrimaryIndex; +import com.sleepycat.persist.StoreConfig; +import com.sleepycat.persist.model.Entity; +import com.sleepycat.persist.model.PrimaryKey; + +public class LocalGridworksBroker extends GridworksBroker { + + protected static final Logger logger = LoggerFactory.getLogger("gridworks.broker.local"); + + Environment env; + EntityStore projectStore; + EntityStore lockStore; + + PrimaryIndex projectById; + PrimaryIndex lockByProject; + + protected HttpClient httpclient; + + @Override + public void init(ServletConfig config) throws Exception { + super.init(config); + + File dataPath = new File("data"); // FIXME: data should be configurable; + + EnvironmentConfig envConfig = new EnvironmentConfig(); + envConfig.setAllowCreate(true); + envConfig.setTransactional(true); + env = new Environment(dataPath, envConfig); + + StoreConfig storeConfig = new StoreConfig(); + storeConfig.setAllowCreate(true); + storeConfig.setTransactional(true); + projectStore = new EntityStore(env, "ProjectsStore", storeConfig); + lockStore = new EntityStore(env, "LockStore", storeConfig); + + projectById = projectStore.getPrimaryIndex(String.class, Project.class); + lockByProject = lockStore.getPrimaryIndex(String.class, Lock.class); + } + + @Override + public void destroy() throws Exception { + super.destroy(); + + if (projectStore != null) { + projectStore.close(); + } + + if (lockStore != null) { + lockStore.close(); + } + + if (env != null) { + env.cleanLog(); + env.close(); + } + } + + // --------------------------------------------------------------------------------- + + protected HttpClient getHttpClient() { + return new DefaultHttpClient(); + } + + // --------------------------------------------------------------------------------- + + protected void getLock(HttpServletResponse response, String pid) throws Exception { + respond(response, lockToJSON(getLock(pid))); + } + + protected void obtainLock(HttpServletResponse response, String pid, String uid) throws Exception { + + Transaction txn = env.beginTransaction(null, null); + + Lock lock = getLock(pid); + if (lock == null) { + try { + lock = new Lock(Long.toHexString(txn.getId()), pid, uid); + lockByProject.put(txn, lock); + txn.commit(); + } finally { + if (txn != null) { + txn.abort(); + txn = null; + } + } + } + + respond(response, lockToJSON(lock)); + } + + protected void releaseLock(HttpServletResponse response, String pid, String uid, String lid) throws Exception { + + Transaction txn = env.beginTransaction(null, null); + + Lock lock = getLock(pid); + if (lock != null) { + try { + if (!lock.id.equals(lid)) { + throw new RuntimeException("Lock id doesn't match, can't release the lock"); + } + if (!lock.uid.equals(uid)) { + throw new RuntimeException("User id doesn't match the lock owner, can't release the lock"); + } + + lockByProject.delete(pid); + + txn.commit(); + } finally { + if (txn != null) { + txn.abort(); + txn = null; + } + } + } + + respond(response, OK); + } + + // ---------------------------------------------------------------------------------------------------- + + protected void startProject(HttpServletResponse response, String pid, String uid, String lid, String data) throws Exception { + + Transaction txn = env.beginTransaction(null, null); + + checkLock(pid, uid, lid); + + if (projectById.contains(pid)) { + throw new RuntimeException("Project '" + pid + "' already exists"); + } + + try { + projectById.put(txn, new Project(pid, data)); + txn.commit(); + } finally { + if (txn != null) { + txn.abort(); + txn = null; + } + } + + respond(response, OK); + } + + protected void addTransformations(HttpServletResponse response, String pid, String uid, String lid, List transformations) throws Exception { + + Transaction txn = env.beginTransaction(null, null); + + checkLock(pid, uid, lid); + + Project project = getProject(pid); + + if (project == null) { + throw new RuntimeException("Project '" + pid + "' not found"); + } + + try { + project.transformations.addAll(transformations); + txn.commit(); + } finally { + if (txn != null) { + txn.abort(); + txn = null; + } + } + + respond(response, OK); + } + + // --------------------------------------------------------------------------------- + + protected void getProject(HttpServletResponse response, String pid) throws Exception { + Project project = getProject(pid); + + Writer w = response.getWriter(); + JSONWriter writer = new JSONWriter(w); + writer.object(); + writer.key("data"); writer.value(project.data); + writer.key("transformations"); + writer.array(); + for (String s : project.transformations) { + writer.value(s); + } + writer.endArray(); + writer.endObject(); + w.flush(); + w.close(); + } + + protected void getHistory(HttpServletResponse response, String pid, int tindex) throws Exception { + Project project = getProject(pid); + + Writer w = response.getWriter(); + JSONWriter writer = new JSONWriter(w); + writer.object(); + writer.key("transformations"); + writer.array(); + int size = project.transformations.size(); + for (int i = tindex; i < size; i++) { + writer.value(project.transformations.get(i)); + } + writer.endArray(); + writer.endObject(); + w.flush(); + w.close(); + } + + // --------------------------------------------------------------------------------- + + Project getProject(String pid) { + Project project = projectById.get(pid); + if (project == null) { + throw new RuntimeException("Project '" + pid + "' is not managed by this broker"); + } + return project; + } + + @Entity + class Project { + + @PrimaryKey + String pid; + + List transformations = new ArrayList(); + + String data; + + Project(String pid, String data) { + this.pid = pid; + this.data = data; + } + + @SuppressWarnings("unused") + private Project() {} + } + + // --------------------------------------------------------------------------------- + + Lock getLock(String pid) { + return lockByProject.get(pid); + } + + void checkLock(String pid, String uid, String lid) { + Lock lock = getLock(pid); + + if (lock == null) { + throw new RuntimeException("No lock was found with the given Lock id '" + lid + "', you have to have a valid lock on a project in order to start it"); + } + + if (!lock.pid.equals(pid)) { + throw new RuntimeException("Lock '" + lid + "' is for another project: " + pid); + } + + if (!lock.uid.equals(uid)) { + throw new RuntimeException("Lock '" + lid + "' is owned by another user: " + uid); + } + } + + JSONObject lockToJSON(Lock lock) throws JSONException { + JSONObject o = new JSONObject(); + if (lock != null) { + o.put("lock_id", lock.id); + o.put("project_id", lock.pid); + o.put("user_id", lock.uid); + } + return o; + } + + @Entity + class Lock { + + @PrimaryKey + String pid; + + String id; + + String uid; + + Lock(String id, String pid, String uid) { + this.id = id; + this.pid = pid; + this.uid = uid; + } + + @SuppressWarnings("unused") + private Lock() {} + } +}