s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

more work on the broker

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1035 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Stefano Mazzocchi 2010-06-25 07:03:30 +00:00
parent 5193630b9a
commit 7fa9d1d812
7 changed files with 41 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
broker/core/.classpath
View File

@ -3,7 +3,7 @@
<classpathentry kind="src" path="src"/> <classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/> <classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/>
<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk.jar"/> <classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk.jar"/>
<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/> <classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/bdb-je-4.0.103.jar" sourcepath="module/MOD-INF/lib-src/bdb-je-4.0.103-sources.jar"/> <classpathentry kind="lib" path="module/MOD-INF/lib/bdb-je-4.0.103.jar" sourcepath="module/MOD-INF/lib-src/bdb-je-4.0.103-sources.jar"/>
<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/> <classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/>

5
broker/core/WEB-INF/web.xml
View File

@ -7,12 +7,13 @@
<web-app> <web-app>
<servlet> <servlet>
<servlet-name>gridworks-broker</servlet-name> <servlet-name>Butterfly</servlet-name>
<servlet-class>edu.mit.simile.butterfly.Butterfly</servlet-class> <servlet-class>edu.mit.simile.butterfly.Butterfly</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet> </servlet>
<servlet-mapping> <servlet-mapping>
<servlet-name>gridworks-broker</servlet-name> <servlet-name>Butterfly</servlet-name>
<url-pattern>/*</url-pattern> <url-pattern>/*</url-pattern>
</servlet-mapping> </servlet-mapping>

21
broker/core/src/com/metaweb/gridworks/broker/GridworksBroker.java
View File

@ -67,16 +67,12 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
protected HttpClient httpclient; protected HttpClient httpclient;
protected boolean developmentMode;
@Override @Override
public void init(ServletConfig config) throws Exception { public void init(ServletConfig config) throws Exception {
super.init(config); super.init(config);
httpclient = getHttpClient(); httpclient = getHttpClient();
developmentMode = Boolean.parseBoolean(config.getInitParameter("gridworks.development"));
if (developmentMode) logger.warn("Running in development mode");
} }
@Override @Override
public void destroy() throws Exception { public void destroy() throws Exception {
httpclient.getConnectionManager().shutdown(); httpclient.getConnectionManager().shutdown();
@ -95,14 +91,14 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
try { try {
String uid = getUserId(request); String uid = getUserId(request);
logger.debug("uid: {}", uid); logger.debug("uid: {}", path);
String pid = getParameter(request, "pid"); String pid = getParameter(request, "pid");
logger.debug("pid: {}", pid); logger.debug("pid: {}", path);
// NOTE: conditionals should be ordered by call frequency estimate to (slightly) improve performance // NOTE: conditionals should be ordered by call frequency estimate to (slightly) improve performance
// we could be using a hashtable and some classes that implement the commands, but the complexity overhead // we could be using a hashtable and some classes that implement the commands, but the complexity overhead
// doesn't seem to justify the marginal benefit. // doesn't seem to justify the marginal benefit.
if ("get_lock".equals(path)) { if ("get_lock".equals(path)) {
getLock(response, pid); getLock(response, pid);
} else if ("expire_locks".equals(path)) { } else if ("expire_locks".equals(path)) {
@ -163,11 +159,6 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
protected String getUserId(HttpServletRequest request) throws Exception { protected String getUserId(HttpServletRequest request) throws Exception {
// This is useful for testing
if (developmentMode) {
return getParameter(request, "uid");
}
String oauth = request.getHeader(DELEGATED_OAUTH_HEADER); String oauth = request.getHeader(DELEGATED_OAUTH_HEADER);
if (oauth == null) { if (oauth == null) {
throw new RuntimeException("The request needs to contain the '" + DELEGATED_OAUTH_HEADER + "' header set to obtain user identity via Freebase."); throw new RuntimeException("The request needs to contain the '" + DELEGATED_OAUTH_HEADER + "' header set to obtain user identity via Freebase.");
@ -197,7 +188,7 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
static protected String getParameter(HttpServletRequest request, String name) throws ServletException { static protected String getParameter(HttpServletRequest request, String name) throws ServletException {
String param = request.getParameter(name); String param = request.getParameter(name);
if (param == null) { if (param == null) {
throw new RuntimeException("request must come with a '" + name + "' parameter"); throw new ServletException("request must come with a '" + name + "' parameter");
} }
return param; return param;
} }
@ -215,7 +206,7 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
static protected int getInteger(HttpServletRequest request, String name) throws ServletException, JSONException { static protected int getInteger(HttpServletRequest request, String name) throws ServletException, JSONException {
return Integer.parseInt(getParameter(request, name)); return Integer.parseInt(getParameter(request, name));
} }
static protected void respondError(HttpServletResponse response, String error) throws IOException, ServletException { static protected void respondError(HttpServletResponse response, String error) throws IOException, ServletException {
if (response == null) { if (response == null) {

38
broker/core/src/com/metaweb/gridworks/broker/GridworksBrokerImpl.java
View File

@ -21,7 +21,6 @@ import org.slf4j.LoggerFactory;
import com.sleepycat.je.Environment; import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.Transaction; import com.sleepycat.je.Transaction;
import com.sleepycat.persist.EntityCursor;
import com.sleepycat.persist.EntityStore; import com.sleepycat.persist.EntityStore;
import com.sleepycat.persist.PrimaryIndex; import com.sleepycat.persist.PrimaryIndex;
import com.sleepycat.persist.StoreConfig; import com.sleepycat.persist.StoreConfig;
@ -41,7 +40,7 @@ public class GridworksBrokerImpl extends GridworksBroker {
Timer timer; Timer timer;
LockExpirer expirer; LockExpirer expirer;
@Override @Override
public void init(ServletConfig config) throws Exception { public void init(ServletConfig config) throws Exception {
super.init(config); super.init(config);
@ -50,11 +49,9 @@ public class GridworksBrokerImpl extends GridworksBroker {
expirer = new LockExpirer(); expirer = new LockExpirer();
timer.schedule(expirer, LOCK_EXPIRATION_CHECK_DELAY, LOCK_EXPIRATION_CHECK_DELAY); timer.schedule(expirer, LOCK_EXPIRATION_CHECK_DELAY, LOCK_EXPIRATION_CHECK_DELAY);
String dataDir = config.getInitParameter("gridworks.data"); File dataPath = new File("data"); // FIXME: data should be configurable;
if (dataDir == null) dataDir = "data";
File dataPath = new File(dataDir);
if (!dataPath.exists()) dataPath.mkdirs(); if (!dataPath.exists()) dataPath.mkdirs();
EnvironmentConfig envConfig = new EnvironmentConfig(); EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setAllowCreate(true); envConfig.setAllowCreate(true);
envConfig.setTransactional(true); envConfig.setTransactional(true);
@ -76,43 +73,30 @@ public class GridworksBrokerImpl extends GridworksBroker {
if (projectStore != null) { if (projectStore != null) {
projectStore.close(); projectStore.close();
projectById = null;
} }
if (lockStore != null) { if (lockStore != null) {
lockStore.close(); lockStore.close();
lockByProject = null;
} }
if (timer != null) {
timer.cancel();
timer.purge();
timer = null;
}
if (env != null) { if (env != null) {
env.cleanLog();
env.close(); env.close();
env = null;
} }
} }
class LockExpirer extends TimerTask { class LockExpirer extends TimerTask {
public void run() { public void run() {
if (lockByProject != null) { if (lockByProject != null) {
EntityCursor<Lock> cursor = lockByProject.entities(); for (Lock lock : lockByProject.entities()) {
try { if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
for (Lock lock = cursor.first(); lock != null; lock = cursor.next()) { try {
if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) { releaseLock(null, lock.pid, lock.uid, lock.id);
try { } catch (Exception e) {
releaseLock(null, lock.pid, lock.uid, lock.id); logger.error("Exception while expiring lock for project '" + lock.pid + "'", e);
} catch (Exception e) {
logger.error("Exception while expiring lock for project '" + lock.pid + "'", e);
}
} }
} }
} finally { }
cursor.close();
}
} }
} }
} }

18
build.xml
View File

@ -118,6 +118,11 @@
<include name="**/http*.jar" /> <include name="**/http*.jar" />
<include name="**/slf4j-api*.jar" /> <include name="**/slf4j-api*.jar" />
</fileset> </fileset>
</path>
<path id="broker.core.class.path">
<pathelement location="${broker.core.dir}/module/MOD-INF/classes"/>
<path refid="broker.class.path"/>
<fileset dir="${broker.core.dir}/module/MOD-INF/lib"> <fileset dir="${broker.core.dir}/module/MOD-INF/lib">
<include name="**/*.jar" /> <include name="**/*.jar" />
</fileset> </fileset>
@ -173,6 +178,14 @@
</javac> </javac>
</target> </target>
<target name="build_broker_local" depends="build_broker_core">
<mkdir dir="${broker.core.dir}/module/MOD-INF/classes" />
<javac destdir="${broker.core.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no">
<src path="${broker.core.dir}/src/"/>
<classpath refid="broker.core.class.path" />
</javac>
</target>
<target name="build_broker_appengine" depends="build_broker_core"> <target name="build_broker_appengine" depends="build_broker_core">
<mkdir dir="${broker.appengine.dir}/module/MOD-INF/classes" /> <mkdir dir="${broker.appengine.dir}/module/MOD-INF/classes" />
<javac destdir="${broker.appengine.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no"> <javac destdir="${broker.appengine.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no">
@ -181,7 +194,7 @@
</javac> </javac>
</target> </target>
<target name="build_broker" depends="build_broker_core, build_broker_appengine"/> <target name="build_broker" depends="build_broker_local, build_broker_appengine"/>
<!-- ================================================================== --> <!-- ================================================================== -->
@ -438,7 +451,7 @@
<!-- ================================================================== --> <!-- ================================================================== -->
<target name="prepare_broker" depends="jar_server, build_broker_core"> <target name="prepare_broker" depends="jar_server, build_broker_local">
<copy todir="${broker.core.dir}/WEB-INF/lib"> <copy todir="${broker.core.dir}/WEB-INF/lib">
<fileset dir="${webapp.lib.dir}"> <fileset dir="${webapp.lib.dir}">
<include name="**/butterfly*.jar" /> <include name="**/butterfly*.jar" />
@ -524,7 +537,6 @@
<delete dir="${webapp.classes.dir}" /> <delete dir="${webapp.classes.dir}" />
<delete dir="${server.tests.classes.dir}" /> <delete dir="${server.tests.classes.dir}" />
<delete dir="${broker.core.dir}/module/MOD-INF/classes" /> <delete dir="${broker.core.dir}/module/MOD-INF/classes" />
<delete dir="${broker.core.dir}/WEB-INF/lib" />
<delete dir="${broker.appengine.dir}/module/MOD-INF/classes" /> <delete dir="${broker.appengine.dir}/module/MOD-INF/classes" />
</target> </target>

5
gridworks
View File

@ -585,7 +585,7 @@ run() {
fi fi
if [ -d $GRIDWORKS_CLASSES_DIR ]; then if [ -d $GRIDWORKS_CLASSES_DIR ]; then
add_option "-Dgridworks.autoreload=true -Dbutterfly.autoreload=true" add_option "-Dgridworks.autoreloading=true"
fi fi
if $DARWIN ; then if $DARWIN ; then
@ -650,8 +650,7 @@ broker_run() {
fi fi
if [ -d $GRIDWORKS_CLASSES_DIR ]; then if [ -d $GRIDWORKS_CLASSES_DIR ]; then
add_option "-Dgridworks.autoreload=true -Dbutterfly.autoreload=true" add_option "-Dgridworks.autoreloading=true"
add_option "-Dgridworks.development=true"
fi fi
add_option "-Dgridworks.webapp=broker/core" add_option "-Dgridworks.webapp=broker/core"

18
server/src/com/metaweb/gridworks/Gridworks.java
View File

@ -156,7 +156,7 @@ class GridworksServer extends Server {
this.setSendServerVersion(true); this.setSendServerVersion(true);
// Enable context autoreloading // Enable context autoreloading
if (Configurations.getBoolean("gridworks.autoreload",false)) { if (Configurations.getBoolean("gridworks.autoreloading",false)) {
scanForUpdates(webapp, context); scanForUpdates(webapp, context);
} }
@ -191,7 +191,6 @@ class GridworksServer extends Server {
scanList.add(new File(contextRoot, "WEB-INF/web.xml")); scanList.add(new File(contextRoot, "WEB-INF/web.xml"));
findFiles(".class", new File(contextRoot, "WEB-INF/classes"), scanList); findFiles(".class", new File(contextRoot, "WEB-INF/classes"), scanList);
findFiles(".jar", new File(contextRoot, "WEB-INF/lib"), scanList);
logger.info("Starting autoreloading scanner... "); logger.info("Starting autoreloading scanner... ");
@ -238,19 +237,8 @@ class GridworksServer extends Server {
// parameters if we set them in the webapp context upon reading the web.xml file // parameters if we set them in the webapp context upon reading the web.xml file
static private void configure(WebAppContext context) throws Exception { static private void configure(WebAppContext context) throws Exception {
ServletHolder servlet = context.getServletHandler().getServlet("gridworks"); ServletHolder servlet = context.getServletHandler().getServlet("gridworks");
if (servlet != null) { servlet.setInitParameter("gridworks.data", getDataDir());
servlet.setInitParameter("gridworks.data", getDataDir()); servlet.doStart();
servlet.setInitOrder(1);
servlet.doStart();
}
servlet = context.getServletHandler().getServlet("gridworks-broker");
if (servlet != null) {
servlet.setInitParameter("gridworks.data", getDataDir() + "/broker");
servlet.setInitParameter("gridworks.development", Configurations.get("gridworks.development","false"));
servlet.setInitOrder(1);
servlet.doStart();
}
} }
static private String getDataDir() { static private String getDataDir() {