s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

more work on the broker

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1036 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Stefano Mazzocchi 2010-06-25 07:03:41 +00:00
parent 7fa9d1d812
commit 3c09b06393
6 changed files with 65 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
broker/core/WEB-INF/web.xml
View File

@ -7,13 +7,12 @@
<web-app>
<servlet>
<servlet-name>Butterfly</servlet-name>
<servlet-name>gridworks-broker</servlet-name>
<servlet-class>edu.mit.simile.butterfly.Butterfly</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
<servlet-name>Butterfly</servlet-name>
<servlet-name>gridworks-broker</servlet-name>
<url-pattern>/*</url-pattern>
</servlet-mapping>

21
broker/core/src/com/metaweb/gridworks/broker/GridworksBroker.java
View File

@ -67,12 +67,16 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
protected HttpClient httpclient;
protected boolean developmentMode;
@Override
public void init(ServletConfig config) throws Exception {
super.init(config);
httpclient = getHttpClient();
developmentMode = Boolean.parseBoolean(config.getInitParameter("gridworks.development"));
if (developmentMode) logger.warn("Running in development mode");
}
@Override
public void destroy() throws Exception {
httpclient.getConnectionManager().shutdown();
@ -91,14 +95,14 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
try {
String uid = getUserId(request);
logger.debug("uid: {}", path);
logger.debug("uid: {}", uid);
String pid = getParameter(request, "pid");
logger.debug("pid: {}", path);
logger.debug("pid: {}", pid);
// NOTE: conditionals should be ordered by call frequency estimate to (slightly) improve performance
// we could be using a hashtable and some classes that implement the commands, but the complexity overhead
// doesn't seem to justify the marginal benefit.
if ("get_lock".equals(path)) {
getLock(response, pid);
} else if ("expire_locks".equals(path)) {
@ -159,6 +163,11 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
@SuppressWarnings("unchecked")
protected String getUserId(HttpServletRequest request) throws Exception {
// This is useful for testing
if (developmentMode) {
return getParameter(request, "uid");
}
String oauth = request.getHeader(DELEGATED_OAUTH_HEADER);
if (oauth == null) {
throw new RuntimeException("The request needs to contain the '" + DELEGATED_OAUTH_HEADER + "' header set to obtain user identity via Freebase.");
@ -188,7 +197,7 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
static protected String getParameter(HttpServletRequest request, String name) throws ServletException {
String param = request.getParameter(name);
if (param == null) {
throw new ServletException("request must come with a '" + name + "' parameter");
throw new RuntimeException("request must come with a '" + name + "' parameter");
}
return param;
}
@ -206,7 +215,7 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
static protected int getInteger(HttpServletRequest request, String name) throws ServletException, JSONException {
return Integer.parseInt(getParameter(request, name));
}
static protected void respondError(HttpServletResponse response, String error) throws IOException, ServletException {
if (response == null) {

38
broker/core/src/com/metaweb/gridworks/broker/GridworksBrokerImpl.java
View File

@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.Transaction;
import com.sleepycat.persist.EntityCursor;
import com.sleepycat.persist.EntityStore;
import com.sleepycat.persist.PrimaryIndex;
import com.sleepycat.persist.StoreConfig;
@ -40,7 +41,7 @@ public class GridworksBrokerImpl extends GridworksBroker {
Timer timer;
LockExpirer expirer;
@Override
public void init(ServletConfig config) throws Exception {
super.init(config);
@ -49,9 +50,11 @@ public class GridworksBrokerImpl extends GridworksBroker {
expirer = new LockExpirer();
timer.schedule(expirer, LOCK_EXPIRATION_CHECK_DELAY, LOCK_EXPIRATION_CHECK_DELAY);
File dataPath = new File("data"); // FIXME: data should be configurable;
String dataDir = config.getInitParameter("gridworks.data");
if (dataDir == null) dataDir = "data";
File dataPath = new File(dataDir);
if (!dataPath.exists()) dataPath.mkdirs();
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setAllowCreate(true);
envConfig.setTransactional(true);
@ -73,30 +76,43 @@ public class GridworksBrokerImpl extends GridworksBroker {
if (projectStore != null) {
projectStore.close();
projectById = null;
}
if (lockStore != null) {
lockStore.close();
lockByProject = null;
}
if (timer != null) {
timer.cancel();
timer.purge();
timer = null;
}
if (env != null) {
env.cleanLog();
env.close();
env = null;
}
}
class LockExpirer extends TimerTask {
public void run() {
if (lockByProject != null) {
for (Lock lock : lockByProject.entities()) {
if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
try {
releaseLock(null, lock.pid, lock.uid, lock.id);
} catch (Exception e) {
logger.error("Exception while expiring lock for project '" + lock.pid + "'", e);
EntityCursor<Lock> cursor = lockByProject.entities();
try {
for (Lock lock = cursor.first(); lock != null; lock = cursor.next()) {
if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
try {
releaseLock(null, lock.pid, lock.uid, lock.id);
} catch (Exception e) {
logger.error("Exception while expiring lock for project '" + lock.pid + "'", e);
}
}
}
}
} finally {
cursor.close();
}
}
}
}

18
build.xml
View File

@ -118,11 +118,6 @@
<include name="**/http*.jar" />
<include name="**/slf4j-api*.jar" />
</fileset>
</path>
<path id="broker.core.class.path">
<pathelement location="${broker.core.dir}/module/MOD-INF/classes"/>
<path refid="broker.class.path"/>
<fileset dir="${broker.core.dir}/module/MOD-INF/lib">
<include name="**/*.jar" />
</fileset>
@ -178,14 +173,6 @@
</javac>
</target>
<target name="build_broker_local" depends="build_broker_core">
<mkdir dir="${broker.core.dir}/module/MOD-INF/classes" />
<javac destdir="${broker.core.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no">
<src path="${broker.core.dir}/src/"/>
<classpath refid="broker.core.class.path" />
</javac>
</target>
<target name="build_broker_appengine" depends="build_broker_core">
<mkdir dir="${broker.appengine.dir}/module/MOD-INF/classes" />
<javac destdir="${broker.appengine.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no">
@ -194,7 +181,7 @@
</javac>
</target>
<target name="build_broker" depends="build_broker_local, build_broker_appengine"/>
<target name="build_broker" depends="build_broker_core, build_broker_appengine"/>
<!-- ================================================================== -->
@ -451,7 +438,7 @@
<!-- ================================================================== -->
<target name="prepare_broker" depends="jar_server, build_broker_local">
<target name="prepare_broker" depends="jar_server, build_broker_core">
<copy todir="${broker.core.dir}/WEB-INF/lib">
<fileset dir="${webapp.lib.dir}">
<include name="**/butterfly*.jar" />
@ -537,6 +524,7 @@
<delete dir="${webapp.classes.dir}" />
<delete dir="${server.tests.classes.dir}" />
<delete dir="${broker.core.dir}/module/MOD-INF/classes" />
<delete dir="${broker.core.dir}/WEB-INF/lib" />
<delete dir="${broker.appengine.dir}/module/MOD-INF/classes" />
</target>

5
gridworks
View File

@ -585,7 +585,7 @@ run() {
fi
if [ -d $GRIDWORKS_CLASSES_DIR ]; then
add_option "-Dgridworks.autoreloading=true"
add_option "-Dgridworks.autoreload=true -Dbutterfly.autoreload=true"
fi
if $DARWIN ; then
@ -650,7 +650,8 @@ broker_run() {
fi
if [ -d $GRIDWORKS_CLASSES_DIR ]; then
add_option "-Dgridworks.autoreloading=true"
add_option "-Dgridworks.autoreload=true -Dbutterfly.autoreload=true"
add_option "-Dgridworks.development=true"
fi
add_option "-Dgridworks.webapp=broker/core"

18
server/src/com/metaweb/gridworks/Gridworks.java
View File

@ -156,7 +156,7 @@ class GridworksServer extends Server {
this.setSendServerVersion(true);
// Enable context autoreloading
if (Configurations.getBoolean("gridworks.autoreloading",false)) {
if (Configurations.getBoolean("gridworks.autoreload",false)) {
scanForUpdates(webapp, context);
}
@ -191,6 +191,7 @@ class GridworksServer extends Server {
scanList.add(new File(contextRoot, "WEB-INF/web.xml"));
findFiles(".class", new File(contextRoot, "WEB-INF/classes"), scanList);
findFiles(".jar", new File(contextRoot, "WEB-INF/lib"), scanList);
logger.info("Starting autoreloading scanner... ");
@ -237,8 +238,19 @@ class GridworksServer extends Server {
// parameters if we set them in the webapp context upon reading the web.xml file
static private void configure(WebAppContext context) throws Exception {
ServletHolder servlet = context.getServletHandler().getServlet("gridworks");
servlet.setInitParameter("gridworks.data", getDataDir());
servlet.doStart();
if (servlet != null) {
servlet.setInitParameter("gridworks.data", getDataDir());
servlet.setInitOrder(1);
servlet.doStart();
}
servlet = context.getServletHandler().getServlet("gridworks-broker");
if (servlet != null) {
servlet.setInitParameter("gridworks.data", getDataDir() + "/broker");
servlet.setInitParameter("gridworks.development", Configurations.get("gridworks.development","false"));
servlet.setInitOrder(1);
servlet.doStart();
}
}
static private String getDataDir() {