s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

updating to the latest butterfly (and remove the version number that is a pain to maintain for now)

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1034 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Stefano Mazzocchi 2010-06-25 07:03:16 +00:00
parent 777e4ce0e8
commit 5193630b9a
11 changed files with 70 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
broker/appengine/.classpath
View File

@ -2,8 +2,8 @@
<classpath> <classpath>
<classpathentry kind="src" path="src"/> <classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="com.google.appengine.eclipse.core.GAE_CONTAINER"/> <classpathentry kind="con" path="com.google.appengine.eclipse.core.GAE_CONTAINER"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk-r25.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk-r25-sources.jar"/> <classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk.jar"/>
<classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/> <classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/>
<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/> <classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/> <classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/>

4
broker/core/.classpath
View File

@ -1,9 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<classpath> <classpath>
<classpathentry kind="src" path="src"/> <classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk-r25.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk-r25-sources.jar"/>
<classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/> <classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/>
<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk.jar"/>
<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/> <classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/bdb-je-4.0.103.jar" sourcepath="module/MOD-INF/lib-src/bdb-je-4.0.103-sources.jar"/> <classpathentry kind="lib" path="module/MOD-INF/lib/bdb-je-4.0.103.jar" sourcepath="module/MOD-INF/lib-src/bdb-je-4.0.103-sources.jar"/>
<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/> <classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/>

5
broker/core/WEB-INF/web.xml
View File

@ -7,13 +7,12 @@
<web-app> <web-app>
<servlet> <servlet>
<servlet-name>Butterfly</servlet-name> <servlet-name>gridworks-broker</servlet-name>
<servlet-class>edu.mit.simile.butterfly.Butterfly</servlet-class> <servlet-class>edu.mit.simile.butterfly.Butterfly</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet> </servlet>
<servlet-mapping> <servlet-mapping>
<servlet-name>Butterfly</servlet-name> <servlet-name>gridworks-broker</servlet-name>
<url-pattern>/*</url-pattern> <url-pattern>/*</url-pattern>
</servlet-mapping> </servlet-mapping>

21
broker/core/src/com/metaweb/gridworks/broker/GridworksBroker.java
View File

@ -67,12 +67,16 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
protected HttpClient httpclient; protected HttpClient httpclient;
protected boolean developmentMode;
@Override @Override
public void init(ServletConfig config) throws Exception { public void init(ServletConfig config) throws Exception {
super.init(config); super.init(config);
httpclient = getHttpClient(); httpclient = getHttpClient();
developmentMode = Boolean.parseBoolean(config.getInitParameter("gridworks.development"));
if (developmentMode) logger.warn("Running in development mode");
} }
@Override @Override
public void destroy() throws Exception { public void destroy() throws Exception {
httpclient.getConnectionManager().shutdown(); httpclient.getConnectionManager().shutdown();
@ -91,14 +95,14 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
try { try {
String uid = getUserId(request); String uid = getUserId(request);
logger.debug("uid: {}", path); logger.debug("uid: {}", uid);
String pid = getParameter(request, "pid"); String pid = getParameter(request, "pid");
logger.debug("pid: {}", path); logger.debug("pid: {}", pid);
// NOTE: conditionals should be ordered by call frequency estimate to (slightly) improve performance // NOTE: conditionals should be ordered by call frequency estimate to (slightly) improve performance
// we could be using a hashtable and some classes that implement the commands, but the complexity overhead // we could be using a hashtable and some classes that implement the commands, but the complexity overhead
// doesn't seem to justify the marginal benefit. // doesn't seem to justify the marginal benefit.
if ("get_lock".equals(path)) { if ("get_lock".equals(path)) {
getLock(response, pid); getLock(response, pid);
} else if ("expire_locks".equals(path)) { } else if ("expire_locks".equals(path)) {
@ -159,6 +163,11 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
protected String getUserId(HttpServletRequest request) throws Exception { protected String getUserId(HttpServletRequest request) throws Exception {
// This is useful for testing
if (developmentMode) {
return getParameter(request, "uid");
}
String oauth = request.getHeader(DELEGATED_OAUTH_HEADER); String oauth = request.getHeader(DELEGATED_OAUTH_HEADER);
if (oauth == null) { if (oauth == null) {
throw new RuntimeException("The request needs to contain the '" + DELEGATED_OAUTH_HEADER + "' header set to obtain user identity via Freebase."); throw new RuntimeException("The request needs to contain the '" + DELEGATED_OAUTH_HEADER + "' header set to obtain user identity via Freebase.");
@ -188,7 +197,7 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
static protected String getParameter(HttpServletRequest request, String name) throws ServletException { static protected String getParameter(HttpServletRequest request, String name) throws ServletException {
String param = request.getParameter(name); String param = request.getParameter(name);
if (param == null) { if (param == null) {
throw new ServletException("request must come with a '" + name + "' parameter"); throw new RuntimeException("request must come with a '" + name + "' parameter");
} }
return param; return param;
} }
@ -206,7 +215,7 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
static protected int getInteger(HttpServletRequest request, String name) throws ServletException, JSONException { static protected int getInteger(HttpServletRequest request, String name) throws ServletException, JSONException {
return Integer.parseInt(getParameter(request, name)); return Integer.parseInt(getParameter(request, name));
} }
static protected void respondError(HttpServletResponse response, String error) throws IOException, ServletException { static protected void respondError(HttpServletResponse response, String error) throws IOException, ServletException {
if (response == null) { if (response == null) {

38
broker/core/src/com/metaweb/gridworks/broker/GridworksBrokerImpl.java
View File

@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory;
import com.sleepycat.je.Environment; import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.Transaction; import com.sleepycat.je.Transaction;
import com.sleepycat.persist.EntityCursor;
import com.sleepycat.persist.EntityStore; import com.sleepycat.persist.EntityStore;
import com.sleepycat.persist.PrimaryIndex; import com.sleepycat.persist.PrimaryIndex;
import com.sleepycat.persist.StoreConfig; import com.sleepycat.persist.StoreConfig;
@ -40,7 +41,7 @@ public class GridworksBrokerImpl extends GridworksBroker {
Timer timer; Timer timer;
LockExpirer expirer; LockExpirer expirer;
@Override @Override
public void init(ServletConfig config) throws Exception { public void init(ServletConfig config) throws Exception {
super.init(config); super.init(config);
@ -49,9 +50,11 @@ public class GridworksBrokerImpl extends GridworksBroker {
expirer = new LockExpirer(); expirer = new LockExpirer();
timer.schedule(expirer, LOCK_EXPIRATION_CHECK_DELAY, LOCK_EXPIRATION_CHECK_DELAY); timer.schedule(expirer, LOCK_EXPIRATION_CHECK_DELAY, LOCK_EXPIRATION_CHECK_DELAY);
File dataPath = new File("data"); // FIXME: data should be configurable; String dataDir = config.getInitParameter("gridworks.data");
if (dataDir == null) dataDir = "data";
File dataPath = new File(dataDir);
if (!dataPath.exists()) dataPath.mkdirs(); if (!dataPath.exists()) dataPath.mkdirs();
EnvironmentConfig envConfig = new EnvironmentConfig(); EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setAllowCreate(true); envConfig.setAllowCreate(true);
envConfig.setTransactional(true); envConfig.setTransactional(true);
@ -73,30 +76,43 @@ public class GridworksBrokerImpl extends GridworksBroker {
if (projectStore != null) { if (projectStore != null) {
projectStore.close(); projectStore.close();
projectById = null;
} }
if (lockStore != null) { if (lockStore != null) {
lockStore.close(); lockStore.close();
lockByProject = null;
} }
if (timer != null) {
timer.cancel();
timer.purge();
timer = null;
}
if (env != null) { if (env != null) {
env.cleanLog();
env.close(); env.close();
env = null;
} }
} }
class LockExpirer extends TimerTask { class LockExpirer extends TimerTask {
public void run() { public void run() {
if (lockByProject != null) { if (lockByProject != null) {
for (Lock lock : lockByProject.entities()) { EntityCursor<Lock> cursor = lockByProject.entities();
if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) { try {
try { for (Lock lock = cursor.first(); lock != null; lock = cursor.next()) {
releaseLock(null, lock.pid, lock.uid, lock.id); if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
} catch (Exception e) { try {
logger.error("Exception while expiring lock for project '" + lock.pid + "'", e); releaseLock(null, lock.pid, lock.uid, lock.id);
} catch (Exception e) {
logger.error("Exception while expiring lock for project '" + lock.pid + "'", e);
}
} }
} }
} } finally {
cursor.close();
}
} }
} }
} }

18
build.xml
View File

@ -118,11 +118,6 @@
<include name="**/http*.jar" /> <include name="**/http*.jar" />
<include name="**/slf4j-api*.jar" /> <include name="**/slf4j-api*.jar" />
</fileset> </fileset>
</path>
<path id="broker.core.class.path">
<pathelement location="${broker.core.dir}/module/MOD-INF/classes"/>
<path refid="broker.class.path"/>
<fileset dir="${broker.core.dir}/module/MOD-INF/lib"> <fileset dir="${broker.core.dir}/module/MOD-INF/lib">
<include name="**/*.jar" /> <include name="**/*.jar" />
</fileset> </fileset>
@ -178,14 +173,6 @@
</javac> </javac>
</target> </target>
<target name="build_broker_local" depends="build_broker_core">
<mkdir dir="${broker.core.dir}/module/MOD-INF/classes" />
<javac destdir="${broker.core.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no">
<src path="${broker.core.dir}/src/"/>
<classpath refid="broker.core.class.path" />
</javac>
</target>
<target name="build_broker_appengine" depends="build_broker_core"> <target name="build_broker_appengine" depends="build_broker_core">
<mkdir dir="${broker.appengine.dir}/module/MOD-INF/classes" /> <mkdir dir="${broker.appengine.dir}/module/MOD-INF/classes" />
<javac destdir="${broker.appengine.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no"> <javac destdir="${broker.appengine.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no">
@ -194,7 +181,7 @@
</javac> </javac>
</target> </target>
<target name="build_broker" depends="build_broker_local, build_broker_appengine"/> <target name="build_broker" depends="build_broker_core, build_broker_appengine"/>
<!-- ================================================================== --> <!-- ================================================================== -->
@ -451,7 +438,7 @@
<!-- ================================================================== --> <!-- ================================================================== -->
<target name="prepare_broker" depends="jar_server, build_broker_local"> <target name="prepare_broker" depends="jar_server, build_broker_core">
<copy todir="${broker.core.dir}/WEB-INF/lib"> <copy todir="${broker.core.dir}/WEB-INF/lib">
<fileset dir="${webapp.lib.dir}"> <fileset dir="${webapp.lib.dir}">
<include name="**/butterfly*.jar" /> <include name="**/butterfly*.jar" />
@ -537,6 +524,7 @@
<delete dir="${webapp.classes.dir}" /> <delete dir="${webapp.classes.dir}" />
<delete dir="${server.tests.classes.dir}" /> <delete dir="${server.tests.classes.dir}" />
<delete dir="${broker.core.dir}/module/MOD-INF/classes" /> <delete dir="${broker.core.dir}/module/MOD-INF/classes" />
<delete dir="${broker.core.dir}/WEB-INF/lib" />
<delete dir="${broker.appengine.dir}/module/MOD-INF/classes" /> <delete dir="${broker.appengine.dir}/module/MOD-INF/classes" />
</target> </target>

5
gridworks
View File

@ -585,7 +585,7 @@ run() {
fi fi
if [ -d $GRIDWORKS_CLASSES_DIR ]; then if [ -d $GRIDWORKS_CLASSES_DIR ]; then
add_option "-Dgridworks.autoreloading=true" add_option "-Dgridworks.autoreload=true -Dbutterfly.autoreload=true"
fi fi
if $DARWIN ; then if $DARWIN ; then
@ -650,7 +650,8 @@ broker_run() {
fi fi
if [ -d $GRIDWORKS_CLASSES_DIR ]; then if [ -d $GRIDWORKS_CLASSES_DIR ]; then
add_option "-Dgridworks.autoreloading=true" add_option "-Dgridworks.autoreload=true -Dbutterfly.autoreload=true"
add_option "-Dgridworks.development=true"
fi fi
add_option "-Dgridworks.webapp=broker/core" add_option "-Dgridworks.webapp=broker/core"

2
main/.classpath
View File

@ -3,6 +3,7 @@
<classpathentry kind="src" path="src"/> <classpathentry kind="src" path="src"/>
<classpathentry kind="src" path="tests/server/src"/> <classpathentry kind="src" path="tests/server/src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/butterfly-trunk.jar" sourcepath="webapp/WEB-INF/lib-src/butterfly-trunk.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/commons-codec-1.4.jar" sourcepath="webapp/WEB-INF/lib-src/commons-codec-1.4-sources.jar"/> <classpathentry kind="lib" path="webapp/WEB-INF/lib/commons-codec-1.4.jar" sourcepath="webapp/WEB-INF/lib-src/commons-codec-1.4-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/> <classpathentry kind="lib" path="webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/commons-fileupload-1.2.1.jar" sourcepath="webapp/WEB-INF/lib-src/commons-fileupload-1.2.1-sources.jar"/> <classpathentry kind="lib" path="webapp/WEB-INF/lib/commons-fileupload-1.2.1.jar" sourcepath="webapp/WEB-INF/lib-src/commons-fileupload-1.2.1-sources.jar"/>
@ -38,6 +39,5 @@
<classpathentry kind="lib" path="tests/server/lib/mockito-all-1.8.4.jar" sourcepath="tests/server/lib-src/mockito-all-1.8.4-sources.jar"/> <classpathentry kind="lib" path="tests/server/lib/mockito-all-1.8.4.jar" sourcepath="tests/server/lib-src/mockito-all-1.8.4-sources.jar"/>
<classpathentry kind="lib" path="tests/server/lib/testng-5.12.1.jar" sourcepath="tests/server/lib-src/testng-5.12.1-sources.jar"/> <classpathentry kind="lib" path="tests/server/lib/testng-5.12.1.jar" sourcepath="tests/server/lib-src/testng-5.12.1-sources.jar"/>
<classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/> <classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/>
<classpathentry kind="lib" path="webapp/WEB-INF/lib/butterfly-trunk-r26.jar"/>
<classpathentry kind="output" path="webapp/WEB-INF/classes"/> <classpathentry kind="output" path="webapp/WEB-INF/classes"/>
</classpath> </classpath>

BIN
main/webapp/WEB-INF/lib-src/butterfly-trunk-r26-sources.jar → main/webapp/WEB-INF/lib-src/butterfly-trunk-sources.jar
View File

Binary file not shown.

BIN
main/webapp/WEB-INF/lib/butterfly-trunk.jar Normal file
View File

Binary file not shown.

18
server/src/com/metaweb/gridworks/Gridworks.java
View File

@ -156,7 +156,7 @@ class GridworksServer extends Server {
this.setSendServerVersion(true); this.setSendServerVersion(true);
// Enable context autoreloading // Enable context autoreloading
if (Configurations.getBoolean("gridworks.autoreloading",false)) { if (Configurations.getBoolean("gridworks.autoreload",false)) {
scanForUpdates(webapp, context); scanForUpdates(webapp, context);
} }
@ -191,6 +191,7 @@ class GridworksServer extends Server {
scanList.add(new File(contextRoot, "WEB-INF/web.xml")); scanList.add(new File(contextRoot, "WEB-INF/web.xml"));
findFiles(".class", new File(contextRoot, "WEB-INF/classes"), scanList); findFiles(".class", new File(contextRoot, "WEB-INF/classes"), scanList);
findFiles(".jar", new File(contextRoot, "WEB-INF/lib"), scanList);
logger.info("Starting autoreloading scanner... "); logger.info("Starting autoreloading scanner... ");
@ -237,8 +238,19 @@ class GridworksServer extends Server {
// parameters if we set them in the webapp context upon reading the web.xml file // parameters if we set them in the webapp context upon reading the web.xml file
static private void configure(WebAppContext context) throws Exception { static private void configure(WebAppContext context) throws Exception {
ServletHolder servlet = context.getServletHandler().getServlet("gridworks"); ServletHolder servlet = context.getServletHandler().getServlet("gridworks");
servlet.setInitParameter("gridworks.data", getDataDir()); if (servlet != null) {
servlet.doStart(); servlet.setInitParameter("gridworks.data", getDataDir());
servlet.setInitOrder(1);
servlet.doStart();
}
servlet = context.getServletHandler().getServlet("gridworks-broker");
if (servlet != null) {
servlet.setInitParameter("gridworks.data", getDataDir() + "/broker");
servlet.setInitParameter("gridworks.development", Configurations.get("gridworks.development","false"));
servlet.setInitOrder(1);
servlet.doStart();
}
} }
static private String getDataDir() { static private String getDataDir() {