updating to the latest butterfly (and remove the version number that is a pain to maintain for now)

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1034 7d457c2a-affb-35e4-300a-418c747d4874

broker/appengine/.classpath

@@ -2,8 +2,8 @@
 <classpath>
 	<classpathentry kind="src" path="src"/>
 	<classpathentry kind="con" path="com.google.appengine.eclipse.core.GAE_CONTAINER"/>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
-	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk-r25.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk-r25-sources.jar"/>
+	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk.jar"/>	
 	<classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/>

broker/core/.classpath

@@ -1,9 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
 	<classpathentry kind="src" path="src"/>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
-	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk-r25.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk-r25-sources.jar"/>
 	<classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/>
+	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/butterfly-trunk.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/butterfly-trunk.jar"/>	
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
 	<classpathentry kind="lib" path="module/MOD-INF/lib/bdb-je-4.0.103.jar" sourcepath="module/MOD-INF/lib-src/bdb-je-4.0.103-sources.jar"/>
 	<classpathentry kind="lib" path="/gridworks/webapp/WEB-INF/lib/httpclient-4.0.1.jar" sourcepath="/gridworks/webapp/WEB-INF/lib-src/httpclient-4.0.1-sources.jar"/>

broker/core/WEB-INF/web.xml

@@ -7,13 +7,12 @@
 <web-app>
      
     <servlet>
-        <servlet-name>Butterfly</servlet-name>
+        <servlet-name>gridworks-broker</servlet-name>
         <servlet-class>edu.mit.simile.butterfly.Butterfly</servlet-class>
-        <load-on-startup>1</load-on-startup>
     </servlet>
 
     <servlet-mapping>
-        <servlet-name>Butterfly</servlet-name>
+        <servlet-name>gridworks-broker</servlet-name>
         <url-pattern>/*</url-pattern>
     </servlet-mapping>
         

broker/core/src/com/metaweb/gridworks/broker/GridworksBroker.java

@@ -67,10 +67,14 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
     
     protected HttpClient httpclient;
 
+    protected boolean developmentMode;
+    
     @Override
     public void init(ServletConfig config) throws Exception {
         super.init(config);
         httpclient = getHttpClient();
+        developmentMode = Boolean.parseBoolean(config.getInitParameter("gridworks.development"));
+        if (developmentMode) logger.warn("Running in development mode");
     }
 
     @Override
@@ -91,9 +95,9 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
         
         try {
             String uid = getUserId(request);
-            logger.debug("uid: {}", path);
+            logger.debug("uid: {}", uid);
             String pid = getParameter(request, "pid");
-            logger.debug("pid: {}", path);
+            logger.debug("pid: {}", pid);
             
             // NOTE: conditionals should be ordered by call frequency estimate to (slightly) improve performance
             // we could be using a hashtable and some classes that implement the commands, but the complexity overhead
@@ -159,6 +163,11 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
     @SuppressWarnings("unchecked")
     protected String getUserId(HttpServletRequest request) throws Exception {
 
+        // This is useful for testing
+        if (developmentMode) {
+            return getParameter(request, "uid");
+        }
+        
         String oauth = request.getHeader(DELEGATED_OAUTH_HEADER);
         if (oauth == null) {
             throw new RuntimeException("The request needs to contain the '" + DELEGATED_OAUTH_HEADER + "' header set to obtain user identity via Freebase.");
@@ -188,7 +197,7 @@ public abstract class GridworksBroker extends ButterflyModuleImpl {
     static protected String getParameter(HttpServletRequest request, String name) throws ServletException {
         String param = request.getParameter(name);
         if (param == null) {
-            throw new ServletException("request must come with a '" + name + "' parameter");
+            throw new RuntimeException("request must come with a '" + name + "' parameter");
         }
         return param;
     }

broker/core/src/com/metaweb/gridworks/broker/GridworksBrokerImpl.java

@@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory;
 import com.sleepycat.je.Environment;
 import com.sleepycat.je.EnvironmentConfig;
 import com.sleepycat.je.Transaction;
+import com.sleepycat.persist.EntityCursor;
 import com.sleepycat.persist.EntityStore;
 import com.sleepycat.persist.PrimaryIndex;
 import com.sleepycat.persist.StoreConfig;
@@ -49,7 +50,9 @@ public class GridworksBrokerImpl extends GridworksBroker {
         expirer = new LockExpirer();
         timer.schedule(expirer, LOCK_EXPIRATION_CHECK_DELAY, LOCK_EXPIRATION_CHECK_DELAY);
         
-        File dataPath = new File("data"); // FIXME: data should be configurable;
+        String dataDir = config.getInitParameter("gridworks.data");
+        if (dataDir == null) dataDir = "data";
+        File dataPath = new File(dataDir);
         if (!dataPath.exists()) dataPath.mkdirs();
         
         EnvironmentConfig envConfig = new EnvironmentConfig();
@@ -73,22 +76,32 @@ public class GridworksBrokerImpl extends GridworksBroker {
         
         if (projectStore != null) {
             projectStore.close();
+            projectById = null;            
         } 
 
         if (lockStore != null) {
             lockStore.close();
+            lockByProject = null;
+        } 
+        
+        if (timer != null) {
+            timer.cancel();
+            timer.purge();
+            timer = null;
         }
         
         if (env != null) {
-            env.cleanLog();
             env.close();
+            env = null;
         }
     }
 
     class LockExpirer extends TimerTask {
         public void run() {
             if (lockByProject != null) {
-                for (Lock lock : lockByProject.entities()) {
+                EntityCursor<Lock> cursor = lockByProject.entities();
+                try {
+                    for (Lock lock = cursor.first(); lock != null; lock = cursor.next()) {
                         if (lock.timestamp + LOCK_DURATION < System.currentTimeMillis()) {
                             try {
                                 releaseLock(null, lock.pid, lock.uid, lock.id);
@@ -97,6 +110,9 @@ public class GridworksBrokerImpl extends GridworksBroker {
                             }
                         }
                     }
+                } finally {
+                    cursor.close();
+                }                
             }
         }
     }

build.xml

@@ -118,11 +118,6 @@
             <include name="**/http*.jar" />
             <include name="**/slf4j-api*.jar" />
         </fileset>
-    </path>
-
-    <path id="broker.core.class.path">
-        <pathelement location="${broker.core.dir}/module/MOD-INF/classes"/>
-        <path refid="broker.class.path"/>
         <fileset dir="${broker.core.dir}/module/MOD-INF/lib">
             <include name="**/*.jar" />
         </fileset>
@@ -178,14 +173,6 @@
         </javac>
     </target>
 
-    <target name="build_broker_local" depends="build_broker_core">
-        <mkdir dir="${broker.core.dir}/module/MOD-INF/classes" />
-        <javac destdir="${broker.core.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no">
-            <src path="${broker.core.dir}/src/"/>
-            <classpath refid="broker.core.class.path" />
-        </javac>
-    </target>
-
     <target name="build_broker_appengine" depends="build_broker_core">
         <mkdir dir="${broker.appengine.dir}/module/MOD-INF/classes" />
         <javac destdir="${broker.appengine.dir}/module/MOD-INF/classes" debug="true" includeAntRuntime="no">
@@ -194,7 +181,7 @@
         </javac>
     </target>
 
-    <target name="build_broker" depends="build_broker_local, build_broker_appengine"/>
+    <target name="build_broker" depends="build_broker_core, build_broker_appengine"/>
 
     <!-- ================================================================== -->
 
@@ -451,7 +438,7 @@
 
     <!-- ================================================================== -->
 
-    <target name="prepare_broker" depends="jar_server, build_broker_local">
+    <target name="prepare_broker" depends="jar_server, build_broker_core">
         <copy todir="${broker.core.dir}/WEB-INF/lib">
             <fileset dir="${webapp.lib.dir}">
                 <include name="**/butterfly*.jar" />
@@ -537,6 +524,7 @@
         <delete dir="${webapp.classes.dir}" />
         <delete dir="${server.tests.classes.dir}" />
         <delete dir="${broker.core.dir}/module/MOD-INF/classes" />
+        <delete dir="${broker.core.dir}/WEB-INF/lib" />
         <delete dir="${broker.appengine.dir}/module/MOD-INF/classes" />
     </target>
 

gridworks

@@ -585,7 +585,7 @@ run() {
     fi
             
     if [ -d $GRIDWORKS_CLASSES_DIR ]; then
-        add_option "-Dgridworks.autoreloading=true"
+        add_option "-Dgridworks.autoreload=true -Dbutterfly.autoreload=true"
     fi
 
     if $DARWIN ; then
@@ -650,7 +650,8 @@ broker_run() {
     fi
 
     if [ -d $GRIDWORKS_CLASSES_DIR ]; then
-        add_option "-Dgridworks.autoreloading=true"
+        add_option "-Dgridworks.autoreload=true -Dbutterfly.autoreload=true"
+        add_option "-Dgridworks.development=true"
     fi
                     
     add_option "-Dgridworks.webapp=broker/core"

main/.classpath

@@ -3,6 +3,7 @@
 	<classpathentry kind="src" path="src"/>
 	<classpathentry kind="src" path="tests/server/src"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+	<classpathentry kind="lib" path="webapp/WEB-INF/lib/butterfly-trunk.jar" sourcepath="webapp/WEB-INF/lib-src/butterfly-trunk.jar"/>
 	<classpathentry kind="lib" path="webapp/WEB-INF/lib/commons-codec-1.4.jar" sourcepath="webapp/WEB-INF/lib-src/commons-codec-1.4-sources.jar"/>
 	<classpathentry kind="lib" path="webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/>
 	<classpathentry kind="lib" path="webapp/WEB-INF/lib/commons-fileupload-1.2.1.jar" sourcepath="webapp/WEB-INF/lib-src/commons-fileupload-1.2.1-sources.jar"/>
@@ -38,6 +39,5 @@
 	<classpathentry kind="lib" path="tests/server/lib/mockito-all-1.8.4.jar" sourcepath="tests/server/lib-src/mockito-all-1.8.4-sources.jar"/>
 	<classpathentry kind="lib" path="tests/server/lib/testng-5.12.1.jar" sourcepath="tests/server/lib-src/testng-5.12.1-sources.jar"/>
 	<classpathentry kind="lib" path="/gridworks-server/lib/servlet-api-2.5.jar" sourcepath="/gridworks-server/lib-src/servlet-api-2.5-sources.jar"/>
-	<classpathentry kind="lib" path="webapp/WEB-INF/lib/butterfly-trunk-r26.jar"/>
 	<classpathentry kind="output" path="webapp/WEB-INF/classes"/>
 </classpath>

server/src/com/metaweb/gridworks/Gridworks.java

@@ -156,7 +156,7 @@ class GridworksServer extends Server {
         this.setSendServerVersion(true);
 
         // Enable context autoreloading
-        if (Configurations.getBoolean("gridworks.autoreloading",false)) {
+        if (Configurations.getBoolean("gridworks.autoreload",false)) {
             scanForUpdates(webapp, context);
         }
         
@@ -191,6 +191,7 @@ class GridworksServer extends Server {
 
         scanList.add(new File(contextRoot, "WEB-INF/web.xml"));
         findFiles(".class", new File(contextRoot, "WEB-INF/classes"), scanList);
+        findFiles(".jar", new File(contextRoot, "WEB-INF/lib"), scanList);
 
         logger.info("Starting autoreloading scanner... ");
 
@@ -237,10 +238,21 @@ class GridworksServer extends Server {
     // parameters if we set them in the webapp context upon reading the web.xml file    
     static private void configure(WebAppContext context) throws Exception {
         ServletHolder servlet = context.getServletHandler().getServlet("gridworks");
+        if (servlet != null) {
             servlet.setInitParameter("gridworks.data", getDataDir());
+            servlet.setInitOrder(1);
             servlet.doStart();
         }
 
+        servlet = context.getServletHandler().getServlet("gridworks-broker");
+        if (servlet != null) {
+            servlet.setInitParameter("gridworks.data", getDataDir() + "/broker");
+            servlet.setInitParameter("gridworks.development", Configurations.get("gridworks.development","false"));
+            servlet.setInitOrder(1);
+            servlet.doStart();
+        }
+    }
+
     static private String getDataDir() {
         
         String data_dir = Configurations.get("gridworks.data_dir");