adding OS-specific data paths

git-svn-id: http://google-refine.googlecode.com/svn/trunk@173 7d457c2a-affb-35e4-300a-418c747d4874

.classpath

@@ -13,5 +13,6 @@
 	<classpathentry kind="lib" path="lib/cos-20081226.jar" sourcepath="lib-src/cos-20081226-sources.jar"/>
 	<classpathentry kind="lib" path="lib/poi-3.6.jar"/>
 	<classpathentry kind="lib" path="lib/poi-ooxml-3.6.jar"/>
+	<classpathentry kind="lib" path="lib/jdatapath-alpha2.jar" sourcepath="lib-src/jdatapath-alpha2-sources.jar"/>
 	<classpathentry kind="output" path="build/classes"/>
 </classpath>

build.xml

@@ -96,10 +96,10 @@
                dontWrapJar="true"
                icon="${graphics.dir}/icon/gridworks.ico">
             <classPath mainClass="com.metaweb.gridworks.Gridworks">
-                <cp>${lib.dir}/*.jar</cp>
+                <cp>lib/*.jar</cp>
             </classPath>
-            <jre minVersion="1.6.0" jdkPreference="preferJre">
-                <opt>${java.version}</opt>
+            <jre minVersion="1.6.0" jdkPreference="preferJre" initialHeapPercent="10" maxHeapPercent="80">
+                <opt>-Djava.library.path=lib/native/windows</opt>
             </jre>
             <versionInfo
                 fileVersion="${num_version}.0.0"
@@ -120,6 +120,11 @@
                 <include name="**/*.jar"/>
             </fileset>
         </copy>
+        <copy todir="${exe.dir}/lib/native/windows">
+            <fileset dir="${lib.dir}/native/windows">
+                <include name="*.dll"/>
+            </fileset>
+        </copy>
         <copy file="${build.dir}/gridworks.jar" tofile="${exe.dir}/lib/gridworks-${version}.jar"/>
         <copy todir="${exe.dir}/webapp">
             <fileset dir="${basedir}/src/main/webapp">

gridworks

@@ -32,10 +32,13 @@ where [options] include:
 
   -w <path> path to the webapp
      default: src/main/webapp
-     
-  -d enable JVM debugging (on port 8000)
+
+  -d <path> path to the data directory
+     default: OS dependent
+          
+  --debug enable JVM debugging (on port 8000)
        
-  -x enable JMX monitoring (for jconsole)
+  --jmx enable JMX monitoring (for jconsole)
   
 and <action> is one of
 
@@ -89,7 +92,8 @@ ant_prepare() {
 }
 
 ant() {
-    ant_prepare    
+    ant_prepare   
+    echo $NUM_VERSION 
     $ANT -f build.xml -Dbuild.dir="$GRIDWORKS_BUILD_DIR" -Ddist.dir="$GRIDWORKS_DIST_DIR" -Dversion="$VERSION" -Dnum_version="$NUM_VERSION" -Djava.options="$JAVA_OPTIONS" $1 || exit 1
 }
 
@@ -114,7 +118,7 @@ get_version() {
         fail "Must specify a version number"
     fi
     
-    NUM_VERSION=`echo $VERSION | sed 's/\a//g'`
+    NUM_VERSION=`echo $VERSION | sed 's/[a-zA-Z]//g'`
 }
     
 make_exe() {
@@ -182,10 +186,18 @@ run() {
         ant build
         echo ""
     fi
+
+    if [ "$SYSTEM" == 'Darwin' ] ; then
+        OPTS="-Xdock:name=Gridworks -Xdock:icon=src/graphics/icon/gridworks.icns $OPTS"
+    fi
+    
+    if [ "$GRIDWORKS_DATA_DIR" != "" ] ; then
+        OPTS="-Dgridworks.data_dir=$GRIDWORKS_DATA_DIR $OPTS"
+    fi
     
     CLASSPATH="$GRIDWORKS_BUILD_DIR/classes:$GRIDWORKS_LIB_DIR/*"
 
-    RUN_CMD="$JAVA -cp $CLASSPATH $OPTS -Xdock:name=Gridworks -Xdock:icon=src/graphics/icon/gridworks.icns com.metaweb.gridworks.Gridworks"
+    RUN_CMD="$JAVA -cp $CLASSPATH $OPTS com.metaweb.gridworks.Gridworks"
   
     echo "Starting Gridworks at 'http://${GRIDWORKS_HOST}:${GRIDWORKS_PORT}/'"
     echo ""
@@ -230,8 +242,9 @@ while [ $# -ne 0 ] ; do
     -p) shift; GRIDWORKS_PORT="$1"; shift; continue;;
     -i) shift; GRIDWORKS_HOST="$1"; shift; continue;;
     -w) shift; GRIDWORKS_WEBAPP="$1"; shift; continue;;
-    -d) shift; add_option '-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n'; continue;;
-    -x) shift; add_option '-Dcom.sun.management.jmxremote'; continue;;
+    -d) shift; GRIDWORKS_DATA_DIR="$1"; shift; continue;;
+    --debug) shift; add_option '-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n'; continue;;
+    --jmx) shift; add_option '-Dcom.sun.management.jmxremote'; continue;;
     -*) fail "Invalid option: $1";;
     *) break;;
   esac

gridworks.bat

@@ -140,7 +140,7 @@ goto usage
 
 :doRun
 set CLASSPATH="%GRIDWORKS_BUILD_DIR%\classes;%GRIDWORKS_LIB_DIR%\*"
-"%JAVA_HOME%\bin\java.exe" -cp %CLASSPATH% %OPTS% com.metaweb.gridworks.Gridworks
+"%JAVA_HOME%\bin\java.exe" -cp %CLASSPATH% %OPTS% -Djava.library.path=lib/native/windows com.metaweb.gridworks.Gridworks
 goto end
 
 :doAnt

src/main/java/com/metaweb/gridworks/Gridworks.java

@@ -28,6 +28,9 @@ import com.metaweb.util.threads.ThreadPoolExecutorAdapter;
 
 public class Gridworks extends Server {
 
+    private static Logger root = Logger.getRootLogger();
+    private static Logger logger = Logger.getLogger("com.metaweb.gridworks");
+    
     public static void main(String[] args) throws Exception  {
         
         // tell jetty to use SLF4J for logging instead of its own stuff
@@ -36,14 +39,12 @@ public class Gridworks extends Server {
 
         // initialize the log4j system
         Appender console = new ConsoleAppender(new IndentingLayout());
-        
-        Logger root = Logger.getRootLogger();
         root.setLevel(Level.ALL);
         root.addAppender(console);
 
         Logger jetty_logger = Logger.getLogger("org.mortbay.log");
-        jetty_logger.setLevel(Level.INFO);
-
+        jetty_logger.setLevel(Level.WARN);
+        
         // get main configurations
         int port = Configurations.getInteger("gridworks.port",3333);
         String host = Configurations.get("gridworks.host","127.0.0.1");
@@ -65,6 +66,10 @@ public class Gridworks extends Server {
         server.join();
     }
 
+    public static void log(String message) {
+        logger.info(message);
+    }
+    
     /* -------------- Gridworks HTTP server ----------------- */
     
     private ThreadPoolExecutor threadPool;
@@ -80,15 +85,6 @@ public class Gridworks extends Server {
         threadPool = new ThreadPoolExecutor(maxThreads, maxQueue, keepAliveTime, TimeUnit.SECONDS, queue);
 
         this.setThreadPool(new ThreadPoolExecutorAdapter(threadPool));
-
-        // NOTE(SM): we use a BIO connector instead of the NIO connector
-        // because NIO can't deal with thread deaths: if the execution
-        // of a script is taking too long and acre kills it, that automatically
-        // shuts down the I/O channel to the user and we can't communicate
-        // that error back to the user (which also causes the I/O channel
-        // to be abruptly terminated and causes all sorts of proxy errors
-        // down the line). This is a performance penalty, but not severe
-        // enough to compensate for this problem.
         
         Connector connector = new SocketConnector();
         connector.setPort(port);

src/main/java/com/metaweb/gridworks/GridworksServlet.java

@@ -1,6 +1,5 @@
 package com.metaweb.gridworks;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
@@ -10,10 +9,6 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONTokener;
-
 import com.metaweb.gridworks.commands.Command;
 import com.metaweb.gridworks.commands.edit.AddColumnCommand;
 import com.metaweb.gridworks.commands.edit.AnnotateOneRowCommand;
@@ -30,17 +25,17 @@ import com.metaweb.gridworks.commands.edit.UndoRedoCommand;
 import com.metaweb.gridworks.commands.info.ComputeFacetsCommand;
 import com.metaweb.gridworks.commands.info.ExportRowsCommand;
 import com.metaweb.gridworks.commands.info.GetAllProjectMetadataCommand;
-import com.metaweb.gridworks.commands.info.GetModelsCommand;
 import com.metaweb.gridworks.commands.info.GetHistoryCommand;
+import com.metaweb.gridworks.commands.info.GetModelsCommand;
 import com.metaweb.gridworks.commands.info.GetOperationsCommand;
 import com.metaweb.gridworks.commands.info.GetProcessesCommand;
 import com.metaweb.gridworks.commands.info.GetProjectMetadataCommand;
 import com.metaweb.gridworks.commands.info.GetRowsCommand;
-import com.metaweb.gridworks.commands.recon.ReconMarkNewTopicsCommand;
-import com.metaweb.gridworks.commands.recon.ReconMatchBestCandidatesCommand;
 import com.metaweb.gridworks.commands.recon.ReconDiscardJudgmentsCommand;
 import com.metaweb.gridworks.commands.recon.ReconJudgeOneCellCommand;
 import com.metaweb.gridworks.commands.recon.ReconJudgeSimilarCellsCommand;
+import com.metaweb.gridworks.commands.recon.ReconMarkNewTopicsCommand;
+import com.metaweb.gridworks.commands.recon.ReconMatchBestCandidatesCommand;
 import com.metaweb.gridworks.commands.recon.ReconMatchSpecificTopicCommand;
 import com.metaweb.gridworks.commands.recon.ReconcileCommand;
 import com.metaweb.gridworks.commands.util.CancelProcessesCommand;
@@ -50,6 +45,7 @@ import com.metaweb.gridworks.commands.util.PreviewExpressionCommand;
 import com.metaweb.gridworks.commands.util.PreviewProtographCommand;
 
 public class GridworksServlet extends HttpServlet {
+    
 	private static final long serialVersionUID = 2386057901503517403L;
 	
 	static protected Map<String, Command> _commands = new HashMap<String, Command>();
@@ -116,7 +112,7 @@ public class GridworksServlet extends HttpServlet {
 	}
 	
     protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
-		ProjectManager.initialize(new File("./data"));
+		ProjectManager.initialize();
 		
     	String commandName = request.getPathInfo().substring(1);
     	Command command = _commands.get(commandName);
@@ -126,7 +122,7 @@ public class GridworksServlet extends HttpServlet {
     }
     
     protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
-		ProjectManager.initialize(new File("./data"));
+		ProjectManager.initialize();
 		
     	String commandName = request.getPathInfo().substring(1);
     	Command command = _commands.get(commandName);
@@ -135,13 +131,4 @@ public class GridworksServlet extends HttpServlet {
     	}
     }
     
-    static public JSONObject evaluateJsonStringToObject(String s) throws JSONException {
-    	JSONTokener t = new JSONTokener(s);
-    	JSONObject o = (JSONObject) t.nextValue();
-    	return o;
-    }
-    
-    protected String encodeString(String s) {
-    	return s.replace("\"", "\\\"");
-    }
 }

src/main/java/com/metaweb/gridworks/ProjectManager.java

@@ -10,10 +10,13 @@ import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.codeberry.jdatapath.DataPath;
+import com.codeberry.jdatapath.JDataPathSystem;
 import com.metaweb.gridworks.model.Project;
 
 public class ProjectManager implements Serializable {
-	private static final long serialVersionUID = -2967415873336723962L;
+	
+    private static final long serialVersionUID = -2967415873336723962L;
 	
 	protected File _dir;
 	protected Map<Long, ProjectMetadata> _projectsMetadata;
@@ -22,18 +25,57 @@ public class ProjectManager implements Serializable {
 	
 	static public ProjectManager singleton;
 	
-	static public void initialize(File dir) {
+	static public void initialize() {
 		if (singleton == null) {
+		    File dir = getProjectLocation();
+		    Gridworks.log("Using data directory: " + dir.getAbsolutePath());
 			File file = new File(dir, "projects");
 			if (file.exists()) {
 				singleton = load(file);
-			}
-			if (singleton == null) {
+			} else {
 				singleton = new ProjectManager(dir);
 			}
 		}
 	}
 	
+	static protected File getProjectLocation() {
+	    String data_dir = Configurations.get("gridworks.data_dir");
+	    if (data_dir != null) {
+	        return new File(data_dir);
+	    }
+	    
+	    String os = Configurations.get("os.name").toLowerCase();
+	    if (os.contains("windows")) {
+	        // NOTE(SM): finding the "local data app" in windows from java is actually a PITA
+	        // see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
+	        // so we're using a library that uses JNI to ask directly the win32 APIs, 
+	        // it's not elegant but it's the safest bet
+	        DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
+            File data = new File(localDataPath.getPath());
+            data.mkdirs();
+            return data;	       
+	    } else if (os.contains("mac os x")) {
+	        // on macosx, use "~/Library/Application Support"
+	        String home = System.getProperty("user.home");
+            String data_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks"; 
+            File data = new File(data_home);
+            data.mkdirs();
+            return data;
+	    } else { // most likely a UNIX flavor
+	        // start with the XDG environment
+	        // see http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+            String data_home = System.getenv("XDG_DATA_HOME");
+            if (data_home == null) { // if not found, default back to ~/.local/share
+                String home = System.getProperty("user.home");
+                if (home == null) home = ".";
+                data_home = home + "/.local/share";
+            }
+            File data = new File(data_home + "/gridworks");
+            data.mkdirs();
+            return data;
+	    }
+	}
+	
 	static protected ProjectManager load(File file) {
 		ProjectManager pm = null;
 		FileInputStream fis = null;

src/main/java/com/metaweb/gridworks/commands/edit/AnnotateOneRowCommand.java

@@ -1,6 +1,6 @@
 package com.metaweb.gridworks.commands.edit;
 
-import java.io.IOException; 
+import java.io.IOException;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;

src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java

@@ -15,10 +15,9 @@ import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.log4j.Logger;
-
 import com.ibm.icu.text.CharsetDetector;
 import com.ibm.icu.text.CharsetMatch;
+import com.metaweb.gridworks.Gridworks;
 import com.metaweb.gridworks.ProjectManager;
 import com.metaweb.gridworks.ProjectMetadata;
 import com.metaweb.gridworks.commands.Command;
@@ -34,8 +33,6 @@ import com.oreilly.servlet.multipart.Part;
 
 public class CreateProjectCommand extends Command {
 
-    private final static Logger logger = Logger.getLogger("gridworks");
-
     @Override
 	public void doPost(HttpServletRequest request, HttpServletResponse response)
 			throws ServletException, IOException {
@@ -131,7 +128,7 @@ public class CreateProjectCommand extends Command {
                         CharsetMatch charsetMatch = detector.setText(enforceMarking(filePart.getInputStream())).detect();
                         options.setProperty("encoding", charsetMatch.getName());
                         options.setProperty("encoding_confidence", Integer.toString(charsetMatch.getConfidence()));
-                        logger.info("Best encoding guess: " + charsetMatch.getName() + " [confidence: " + charsetMatch.getConfidence() + "]");
+                        Gridworks.log("Best encoding guess: " + charsetMatch.getName() + " [confidence: " + charsetMatch.getConfidence() + "]");
                         Reader reader = charsetMatch.getReader();
                         try {
                             importer.read(charsetMatch.getReader(), project, options, skip, limit);

src/main/java/com/metaweb/gridworks/commands/edit/DoTextTransformCommand.java

@@ -4,7 +4,6 @@ import javax.servlet.http.HttpServletRequest;
 
 import org.json.JSONObject;
 
-
 import com.metaweb.gridworks.commands.EngineDependentCommand;
 import com.metaweb.gridworks.model.AbstractOperation;
 import com.metaweb.gridworks.operations.TextTransformOperation;

src/main/java/com/metaweb/gridworks/commands/edit/FacetBasedEditCommand.java

@@ -4,7 +4,6 @@ import javax.servlet.http.HttpServletRequest;
 
 import org.json.JSONObject;
 
-
 import com.metaweb.gridworks.commands.EngineDependentCommand;
 import com.metaweb.gridworks.model.AbstractOperation;
 import com.metaweb.gridworks.operations.FacetBasedEditOperation;

src/main/java/com/metaweb/gridworks/commands/util/GuessTypesOfColumnCommand.java

@@ -22,7 +22,6 @@ import org.json.JSONArray;
 import org.json.JSONObject;
 import org.json.JSONWriter;
 
-
 import com.metaweb.gridworks.commands.Command;
 import com.metaweb.gridworks.expr.ExpressionUtils;
 import com.metaweb.gridworks.model.Column;

src/main/java/com/metaweb/gridworks/commands/util/PreviewExpressionCommand.java

@@ -10,7 +10,6 @@ import javax.servlet.http.HttpServletResponse;
 import org.json.JSONArray;
 import org.json.JSONWriter;
 
-
 import com.metaweb.gridworks.commands.Command;
 import com.metaweb.gridworks.expr.EvalError;
 import com.metaweb.gridworks.expr.Evaluable;

src/main/java/com/metaweb/gridworks/expr/ExpressionUtils.java

@@ -1,6 +1,6 @@
 package com.metaweb.gridworks.expr;
 
-import java.util.Properties; 
+import java.util.Properties;
 
 import com.metaweb.gridworks.model.Cell;
 import com.metaweb.gridworks.model.Project;

src/main/java/com/metaweb/gridworks/gel/controls/IsTest.java

@@ -1,6 +1,6 @@
 package com.metaweb.gridworks.gel.controls;
 
-import java.util.Properties; 
+import java.util.Properties;
 
 import org.json.JSONException;
 import org.json.JSONWriter;

src/main/java/com/metaweb/gridworks/gel/controls/With.java

@@ -1,6 +1,6 @@
 package com.metaweb.gridworks.gel.controls;
 
-import java.util.Properties; 
+import java.util.Properties;
 
 import org.json.JSONException;
 import org.json.JSONWriter;

src/main/java/com/metaweb/gridworks/model/recon/ReconConfig.java

@@ -1,6 +1,6 @@
 package com.metaweb.gridworks.model.recon;
 
-import java.io.Serializable; 
+import java.io.Serializable;
 import java.util.List;
 
 import org.json.JSONObject;

src/main/java/com/metaweb/gridworks/operations/MultiValuedCellJoinOperation.java

@@ -1,6 +1,6 @@
 package com.metaweb.gridworks.operations;
 
-import java.util.ArrayList;  
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 

src/main/java/com/metaweb/gridworks/operations/MultiValuedCellSplitOperation.java

@@ -1,6 +1,6 @@
 package com.metaweb.gridworks.operations;
 
- import java.util.ArrayList; 
+ import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;