diff --git a/.gitignore b/.gitignore
index 1f2288b5d..16d995959 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,8 @@
+.DS_Store
build/
-src/main/webapp/WEB-INF/classes/
-tests/java/classes/
+server/classes/
+main/webapp/WEB-INF/classes/
+main/tests/server/classes/
+main/test-output/
+appengine/classes/
tools/
diff --git a/build.xml b/build.xml
index 82097d3ea..2dc4a6b82 100644
--- a/build.xml
+++ b/build.xml
@@ -118,6 +118,7 @@
+
@@ -125,6 +126,7 @@
+
diff --git a/main/IDEs/eclipse/GridworksTests.launch b/main/IDEs/eclipse/GridworksTests.launch
index 20ebd70ff..0cc57b89a 100644
--- a/main/IDEs/eclipse/GridworksTests.launch
+++ b/main/IDEs/eclipse/GridworksTests.launch
@@ -6,7 +6,6 @@
-
@@ -15,7 +14,7 @@
-
+
diff --git a/main/src/com/metaweb/gridworks/GridworksServlet.java b/main/src/com/metaweb/gridworks/GridworksServlet.java
index 46aef38b1..d884d5ef6 100644
--- a/main/src/com/metaweb/gridworks/GridworksServlet.java
+++ b/main/src/com/metaweb/gridworks/GridworksServlet.java
@@ -1,11 +1,13 @@
package com.metaweb.gridworks;
+import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
+import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
@@ -18,8 +20,12 @@ import com.metaweb.gridworks.commands.Command;
public class GridworksServlet extends HttpServlet {
+ static private final String VERSION = "1.0";
+
private static final long serialVersionUID = 2386057901503517403L;
+ private static final String JAVAX_SERVLET_CONTEXT_TEMPDIR = "javax.servlet.context.tempdir";
+
static final private Map commands = new HashMap();
// timer for periodically saving projects
@@ -104,11 +110,12 @@ public class GridworksServlet extends HttpServlet {
{"mqlwrite", "com.metaweb.gridworks.commands.freebase.MQLWriteCommand"},
};
- static {
- registerCommands(commandNames);
+ public static String getVersion() {
+ return VERSION;
}
final static protected long s_autoSavePeriod = 1000 * 60 * 5; // 5 minutes
+
static protected class AutoSaveTimerTask extends TimerTask {
public void run() {
try {
@@ -121,12 +128,21 @@ public class GridworksServlet extends HttpServlet {
}
}
+ protected ServletConfig config;
+
@Override
public void init() throws ServletException {
- super.init();
logger.trace("> initialize");
- ProjectManager.initialize();
+ String data = getInitParameter("gridworks.data");
+
+ if (data == null) {
+ throw new ServletException("can't find servlet init config 'gridworks.data', I have to give up initializing");
+ }
+
+ registerCommands(commandNames);
+
+ ProjectManager.initialize(new File(data));
if (_timer == null) {
_timer = new Timer("autosave");
@@ -149,6 +165,8 @@ public class GridworksServlet extends HttpServlet {
ProjectManager.singleton.save(true); // complete save
ProjectManager.singleton = null;
}
+
+ this.config = null;
super.destroy();
@@ -188,6 +206,26 @@ public class GridworksServlet extends HttpServlet {
return slash > 0 ? commandName.substring(0, slash) : commandName;
}
+ private File tempDir = null;
+
+ public File getTempDir() {
+ if (tempDir == null) {
+ File tempDir = (File) this.config.getServletContext().getAttribute(JAVAX_SERVLET_CONTEXT_TEMPDIR);
+ if (tempDir == null) {
+ throw new RuntimeException("This app server doesn't support temp directories");
+ }
+ }
+ return tempDir;
+ }
+
+ public File getTempFile(String name) {
+ return new File(getTempDir(), name);
+ }
+
+ public String getConfiguration(String name, String def) {
+ return null;
+ }
+
/**
* Register an array of commands
*
@@ -198,7 +236,7 @@ public class GridworksServlet extends HttpServlet {
* the second.
* @return false if any commands failed to load
*/
- static public boolean registerCommands(String[][] commands) {
+ private boolean registerCommands(String[][] commands) {
boolean status = true;
for (String[] command : commandNames) {
String commandName = command[0];
@@ -206,8 +244,8 @@ public class GridworksServlet extends HttpServlet {
logger.debug("Loading command " + commandName + " class: " + className);
Command cmd;
try {
- // TODO: May need to use the servlet container's class loader here
- cmd = (Command) Class.forName(className).newInstance();
+ cmd = (Command) this.getClass().getClassLoader().loadClass(className).newInstance();
+ cmd.init(this);
} catch (InstantiationException e) {
logger.error("Failed to load command class " + className, e);
status = false;
@@ -235,8 +273,7 @@ public class GridworksServlet extends HttpServlet {
* object implementing the command
* @return true if command was loaded and registered successfully
*/
- static public boolean registerCommand(String name,
- Command commandObject) {
+ protected boolean registerCommand(String name, Command commandObject) {
if (commands.containsKey(name)) {
return false;
}
@@ -245,7 +282,7 @@ public class GridworksServlet extends HttpServlet {
}
// Currently only for test purposes
- static protected boolean unregisterCommand(String verb) {
+ protected boolean unregisterCommand(String verb) {
return commands.remove(verb) != null;
}
}
diff --git a/main/src/com/metaweb/gridworks/ProjectManager.java b/main/src/com/metaweb/gridworks/ProjectManager.java
index a5885b819..19d0a32df 100644
--- a/main/src/com/metaweb/gridworks/ProjectManager.java
+++ b/main/src/com/metaweb/gridworks/ProjectManager.java
@@ -22,8 +22,6 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.codeberry.jdatapath.DataPath;
-import com.codeberry.jdatapath.JDataPathSystem;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.util.JSONUtilities;
@@ -58,112 +56,13 @@ public class ProjectManager {
static public ProjectManager singleton;
- static public synchronized void initialize() {
+ static public synchronized void initialize(File dir) {
if (singleton == null) {
- File dir = getProjectLocation();
logger.info("Using workspace directory: {}", dir.getAbsolutePath());
-
singleton = new ProjectManager(dir);
}
}
-
- static protected File getProjectLocation() {
- String data_dir = Configurations.get("gridworks.data_dir");
- if (data_dir != null) {
- return new File(data_dir);
- }
- String os = Configurations.get("os.name").toLowerCase();
- if (os.contains("windows")) {
- try {
- // NOTE(SM): finding the "local data app" in windows from java is actually a PITA
- // see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
- // so we're using a library that uses JNI to ask directly the win32 APIs,
- // it's not elegant but it's the safest bet.
-
- DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
- File data = new File(fixWindowsUnicodePath(localDataPath.getPath()));
- data.mkdirs();
- return data;
- } catch (Error e) {
- /*
- * The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll
- * we include is compiled for 32-bit. In this case, we just have to dig up
- * environment variables and try our best to find a user-specific path.
- */
-
- logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables");
-
- File parentDir = null;
- {
- String appData = System.getenv("APPDATA");
- if (appData != null && appData.length() > 0) {
- // e.g., C:\Users\[userid]\AppData\Roaming
- parentDir = new File(appData);
- } else {
- String userProfile = System.getenv("USERPROFILE");
- if (userProfile != null && userProfile.length() > 0) {
- // e.g., C:\Users\[userid]
- parentDir = new File(userProfile);
- }
- }
- }
- if (parentDir == null) {
- parentDir = new File(".");
- }
-
- File data = new File(parentDir, "Gridworks");
- data.mkdirs();
-
- return data;
- }
- } else if (os.contains("mac os x")) {
- // on macosx, use "~/Library/Application Support"
- String home = System.getProperty("user.home");
- String data_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks";
- File data = new File(data_home);
- data.mkdirs();
- return data;
- } else { // most likely a UNIX flavor
- // start with the XDG environment
- // see http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
- String data_home = System.getenv("XDG_DATA_HOME");
- if (data_home == null) { // if not found, default back to ~/.local/share
- String home = System.getProperty("user.home");
- if (home == null) home = ".";
- data_home = home + "/.local/share";
- }
- File data = new File(data_home + "/gridworks");
- data.mkdirs();
- return data;
- }
- }
-
- /**
- * For Windows file paths that contain user IDs with non ASCII characters,
- * those characters might get replaced with ?. We need to use the environment
- * APPDATA value to substitute back the original user ID.
- */
- static protected String fixWindowsUnicodePath(String path) {
- int q = path.indexOf('?');
- if (q < 0) {
- return path;
- }
- int pathSep = path.indexOf(File.separatorChar, q);
-
- String goodPath = System.getenv("APPDATA");
- if (goodPath == null || goodPath.length() == 0) {
- goodPath = System.getenv("USERPROFILE");
- if (!goodPath.endsWith(File.separator)) {
- goodPath = goodPath + File.separator;
- }
- }
-
- int goodPathSep = goodPath.indexOf(File.separatorChar, q);
-
- return path.substring(0, q) + goodPath.substring(q, goodPathSep) + path.substring(pathSep);
- }
-
private ProjectManager(File dir) {
_workspaceDir = dir;
_workspaceDir.mkdirs();
diff --git a/main/src/com/metaweb/gridworks/commands/Command.java b/main/src/com/metaweb/gridworks/commands/Command.java
index 349eea62c..d998c4ed1 100644
--- a/main/src/com/metaweb/gridworks/commands/Command.java
+++ b/main/src/com/metaweb/gridworks/commands/Command.java
@@ -16,6 +16,7 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata;
@@ -33,6 +34,12 @@ public abstract class Command {
final static protected Logger logger = LoggerFactory.getLogger("command");
+ protected GridworksServlet servlet;
+
+ public void init(GridworksServlet servlet) {
+ this.servlet = servlet;
+ }
+
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
diff --git a/main/src/com/metaweb/gridworks/commands/auth/AuthorizeCommand.java b/main/src/com/metaweb/gridworks/commands/auth/AuthorizeCommand.java
index 2c1ea053e..a5d8709a1 100644
--- a/main/src/com/metaweb/gridworks/commands/auth/AuthorizeCommand.java
+++ b/main/src/com/metaweb/gridworks/commands/auth/AuthorizeCommand.java
@@ -2,6 +2,8 @@ package com.metaweb.gridworks.commands.auth;
import java.io.IOException;
import java.io.PrintWriter;
+import java.net.URI;
+import java.net.URISyntaxException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@@ -10,13 +12,12 @@ import javax.servlet.http.HttpServletResponse;
import oauth.signpost.OAuthConsumer;
import oauth.signpost.OAuthProvider;
-import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.oauth.Credentials;
import com.metaweb.gridworks.oauth.OAuthUtilities;
import com.metaweb.gridworks.oauth.Provider;
-public class AuthorizeCommand extends Command {
+public class AuthorizeCommand extends Command {
private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier";
@@ -124,7 +125,21 @@ public class AuthorizeCommand extends Command {
private String getBaseURL(HttpServletRequest request, Provider provider) {
String host = request.getHeader("host");
- if (host == null) host = Gridworks.getFullHost();
+ if (host == null) {
+ String referrer = request.getHeader("referer");
+ if (referrer != null) {
+ URI url;
+ try {
+ url = new URI(referrer);
+ int port = url.getPort();
+ host = url.getHost() + ((port > -1) ? ":" + url.getPort() : "");
+ } catch (URISyntaxException e) {
+ throw new RuntimeException("referrer '" + referrer + "' can't be parsed as a URL");
+ }
+ } else {
+ throw new RuntimeException("neither the 'host' nor 'referer' headers were present in the HTTP response, I can't determine what URL gridworks is listening to.");
+ }
+ }
return "http://" + host + "/command/authorize/" + provider.getHost();
}
}
diff --git a/main/src/com/metaweb/gridworks/commands/project/CreateProjectCommand.java b/main/src/com/metaweb/gridworks/commands/project/CreateProjectCommand.java
index c4e567ff4..36f87ec0a 100644
--- a/main/src/com/metaweb/gridworks/commands/project/CreateProjectCommand.java
+++ b/main/src/com/metaweb/gridworks/commands/project/CreateProjectCommand.java
@@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
-import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata;
import com.metaweb.gridworks.commands.Command;
@@ -367,7 +366,7 @@ public class CreateProjectCommand extends Command {
}
private File save(InputStream is) throws IOException {
- File temp = Gridworks.getTempFile(Long.toString(System.currentTimeMillis()));
+ File temp = this.servlet.getTempFile(Long.toString(System.currentTimeMillis()));
temp.deleteOnExit();
IOUtils.copy(is,temp);
is.close();
diff --git a/main/src/com/metaweb/gridworks/exporters/CsvExporter.java b/main/src/com/metaweb/gridworks/exporters/CsvExporter.java
index 2e6548b6e..f001233cf 100644
--- a/main/src/com/metaweb/gridworks/exporters/CsvExporter.java
+++ b/main/src/com/metaweb/gridworks/exporters/CsvExporter.java
@@ -40,66 +40,66 @@ public class CsvExporter implements Exporter{
@Override
public void export(Project project, Properties options, Engine engine, Writer writer) throws IOException {
- {
- boolean printColumnHeader = true;
- if(options != null)
- printColumnHeader = options.getProperty("printColumnHeader")=="false"?false:true;
+ boolean printColumnHeader = true;
- RowVisitor visitor = new RowVisitor() {
- CSVWriter csvWriter;
- boolean printColumnHeader = true;
- boolean isFirstRow = true; //the first row should also add the column headers
-
- public RowVisitor init(CSVWriter writer, boolean printColumnHeader){
- this.csvWriter = writer;
- this.printColumnHeader = printColumnHeader;
- return this;
- }
-
- public boolean visit(Project project, int rowIndex, Row row) {
- String[] cols = new String[project.columnModel.columns.size()];
- String[] vals = new String[row.cells.size()];
-
- int i = 0;
- for(Column col : project.columnModel.columns){
- int cellIndex = col.getCellIndex();
- cols[i] = col.getName();
-
- Cell cell = row.cells.get(cellIndex);
- if(cell != null){
- vals[i] = cell.value.toString();
- }
- i++;
- }
-
- if( printColumnHeader && isFirstRow ){
- csvWriter.writeNext(cols,false);
- isFirstRow = false; //switch off flag
- }
- csvWriter.writeNext(vals,false);
-
- return false;
- }
-
- @Override
- public void start(Project project) {
- // nothing to do
- }
-
- @Override
- public void end(Project project) {
- try {
- csvWriter.close();
- } catch (IOException e) {
- logger.error("CsvExporter could not close writer : " + e.getMessage());
- }
- }
-
- }.init(new CSVWriter(writer, separator), printColumnHeader);
-
- FilteredRows filteredRows = engine.getAllFilteredRows();
- filteredRows.accept(project, visitor);
+ if (options != null) {
+ printColumnHeader = Boolean.parseBoolean(options.getProperty("printColumnHeader"));
}
+
+ RowVisitor visitor = new RowVisitor() {
+ CSVWriter csvWriter;
+ boolean printColumnHeader = true;
+ boolean isFirstRow = true; //the first row should also add the column headers
+
+ public RowVisitor init(CSVWriter writer, boolean printColumnHeader){
+ this.csvWriter = writer;
+ this.printColumnHeader = printColumnHeader;
+ return this;
+ }
+
+ public boolean visit(Project project, int rowIndex, Row row) {
+ String[] cols = new String[project.columnModel.columns.size()];
+ String[] vals = new String[row.cells.size()];
+
+ int i = 0;
+ for(Column col : project.columnModel.columns){
+ int cellIndex = col.getCellIndex();
+ cols[i] = col.getName();
+
+ Cell cell = row.cells.get(cellIndex);
+ if(cell != null){
+ vals[i] = cell.value.toString();
+ }
+ i++;
+ }
+
+ if( printColumnHeader && isFirstRow ){
+ csvWriter.writeNext(cols,false);
+ isFirstRow = false; //switch off flag
+ }
+ csvWriter.writeNext(vals,false);
+
+ return false;
+ }
+
+ @Override
+ public void start(Project project) {
+ // nothing to do
+ }
+
+ @Override
+ public void end(Project project) {
+ try {
+ csvWriter.close();
+ } catch (IOException e) {
+ logger.error("CsvExporter could not close writer : " + e.getMessage());
+ }
+ }
+
+ }.init(new CSVWriter(writer, separator), printColumnHeader);
+
+ FilteredRows filteredRows = engine.getAllFilteredRows();
+ filteredRows.accept(project, visitor);
}
@Override
diff --git a/main/src/com/metaweb/gridworks/expr/JythonEvaluable.java b/main/src/com/metaweb/gridworks/expr/JythonEvaluable.java
index 1125409d5..5264db879 100644
--- a/main/src/com/metaweb/gridworks/expr/JythonEvaluable.java
+++ b/main/src/com/metaweb/gridworks/expr/JythonEvaluable.java
@@ -19,7 +19,7 @@ public class JythonEvaluable implements Evaluable {
private static PythonInterpreter _engine;
static {
- File libPath = new File("lib/jython");
+ File libPath = new File("webapp/WEB-INF/lib/jython");
if (libPath.exists()) {
Properties props = new Properties();
props.setProperty("python.path", libPath.getAbsolutePath());
diff --git a/main/src/com/metaweb/gridworks/history/History.java b/main/src/com/metaweb/gridworks/history/History.java
index bc600bd87..52789b412 100644
--- a/main/src/com/metaweb/gridworks/history/History.java
+++ b/main/src/com/metaweb/gridworks/history/History.java
@@ -16,7 +16,7 @@ import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
-import com.metaweb.gridworks.Gridworks;
+import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.model.Project;
@@ -68,7 +68,7 @@ public class History implements Jsonizable {
}
static public void writeOneChange(Writer writer, Change change, Properties options) throws IOException {
- writer.write(Gridworks.getVersion()); writer.write('\n');
+ writer.write(GridworksServlet.getVersion()); writer.write('\n');
writer.write(change.getClass().getName()); writer.write('\n');
change.save(writer, options);
diff --git a/main/src/com/metaweb/gridworks/importers/XmlImportUtilities.java b/main/src/com/metaweb/gridworks/importers/XmlImportUtilities.java
index 89de297a2..78ab92ebe 100644
--- a/main/src/com/metaweb/gridworks/importers/XmlImportUtilities.java
+++ b/main/src/com/metaweb/gridworks/importers/XmlImportUtilities.java
@@ -72,8 +72,6 @@ public class XmlImportUtilities {
}
static public String[] detectPathFromTag(InputStream inputStream, String tag) {
- //List candidates = new ArrayList();
-
try {
XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(inputStream);
@@ -99,6 +97,8 @@ public class XmlImportUtilities {
}
static protected List detectRecordElement(XMLStreamReader parser, String tag) throws XMLStreamException {
+ if(parser.getEventType() == XMLStreamConstants.START_DOCUMENT)
+ parser.next();
String localName = parser.getLocalName();
String fullName = composeName(parser.getPrefix(), localName);
if (tag.equals(parser.getLocalName()) || tag.equals(fullName)) {
@@ -327,6 +327,10 @@ public class XmlImportUtilities {
int pathIndex,
ImportColumnGroup rootColumnGroup
) throws XMLStreamException {
+ if(parser.getEventType() == XMLStreamConstants.START_DOCUMENT){
+ logger.warn("Cannot use findRecord method for START_DOCUMENT event");
+ return;
+ }
String tagName = parser.getLocalName();
if (tagName.equals(recordPath[pathIndex])) {
if (pathIndex < recordPath.length - 1) {
@@ -466,7 +470,7 @@ public class XmlImportUtilities {
ImportRecord record,
String columnLocalName,
String text,
- int commonStaringRowIndex
+ int commonStartingRowIndex
) {
if (text == null || ((String) text).isEmpty()) {
return;
@@ -478,7 +482,7 @@ public class XmlImportUtilities {
int cellIndex = column.cellIndex;
while (cellIndex >= record.columnEmptyRowIndices.size()) {
- record.columnEmptyRowIndices.add(commonStaringRowIndex);
+ record.columnEmptyRowIndices.add(commonStartingRowIndex);
}
int rowIndex = record.columnEmptyRowIndices.get(cellIndex);
@@ -491,7 +495,9 @@ public class XmlImportUtilities {
row.add(null);
}
- row.set(cellIndex, new Cell(value, null));
+ logger.trace("Adding cell with value : " + value + " to row : " + rowIndex + " at cell index : " + (cellIndex-1));
+
+ row.set(cellIndex-1, new Cell(value, null));
record.columnEmptyRowIndices.set(cellIndex, rowIndex + 1);
diff --git a/main/src/com/metaweb/gridworks/importers/XmlImporter.java b/main/src/com/metaweb/gridworks/importers/XmlImporter.java
index 23c9a6369..6a2f42001 100644
--- a/main/src/com/metaweb/gridworks/importers/XmlImporter.java
+++ b/main/src/com/metaweb/gridworks/importers/XmlImporter.java
@@ -57,6 +57,9 @@ public class XmlImporter implements Importer {
}
}
+ if(recordPath == null)
+ return;
+
ImportColumnGroup rootColumnGroup = new ImportColumnGroup();
XmlImportUtilities.importXml(pis, project, recordPath, rootColumnGroup);
diff --git a/main/src/com/metaweb/gridworks/logging/IndentingLayout.java b/main/src/com/metaweb/gridworks/logging/IndentingLayout.java
new file mode 100644
index 000000000..4886431e4
--- /dev/null
+++ b/main/src/com/metaweb/gridworks/logging/IndentingLayout.java
@@ -0,0 +1,143 @@
+package com.metaweb.gridworks.logging;
+
+/*
+ * Copyright (c) Massachusetts Institute of Technology, 2007
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. The name of the author may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Original code: http://simile.mit.edu/repository/tracer/trunk/
+ */
+
+import java.util.Calendar;
+import java.util.Date;
+
+import org.apache.log4j.Layout;
+import org.apache.log4j.spi.LoggingEvent;
+
+/**
+ * This is a special Log4j log formatter that is capable of reacting on special log messages
+ * and 'indent' the logs accordingly. This is very useful to visually inspect a debug log
+ * and see what calls what. An example of logs are "> method()" and "< method()" where > and <
+ * are used to indicate respectively "entering" and "exiting".
+ */
+public class IndentingLayout extends Layout {
+
+ protected static final int CONTEXT_SIZE = 25;
+ protected static final long MAX_DELTA = 10000;
+
+ protected Calendar calendar = Calendar.getInstance();
+ protected long previousTime = 0;
+ protected int indentation = 0;
+
+ public void activateOptions() {
+ // no options at this time
+ }
+
+ public String format(LoggingEvent event) {
+ String message = event.getRenderedMessage();
+ if (message == null) return "";
+ if (message.length() < 2) return message;
+
+ char leader = message.charAt(0);
+ char secondLeader = message.charAt(1);
+ if ((leader == '<') && (secondLeader == ' ') && (this.indentation > 0)) this.indentation--;
+
+ // Reset buf
+ StringBuffer buf = new StringBuffer(256);
+
+ Date date = new Date();
+ long now = date.getTime();
+ calendar.setTime(date);
+
+ long delta = 0;
+ if (previousTime > 0) {
+ delta = now - previousTime;
+ }
+ previousTime = now;
+
+// if ((previousTime == 0) || (delta > MAX_DELTA)) {
+// buf.append('\n');
+// indentation = 0; // reset indentation after a while, as we might
+// // have runaway/unmatched log entries
+// }
+
+ int hour = calendar.get(Calendar.HOUR_OF_DAY);
+ if (hour < 10) buf.append('0');
+ buf.append(hour);
+ buf.append(':');
+
+ int mins = calendar.get(Calendar.MINUTE);
+ if (mins < 10) buf.append('0');
+ buf.append(mins);
+ buf.append(':');
+
+ int secs = calendar.get(Calendar.SECOND);
+ if (secs < 10) buf.append('0');
+ buf.append(secs);
+ buf.append('.');
+
+ int millis = (int) (now % 1000);
+ if (millis < 100) buf.append('0');
+ if (millis < 10) buf.append('0');
+ buf.append(millis);
+
+ buf.append(" [");
+ String context = ((String) event.getMDC("LogEvent"));
+ if (context == null) {
+ context = event.getLoggerName();
+ }
+ if (context.length() < CONTEXT_SIZE) {
+ pad(buf, CONTEXT_SIZE - context.length(), ' ');
+ buf.append(context);
+ } else {
+ buf.append("..");
+ buf.append(context.substring(context.length() - CONTEXT_SIZE + 2));
+ }
+ buf.append("] ");
+
+ pad(buf, indentation, ' ');
+
+ buf.append(message);
+
+ buf.append(" (");
+ buf.append(delta);
+ buf.append("ms)\n");
+
+ if ((leader == '>') && (secondLeader == ' ')) indentation++;
+
+ return buf.toString();
+ }
+
+ private void pad(StringBuffer buffer, int pads, char padchar) {
+ for (int i = 0; i < pads; i++) {
+ buffer.append(padchar);
+ }
+ }
+
+ public boolean ignoresThrowable() {
+ return true;
+ }
+}
diff --git a/main/src/com/metaweb/gridworks/model/Project.java b/main/src/com/metaweb/gridworks/model/Project.java
index 6a22e51d3..d73bee522 100644
--- a/main/src/com/metaweb/gridworks/model/Project.java
+++ b/main/src/com/metaweb/gridworks/model/Project.java
@@ -19,7 +19,7 @@ import java.util.zip.ZipOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.metaweb.gridworks.Gridworks;
+import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata;
import com.metaweb.gridworks.history.History;
@@ -128,7 +128,7 @@ public class Project {
}
protected void saveToWriter(Writer writer, Properties options) throws IOException {
- writer.write(Gridworks.getVersion()); writer.write('\n');
+ writer.write(GridworksServlet.getVersion()); writer.write('\n');
writer.write("columnModel=\n"); columnModel.save(writer, options);
writer.write("history=\n"); history.save(writer, options);
diff --git a/main/src/com/metaweb/gridworks/util/FreebaseUtils.java b/main/src/com/metaweb/gridworks/util/FreebaseUtils.java
index 50c56ceca..dfc93510f 100644
--- a/main/src/com/metaweb/gridworks/util/FreebaseUtils.java
+++ b/main/src/com/metaweb/gridworks/util/FreebaseUtils.java
@@ -27,7 +27,7 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
-import com.metaweb.gridworks.Gridworks;
+import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.oauth.Credentials;
import com.metaweb.gridworks.oauth.OAuthUtilities;
import com.metaweb.gridworks.oauth.Provider;
@@ -59,7 +59,7 @@ public class FreebaseUtils {
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
- httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion());
+ httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
// this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1");
@@ -102,7 +102,7 @@ public class FreebaseUtils {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
- httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion());
+ httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS
@@ -128,7 +128,7 @@ public class FreebaseUtils {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
- httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion());
+ httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS
@@ -176,7 +176,7 @@ public class FreebaseUtils {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(FREEQ_URL);
- httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion());
+ httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
httpRequest.setEntity(entity);
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
diff --git a/main/src/com/metaweb/gridworks/util/Pool.java b/main/src/com/metaweb/gridworks/util/Pool.java
index ecc24d7ca..6f4893521 100644
--- a/main/src/com/metaweb/gridworks/util/Pool.java
+++ b/main/src/com/metaweb/gridworks/util/Pool.java
@@ -15,7 +15,7 @@ import java.util.Map.Entry;
import org.json.JSONException;
import org.json.JSONWriter;
-import com.metaweb.gridworks.Gridworks;
+import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.ReconCandidate;
@@ -59,7 +59,7 @@ public class Pool implements Jsonizable {
}
public void save(Writer writer) throws IOException {
- writer.write(Gridworks.getVersion()); writer.write('\n');
+ writer.write(GridworksServlet.getVersion()); writer.write('\n');
Properties options = new Properties();
options.setProperty("mode", "save");
diff --git a/main/src/log4j.properties b/main/src/log4j.properties
new file mode 100644
index 000000000..fea32e42d
--- /dev/null
+++ b/main/src/log4j.properties
@@ -0,0 +1,7 @@
+log4j.rootLogger=INFO, console
+log4j.logger.org.apache.http.headers=WARN
+log4j.logger.org.apache.http.impl=WARN
+log4j.logger.org.apache.http.client=WARN
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=com.metaweb.gridworks.logging.IndentingLayout
diff --git a/main/tests/client/js/facets.js b/main/tests/client/src/facets.js
similarity index 100%
rename from main/tests/client/js/facets.js
rename to main/tests/client/src/facets.js
diff --git a/main/tests/client/js/initialize.js b/main/tests/client/src/initialize.js
similarity index 100%
rename from main/tests/client/js/initialize.js
rename to main/tests/client/src/initialize.js
diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletStub.java b/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletStub.java
index 351f8fd71..0e1bf1045 100644
--- a/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletStub.java
+++ b/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletStub.java
@@ -36,7 +36,7 @@ public class GridworksServletStub extends GridworksServlet {
* @param commandName
* @param command
*/
- static public void InsertCommand( String commandName, Command command ){
+ public void insertCommand( String commandName, Command command ){
registerCommand(commandName, command);
}
@@ -44,7 +44,7 @@ public class GridworksServletStub extends GridworksServlet {
* Helper method for clearing up after testing
* @param commandName
*/
- static public void RemoveCommand( String commandName ){
+ public void removeCommand( String commandName ){
unregisterCommand(commandName);
}
}
diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletTests.java b/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletTests.java
index f40f42053..e3a089184 100644
--- a/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletTests.java
+++ b/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletTests.java
@@ -40,21 +40,22 @@ public class GridworksServletTests {
@BeforeMethod
public void SetUp() {
- SUT = new GridworksServletStub();
request = mock(HttpServletRequest.class);
response = mock(HttpServletResponse.class);
command = mock(Command.class);
- GridworksServletStub.InsertCommand(TEST_COMMAND_NAME,command); //inject mock into command container
+ SUT = new GridworksServletStub();
+ SUT.insertCommand(TEST_COMMAND_NAME,command); //inject mock into command container
}
@AfterMethod
public void TearDown() {
+ SUT.removeCommand(TEST_COMMAND_NAME); //remove mock to clean command container
SUT = null;
+
request = null;
response = null;
command = null;
- GridworksServletStub.RemoveCommand(TEST_COMMAND_NAME); //remove mock to clean command container
}
//-------------------AutoSaveTimerTask tests-----------
diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/importers/RdfTripleImporterTests.java b/main/tests/server/src/com/metaweb/gridworks/tests/importers/RdfTripleImporterTests.java
index 694521e80..91a4f5059 100644
--- a/main/tests/server/src/com/metaweb/gridworks/tests/importers/RdfTripleImporterTests.java
+++ b/main/tests/server/src/com/metaweb/gridworks/tests/importers/RdfTripleImporterTests.java
@@ -1,153 +1,153 @@
-package com.metaweb.gridworks.tests.importers;
-
-import java.io.StringReader;
-import java.util.Properties;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import com.metaweb.gridworks.importers.RdfTripleImporter;
-import com.metaweb.gridworks.model.Project;
-
-
-public class RdfTripleImporterTests {
- // logging
- final static protected Logger logger = LoggerFactory.getLogger("RdfImporterTests");
-
- //System Under Test
- RdfTripleImporter SUT = null;
- Project project = null;
- Properties options = null;
-
- @BeforeMethod
- public void SetUp(){
- SUT = new RdfTripleImporter();
- project = new Project();
- options = new Properties();
- options.put("base-url", "http://rdf.freebase.com");
- }
-
- @Test(enabled=false)
- public void CanParseSingleLineTriple(){
- String sampleRdf = " .";
- StringReader reader = new StringReader(sampleRdf);
-
- try {
- SUT.read(reader, project, options);
- project.update();
- } catch (Exception e) {
- Assert.fail();
- }
-
- Assert.assertEquals(project.columnModel.columns.size(), 2);
- Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
- Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album");
- Assert.assertEquals(project.rows.size(), 1);
- Assert.assertEquals(project.rows.get(0).cells.size(), 2);
- Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
- Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks");
- }
-
- @Test
- public void CanParseMultiLineTriple(){
- String sampleRdf = " .\n" +
- " .\n" +
- " .";
- StringReader reader = new StringReader(sampleRdf);
-
- try {
- SUT.read(reader, project, options);
- project.update();
- } catch (Exception e) {
- Assert.fail();
- }
-
- //columns
- Assert.assertEquals(project.columnModel.columns.size(), 2);
- Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
- Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album");
-
- //rows
- Assert.assertEquals(project.rows.size(), 3);
-
- //row0
- Assert.assertEquals(project.rows.get(0).cells.size(), 2);
- Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
- Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks");
-
- //row1
- Assert.assertEquals(project.rows.get(1).cells.size(), 2);
- Assert.assertNull(project.rows.get(1).cells.get(0));
- Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); //NB triples aren't created in order they were input
- Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0);
- Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0);
-
- //row2
- Assert.assertEquals(project.rows.get(2).cells.size(), 2);
- Assert.assertNull(project.rows.get(2).cells.get(0));
- Assert.assertEquals(project.rows.get(2).cells.get(1).value, "http://rdf.freebase.com/ns/en.under_the_red_sky"); //NB triples aren't created in order they were input
- Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].rowIndex, 0);
- Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].cellIndex, 0);
- }
-
- @Test
- public void CanParseMultiLineMultiPredicatesTriple(){
- String sampleRdf = " .\n" +
- " .\n" +
- " .";
- StringReader reader = new StringReader(sampleRdf);
-
- try {
- SUT.read(reader, project, options);
- project.update();
- } catch (Exception e) {
- Assert.fail();
- }
-
- //columns
- Assert.assertEquals(project.columnModel.columns.size(), 3);
- Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
- Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album");
- Assert.assertEquals(project.columnModel.columns.get(2).getName(), "http://rdf.freebase.com/ns/music.artist.genre");
-
- //rows
- Assert.assertEquals(project.rows.size(), 2);
-
- //row0
- Assert.assertEquals(project.rows.get(0).cells.size(), 3);
- Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
- Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks");
- Assert.assertEquals(project.rows.get(0).cells.get(2).value, "http://rdf.freebase.com/ns/en.folk_rock");
-
- //row1
- Assert.assertEquals(project.rows.get(1).cells.size(), 2);
- Assert.assertNull(project.rows.get(1).cells.get(0));
- Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home");
- Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0);
- Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0);
- }
-
- @Test
- public void CanParseTripleWithValue(){
- String sampleRdf = " \"Robert Zimmerman\"@en.";
- StringReader reader = new StringReader(sampleRdf);
-
- try {
- SUT.read(reader, project, options);
- project.update();
- } catch (Exception e) {
- Assert.fail();
- }
-
- Assert.assertEquals(project.columnModel.columns.size(), 2);
- Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
- Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/common.topic.alias");
- Assert.assertEquals(project.rows.size(), 1);
- Assert.assertEquals(project.rows.get(0).cells.size(), 2);
- Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
- Assert.assertEquals(project.rows.get(0).cells.get(1).value, "\"Robert Zimmerman\"@en");
- }
-}
+package com.metaweb.gridworks.tests.importers;
+
+import java.io.StringReader;
+import java.util.Properties;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.metaweb.gridworks.importers.RdfTripleImporter;
+import com.metaweb.gridworks.model.Project;
+
+
+public class RdfTripleImporterTests {
+ // logging
+ final static protected Logger logger = LoggerFactory.getLogger("RdfImporterTests");
+
+ //System Under Test
+ RdfTripleImporter SUT = null;
+ Project project = null;
+ Properties options = null;
+
+ @BeforeMethod
+ public void SetUp(){
+ SUT = new RdfTripleImporter();
+ project = new Project();
+ options = new Properties();
+ options.put("base-url", "http://rdf.freebase.com");
+ }
+
+ @Test(enabled=false)
+ public void CanParseSingleLineTriple(){
+ String sampleRdf = " .";
+ StringReader reader = new StringReader(sampleRdf);
+
+ try {
+ SUT.read(reader, project, options);
+ project.update();
+ } catch (Exception e) {
+ Assert.fail();
+ }
+
+ Assert.assertEquals(project.columnModel.columns.size(), 2);
+ Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
+ Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album");
+ Assert.assertEquals(project.rows.size(), 1);
+ Assert.assertEquals(project.rows.get(0).cells.size(), 2);
+ Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
+ Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks");
+ }
+
+ @Test
+ public void CanParseMultiLineTriple(){
+ String sampleRdf = " .\n" +
+ " .\n" +
+ " .";
+ StringReader reader = new StringReader(sampleRdf);
+
+ try {
+ SUT.read(reader, project, options);
+ project.update();
+ } catch (Exception e) {
+ Assert.fail();
+ }
+
+ //columns
+ Assert.assertEquals(project.columnModel.columns.size(), 2);
+ Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
+ Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album");
+
+ //rows
+ Assert.assertEquals(project.rows.size(), 3);
+
+ //row0
+ Assert.assertEquals(project.rows.get(0).cells.size(), 2);
+ Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
+ Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks");
+
+ //row1
+ Assert.assertEquals(project.rows.get(1).cells.size(), 2);
+ Assert.assertNull(project.rows.get(1).cells.get(0));
+ Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); //NB triples aren't created in order they were input
+ Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0);
+ Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0);
+
+ //row2
+ Assert.assertEquals(project.rows.get(2).cells.size(), 2);
+ Assert.assertNull(project.rows.get(2).cells.get(0));
+ Assert.assertEquals(project.rows.get(2).cells.get(1).value, "http://rdf.freebase.com/ns/en.under_the_red_sky"); //NB triples aren't created in order they were input
+ Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].rowIndex, 0);
+ Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].cellIndex, 0);
+ }
+
+ @Test
+ public void CanParseMultiLineMultiPredicatesTriple(){
+ String sampleRdf = " .\n" +
+ " .\n" +
+ " .";
+ StringReader reader = new StringReader(sampleRdf);
+
+ try {
+ SUT.read(reader, project, options);
+ project.update();
+ } catch (Exception e) {
+ Assert.fail();
+ }
+
+ //columns
+ Assert.assertEquals(project.columnModel.columns.size(), 3);
+ Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
+ Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album");
+ Assert.assertEquals(project.columnModel.columns.get(2).getName(), "http://rdf.freebase.com/ns/music.artist.genre");
+
+ //rows
+ Assert.assertEquals(project.rows.size(), 2);
+
+ //row0
+ Assert.assertEquals(project.rows.get(0).cells.size(), 3);
+ Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
+ Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks");
+ Assert.assertEquals(project.rows.get(0).cells.get(2).value, "http://rdf.freebase.com/ns/en.folk_rock");
+
+ //row1
+ Assert.assertEquals(project.rows.get(1).cells.size(), 2);
+ Assert.assertNull(project.rows.get(1).cells.get(0));
+ Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home");
+ Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0);
+ Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0);
+ }
+
+ @Test
+ public void CanParseTripleWithValue(){
+ String sampleRdf = " \"Robert Zimmerman\"@en.";
+ StringReader reader = new StringReader(sampleRdf);
+
+ try {
+ SUT.read(reader, project, options);
+ project.update();
+ } catch (Exception e) {
+ Assert.fail();
+ }
+
+ Assert.assertEquals(project.columnModel.columns.size(), 2);
+ Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
+ Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/common.topic.alias");
+ Assert.assertEquals(project.rows.size(), 1);
+ Assert.assertEquals(project.rows.get(0).cells.size(), 2);
+ Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
+ Assert.assertEquals(project.rows.get(0).cells.get(1).value, "\"Robert Zimmerman\"@en");
+ }
+}
diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/importers/TestTools.java b/main/tests/server/src/com/metaweb/gridworks/tests/importers/TestTools.java
new file mode 100644
index 000000000..68fb8321a
--- /dev/null
+++ b/main/tests/server/src/com/metaweb/gridworks/tests/importers/TestTools.java
@@ -0,0 +1,50 @@
+package com.metaweb.gridworks.tests.importers;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+
+import com.metaweb.gridworks.model.Cell;
+import com.metaweb.gridworks.model.Column;
+import com.metaweb.gridworks.model.Project;
+import com.metaweb.gridworks.model.Row;
+
+/**
+ * Helper methods for Importer testing
+ *
+ */
+public class TestTools {
+ final static Logger logger = LoggerFactory.getLogger("Common");
+
+ public static void AssertGridCreated(Project project, int numCols, int numRows){
+ Assert.assertNotNull(project);
+ Assert.assertNotNull(project.columnModel);
+ Assert.assertNotNull(project.columnModel.columns);
+ Assert.assertEquals(project.columnModel.columns.size(), numCols);
+ Assert.assertNotNull(project.rows);
+ Assert.assertEquals(project.rows.size(), numRows);
+ }
+
+ public static void PrintProject(Project project){
+ //some quick and dirty debugging
+ StringBuilder sb = new StringBuilder();
+ for(Column c : project.columnModel.columns){
+ sb.append(c.getName());
+ sb.append("; ");
+ }
+ logger.info(sb.toString());
+ for(Row r : project.rows){
+ sb = new StringBuilder();
+ for(int i = 0; i < r.cells.size(); i++){
+ Cell c = r.getCell(i);
+ if(c != null){
+ sb.append(c.value);
+ sb.append("; ");
+ }else{
+ sb.append("null; ");
+ }
+ }
+ logger.info(sb.toString());
+ }
+ }
+}
diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesStub.java b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesStub.java
new file mode 100644
index 000000000..a35cdb6e7
--- /dev/null
+++ b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesStub.java
@@ -0,0 +1,32 @@
+package com.metaweb.gridworks.tests.importers;
+
+import java.util.List;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+
+import com.metaweb.gridworks.importers.XmlImportUtilities;
+import com.metaweb.gridworks.model.Project;
+
+
+public class XmlImportUtilitiesStub extends XmlImportUtilities{
+ public List detectRecordElementWrapper(XMLStreamReader parser, String tag) throws XMLStreamException{
+ return super.detectRecordElement(parser, tag);
+ }
+
+ public void ProcessSubRecordWrapper(Project project, XMLStreamReader parser, ImportColumnGroup columnGroup, ImportRecord record) throws XMLStreamException{
+ super.processSubRecord(project, parser, columnGroup, record);
+ }
+
+ public void findRecordWrapper(Project project, XMLStreamReader parser, String[] recordPath, int pathIndex, ImportColumnGroup rootColumnGroup) throws XMLStreamException{
+ super.findRecord(project, parser, recordPath, pathIndex, rootColumnGroup);
+ }
+
+ public void processRecordWrapper(Project project, XMLStreamReader parser, ImportColumnGroup rootColumnGroup) throws XMLStreamException{
+ super.processRecord(project, parser, rootColumnGroup);
+ }
+
+ public void addCellWrapper(Project project, ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, String text, int commonStartingRowIndex){
+ super.addCell(project, columnGroup, record, columnLocalName, text, commonStartingRowIndex);
+ }
+}
diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesTests.java b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesTests.java
new file mode 100644
index 000000000..c21c94314
--- /dev/null
+++ b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesTests.java
@@ -0,0 +1,336 @@
+package com.metaweb.gridworks.tests.importers;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.stream.FactoryConfigurationError;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+
+import org.apache.log4j.Level;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.metaweb.gridworks.importers.XmlImportUtilities.ImportColumn;
+import com.metaweb.gridworks.importers.XmlImportUtilities.ImportColumnGroup;
+import com.metaweb.gridworks.importers.XmlImportUtilities.ImportRecord;
+import com.metaweb.gridworks.model.Project;
+import com.metaweb.gridworks.model.Row;
+
+
+public class XmlImportUtilitiesTests {
+ final static Logger logger = LoggerFactory.getLogger("XmlImporterUtilitiesTests");
+
+ //dependencies
+ Project project;
+ XMLStreamReader parser;
+ ImportColumnGroup columnGroup;
+ ImportRecord record;
+ ByteArrayInputStream inputStream;
+
+ //System Under Test
+ XmlImportUtilitiesStub SUT;
+
+ @BeforeMethod
+ public void SetUp(){
+ org.apache.log4j.Logger.getRootLogger().setLevel(Level.toLevel("trace"));
+ SUT = new XmlImportUtilitiesStub();
+ project = new Project();
+ columnGroup = new ImportColumnGroup();
+ record = new ImportRecord();
+ }
+
+ @AfterMethod
+ public void TearDown() throws IOException{
+ SUT = null;
+ project = null;
+ parser = null;
+ columnGroup = null;
+ record = null;
+ if(inputStream != null)
+ inputStream.close();
+ inputStream = null;
+ }
+
+ @Test
+ public void detectPathFromTagTest(){
+ loadXml("author1genre1");
+ String tag = "library";
+
+ String[] response = XmlImportUtilitiesStub.detectPathFromTag(inputStream, tag);
+ Assert.assertNotNull(response);
+ Assert.assertEquals(response.length, 1);
+ Assert.assertEquals(response[0], "library");
+ }
+
+ @Test
+ public void detectPathFromTagWithNestedElement(){
+ loadXml("author1genre1");
+ String tag = "book";
+ String[] response = XmlImportUtilitiesStub.detectPathFromTag(inputStream, tag);
+ Assert.assertNotNull(response);
+ Assert.assertEquals(response.length, 2);
+ Assert.assertEquals(response[0], "library");
+ Assert.assertEquals(response[1], "book");
+ }
+
+ @Test
+ public void detectRecordElementTest(){
+ loadXml("author1genre1");
+ createParser();
+ String tag="library";
+
+ List response = new ArrayList();
+ try {
+ response = SUT.detectRecordElementWrapper(parser, tag);
+ } catch (XMLStreamException e) {
+ Assert.fail();
+ }
+ Assert.assertNotNull(response);
+ Assert.assertEquals(response.size(), 1);
+ Assert.assertEquals(response.get(0), "library");
+ }
+
+ @Test
+ public void detectRecordElementCanHandleWithNestedElements(){
+ loadXml("author1genre1");
+ createParser();
+ String tag="book";
+
+ List response = new ArrayList();
+ try {
+ response = SUT.detectRecordElementWrapper(parser, tag);
+ } catch (XMLStreamException e) {
+ Assert.fail();
+ }
+ Assert.assertNotNull(response);
+ Assert.assertEquals(response.size(), 2);
+ Assert.assertEquals(response.get(0), "library");
+ Assert.assertEquals(response.get(1), "book");
+ }
+
+ @Test
+ public void detectRecordElementIsNullForUnfoundTag(){
+ loadXml("author1genre1");
+ createParser();
+ String tag="";
+
+ List response = new ArrayList();
+ try {
+ response = SUT.detectRecordElementWrapper(parser, tag);
+ } catch (XMLStreamException e) {
+ Assert.fail();
+ }
+ Assert.assertNull(response);
+ }
+
+ @Test
+ public void detectRecordElementRegressionTest(){
+ loadSampleXml();
+
+ String[] path = XmlImportUtilitiesStub.detectRecordElement(inputStream);
+ Assert.assertNotNull(path);
+ Assert.assertEquals(path.length, 2);
+ Assert.assertEquals(path[0], "library");
+ Assert.assertEquals(path[1], "book");
+ }
+
+ @Test
+ public void importXmlTest(){
+ loadSampleXml();
+
+ String[] recordPath = new String[]{"library","book"};
+ XmlImportUtilitiesStub.importXml(inputStream, project, recordPath, columnGroup );
+
+ TestTools.PrintProject(project);
+ TestTools.AssertGridCreated(project, 0, 6);
+ Assert.assertEquals(project.rows.get(0).cells.size(), 4);
+ //TODO
+ }
+
+ @Test
+ public void createColumnsFromImportTest(){
+
+ ImportColumnGroup columnGroup = new ImportColumnGroup();
+ ImportColumn ic1 = new ImportColumn();
+ ic1.name = "hello";
+ ImportColumn ic2 = new ImportColumn();
+ ic2.name = "world";
+ ImportColumnGroup subGroup = new ImportColumnGroup();
+ ImportColumn ic3 = new ImportColumn();
+ ic3.name = "foo";
+ ImportColumn ic4 = new ImportColumn();
+ ic4.name = "bar";
+ subGroup.columns.put("c", ic3);
+ subGroup.columns.put("d", ic4);
+ columnGroup.columns.put("a", ic1);
+ columnGroup.columns.put("b", ic2);
+ columnGroup.subgroups.put("e", subGroup);
+ XmlImportUtilitiesStub.createColumnsFromImport(project, columnGroup);
+ TestTools.PrintProject(project);
+ TestTools.AssertGridCreated(project, 4, 0);
+ Assert.assertEquals(project.columnModel.columns.get(0).getName(), "world");
+ Assert.assertEquals(project.columnModel.columns.get(1).getName(), "hello");
+ Assert.assertEquals(project.columnModel.columns.get(2).getName(), "bar");
+ Assert.assertEquals(project.columnModel.columns.get(3).getName(), "foo");
+ Assert.assertEquals(project.columnModel.columnGroups.get(0).keyColumnIndex, 2);
+ Assert.assertEquals(project.columnModel.columnGroups.get(0).startColumnIndex, 2);
+ Assert.assertEquals(project.columnModel.columnGroups.get(0).columnSpan, 2);
+ }
+
+ @Test
+ public void findRecordTest(){
+ loadSampleXml();
+ createParser();
+ ParserSkip();
+
+ String[] recordPath = new String[]{"library","book"};
+ int pathIndex = 0;
+
+ try {
+ SUT.findRecordWrapper(project, parser, recordPath, pathIndex, columnGroup);
+ } catch (XMLStreamException e) {
+ Assert.fail();
+ }
+
+ TestTools.PrintProject(project);
+ TestTools.AssertGridCreated(project, 0, 6);
+ Assert.assertEquals(project.rows.get(0).cells.size(), 4);
+ //TODO
+ }
+
+ @Test
+ public void processRecordTest(){
+ loadXml("author1genre1");
+ createParser();
+ ParserSkip();
+
+ try {
+ SUT.processRecordWrapper(project, parser, columnGroup);
+ } catch (XMLStreamException e) {
+ Assert.fail();
+ }
+ TestTools.PrintProject(project);
+ Assert.assertNotNull(project.rows);
+ Assert.assertEquals(project.rows.size(), 1);
+ Row row = project.rows.get(0);
+ Assert.assertNotNull(row);
+ Assert.assertNotNull(row.getCell(1));
+ Assert.assertEquals(row.getCell(1).value, "author1");
+
+ }
+
+ @Test
+ public void processRecordTestDuplicateColumns(){
+ loadXml("author1author2genre1");
+ createParser();
+ ParserSkip();
+
+ try {
+ SUT.processRecordWrapper(project, parser, columnGroup);
+ } catch (XMLStreamException e) {
+ Assert.fail();
+ }
+ TestTools.PrintProject(project);
+ Assert.assertNotNull(project.rows);
+ Assert.assertEquals(project.rows.size(), 2);
+ Row row = project.rows.get(0);
+ Assert.assertNotNull(row);
+ Assert.assertEquals(row.cells.size(), 3);
+ Assert.assertNotNull(row.getCell(1));
+ Assert.assertEquals(row.getCell(1).value, "author1");
+ row = project.rows.get(1);
+ Assert.assertEquals(row.getCell(1).value, "author2");
+ }
+
+ @Test
+ public void processRecordTestNestedElement(){
+ loadXml("author1a dategenre1");
+ createParser();
+ ParserSkip();
+
+ try {
+ SUT.processRecordWrapper(project, parser, columnGroup);
+ } catch (XMLStreamException e) {
+ Assert.fail();
+ }
+ TestTools.PrintProject(project);
+ Assert.assertNotNull(project.rows);
+ Assert.assertEquals(project.rows.size(), 1);
+ Row row = project.rows.get(0);
+ Assert.assertNotNull(row);
+ Assert.assertEquals(row.cells.size(), 4);
+ Assert.assertNotNull(row.getCell(1));
+ Assert.assertEquals(row.getCell(1).value, "author1");
+ Assert.assertNotNull(row.getCell(2));
+ Assert.assertEquals(row.getCell(2).value, "a date");
+ }
+
+
+ @Test(groups={"broken"})
+ public void processSubRecordTest(){
+ loadXml("author1genre1");
+ createParser();
+ ParserSkip();
+
+ try {
+ SUT.ProcessSubRecordWrapper(project, parser, columnGroup, record);
+ } catch (XMLStreamException e) {
+ Assert.fail();
+ }
+ TestTools.PrintProject(project);
+ Assert.fail();
+ //TODO need to verify 'record' was set correctly which we can't do as ImportRecord is an internal class
+ }
+
+ @Test(groups={"broken"})
+ public void addCellTest(){
+ String columnLocalName = "author";
+ String text = "Author1, The";
+ int commonStartingRowIndex = 0;
+ project.rows.add(new Row(0));
+ SUT.addCellWrapper(project, columnGroup, record, columnLocalName, text, commonStartingRowIndex);
+
+ Assert.fail();
+ //TODO need to verify 'record' was set correctly which we can't do as ImportRecord is an internal class
+ }
+
+ //----------------helpers-------------
+ public void loadSampleXml(){
+ loadXml( XmlImporterTests.getSample() );
+ }
+
+ public void loadXml(String xml){
+ try {
+ inputStream = new ByteArrayInputStream( xml.getBytes( "UTF-8" ) );
+ } catch (UnsupportedEncodingException e1) {
+ Assert.fail();
+ }
+ }
+
+ public void ParserSkip(){
+ try {
+ parser.next(); //move parser forward once e.g. skip the START_DOCUMENT parser event
+ } catch (XMLStreamException e1) {
+ Assert.fail();
+ }
+ }
+
+ public void createParser(){
+ try {
+ parser = XMLInputFactory.newInstance().createXMLStreamReader(inputStream);
+ } catch (XMLStreamException e1) {
+ Assert.fail();
+ } catch (FactoryConfigurationError e1) {
+ Assert.fail();
+ }
+ }
+}
diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImporterTests.java b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImporterTests.java
index e4272b6bf..613d5c80d 100644
--- a/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImporterTests.java
+++ b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImporterTests.java
@@ -1,207 +1,225 @@
-package com.metaweb.gridworks.tests.importers;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.util.Properties;
-
-import static org.mockito.Mockito.mock;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import com.metaweb.gridworks.importers.XmlImporter;
-import com.metaweb.gridworks.model.Cell;
-import com.metaweb.gridworks.model.Column;
-import com.metaweb.gridworks.model.Project;
-import com.metaweb.gridworks.model.Row;
-
-
-public class XmlImporterTests {
- final static Logger logger = LoggerFactory.getLogger("XmlImporterTests");
-
- //dependencies
- Project project = null;
- Properties options = null;
- ByteArrayInputStream inputStream = null;
-
- //System Under Test
- XmlImporter SUT = null;
-
-
- @BeforeMethod
- public void SetUp(){
- SUT = new XmlImporter();
- project = new Project();
- options = mock(Properties.class);
- }
-
- @AfterMethod
- public void TearDown(){
- SUT = null;
- project = null;
- options = null;
- }
-
- @Test
- public void canParseSample(){
-
- RunTest(getSample());
-
- AssertGridCreate(project, 4, 6);
- PrintProject(project);
-
- Row row = project.rows.get(0);
- Assert.assertNotNull(row);
- Assert.assertNotNull(row.cells);
- Assert.assertNotNull(row.cells.get(2));
- Assert.assertEquals(row.cells.get(2).value, "Author 1, The");
-
-
- }
-
- @Test
- public void testCanParseLineBreak(){
-
- RunTest(getSampleWithLineBreak());
-
- AssertGridCreate(project, 4, 6);
- PrintProject(project);
-
- Row row = project.rows.get(3);
- Assert.assertNotNull(row);
- Assert.assertNotNull(row.cells);
- Assert.assertNotNull(row.cells.get(2));
- Assert.assertEquals(row.cells.get(2).value, "With line\n break");
- }
-
- @Test(groups={"broken"})
- public void testElementsWithVaryingStructure(){
-
-
- RunTest(getSampleWithVaryingStructure());
-
- AssertGridCreate(project, 5, 6);
- PrintProject(project);
-
- Row row0 = project.rows.get(0);
- Assert.assertNotNull(row0);
- Assert.assertNotNull(row0.cells);
- Assert.assertEquals(row0.cells.size(),6);
-
- Row row5 = project.rows.get(5);
- Assert.assertNotNull(row5);
- Assert.assertNotNull(row5.cells);
- Assert.assertEquals(row5.cells.size(),6);
-
-
- }
-
- //------------helper methods---------------
-
- protected String getTypicalElement(int id){
- return "" +
- "Author " + id + ", The" +
- "Book title " + id + "" +
- "2010-05-26" +
- "";
- }
-
- protected String getSample(){
- StringBuilder sb = new StringBuilder();
- sb.append("");
- for(int i = 1; i < 7; i++){
- sb.append(getTypicalElement(i));
- }
- sb.append("");
- return sb.toString();
- }
-
- protected String getSampleWithLineBreak(){
- StringBuilder sb = new StringBuilder();
- sb.append("");
- for(int i = 1; i < 4; i++){
- sb.append(getTypicalElement(i));
- }
- sb.append("" +
- "With line\n break" +
- "Book title 4" +
- "2010-05-26" +
- "");
- sb.append(getTypicalElement(5));
- sb.append(getTypicalElement(6));
- sb.append("");
- return sb.toString();
- }
-
- protected String getSampleWithVaryingStructure(){
- StringBuilder sb = new StringBuilder();
- sb.append("");
- for(int i = 1; i < 6; i++){
- sb.append(getTypicalElement(i));
- }
- sb.append("" +
- "With line\n break" +
- "Book title 6" +
- "New element not seen in other records" +
- "2010-05-26" +
- "");
- sb.append("");
- return sb.toString();
- }
-
- private void RunTest(String testString){
- try {
- inputStream = new ByteArrayInputStream( testString.getBytes( "UTF-8" ) );
- } catch (UnsupportedEncodingException e1) {
- Assert.fail();
- }
-
- try {
- SUT.read(inputStream, project, options);
- } catch (Exception e) {
- Assert.fail();
- }
-
- try {
- inputStream.close();
- } catch (IOException e) {
- Assert.fail();
- }
- }
-
- private void AssertGridCreate(Project project, int numCols, int numRows){
- Assert.assertNotNull(project);
- Assert.assertNotNull(project.columnModel);
- Assert.assertNotNull(project.columnModel.columns);
- Assert.assertEquals(project.columnModel.columns.size(), numCols);
- Assert.assertNotNull(project.rows);
- Assert.assertEquals(project.rows.size(), numRows);
- }
-
- private void PrintProject(Project project){
- //some quick and dirty debugging
- StringBuilder sb = new StringBuilder();
- for(Column c : project.columnModel.columns){
- sb.append(c.getName());
- sb.append("; ");
- }
- logger.info(sb.toString());
- for(Row r : project.rows){
- sb = new StringBuilder();
- for(Cell c : r.cells){
- if(c != null){
- sb.append(c.value);
- sb.append("; ");
- }else{
- sb.append("null; ");
- }
- }
- logger.info(sb.toString());
- }
- }
-}
+package com.metaweb.gridworks.tests.importers;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Properties;
+
+import static org.mockito.Mockito.mock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.metaweb.gridworks.importers.XmlImporter;
+import com.metaweb.gridworks.model.Project;
+import com.metaweb.gridworks.model.Row;
+
+
+public class XmlImporterTests {
+ final static Logger logger = LoggerFactory.getLogger("XmlImporterTests");
+
+ //dependencies
+ Project project = null;
+ Properties options = null;
+ ByteArrayInputStream inputStream = null;
+
+ //System Under Test
+ XmlImporter SUT = null;
+
+
+ @BeforeMethod
+ public void SetUp(){
+ SUT = new XmlImporter();
+ project = new Project();
+ options = mock(Properties.class);
+ }
+
+ @AfterMethod
+ public void TearDown() throws IOException{
+ SUT = null;
+ project = null;
+ options = null;
+ inputStream.close();
+ inputStream = null;
+ }
+
+ @Test
+ public void canParseSample(){
+ RunTest(getSample());
+
+ TestTools.AssertGridCreated(project, 4, 6);
+ TestTools.PrintProject(project);
+
+ Row row = project.rows.get(0);
+ Assert.assertNotNull(row);
+ Assert.assertNotNull(row.getCell(1));
+ Assert.assertEquals(row.getCell(1).value, "Author 1, The");
+ }
+
+ @Test
+ public void canParseSampleWithDuplicateNestedElements(){
+ RunTest(getSampleWithDuplicateNestedElements());
+
+ TestTools.PrintProject(project);
+ TestTools.AssertGridCreated(project, 4, 12);
+
+ Row row = project.rows.get(0);
+ Assert.assertNotNull(row);
+ Assert.assertEquals(row.cells.size(), 4);
+ Assert.assertNotNull(row.getCell(2));
+ Assert.assertEquals(row.getCell(1).value, "Author 1, The");
+ Assert.assertEquals(project.rows.get(1).getCell(1).value, "Author 1, Another");
+ }
+
+ @Test
+ public void testCanParseLineBreak(){
+
+ RunTest(getSampleWithLineBreak());
+
+ TestTools.AssertGridCreated(project, 4, 6);
+ TestTools.PrintProject(project);
+
+ Row row = project.rows.get(3);
+ Assert.assertNotNull(row);
+ Assert.assertEquals(row.cells.size(), 4);
+ Assert.assertNotNull(row.getCell(1));
+ Assert.assertEquals(row.getCell(1).value, "With line\n break");
+ }
+
+ @Test(groups={"broken"})
+ public void testElementsWithVaryingStructure(){
+ RunTest(getSampleWithVaryingStructure());
+
+ TestTools.AssertGridCreated(project, 5, 6);
+ TestTools.PrintProject(project);
+
+ Row row0 = project.rows.get(0);
+ Assert.assertNotNull(row0);
+ Assert.assertEquals(row0.cells.size(),6);
+
+ Row row5 = project.rows.get(5);
+ Assert.assertNotNull(row5);
+ Assert.assertEquals(row5.cells.size(),6);
+ }
+
+ @Test
+ public void testElementWithNestedTree(){
+ RunTest(getSampleWithTreeStructure());
+ TestTools.AssertGridCreated(project, 5, 6);
+ TestTools.PrintProject(project);
+ Assert.assertEquals(project.columnModel.columnGroups.size(),1);
+ Assert.assertEquals(project.columnModel.columnGroups.get(0).keyColumnIndex, 2);
+ Assert.assertEquals(project.columnModel.columnGroups.get(0).startColumnIndex, 2);
+ Assert.assertNull(project.columnModel.columnGroups.get(0).parentGroup);
+ Assert.assertEquals(project.columnModel.columnGroups.get(0).subgroups.size(),0);
+ Assert.assertEquals(project.columnModel.columnGroups.get(0).columnSpan,2);
+ }
+
+ //------------helper methods---------------
+
+ public static String getTypicalElement(int id){
+ return "" +
+ "Author " + id + ", The" +
+ "Book title " + id + "" +
+ "2010-05-26" +
+ "";
+ }
+
+ public static String getElementWithDuplicateSubElement(int id){
+ return "" +
+ "Author " + id + ", The" +
+ "Author " + id + ", Another" +
+ "Book title " + id + "" +
+ "2010-05-26" +
+ "";
+ }
+
+ public static String getSample(){
+ StringBuilder sb = new StringBuilder();
+ sb.append("");
+ for(int i = 1; i < 7; i++){
+ sb.append(getTypicalElement(i));
+ }
+ sb.append("");
+ return sb.toString();
+ }
+
+ public static String getSampleWithDuplicateNestedElements(){
+ StringBuilder sb = new StringBuilder();
+ sb.append("");
+ for(int i = 1; i < 7; i++){
+ sb.append(getElementWithDuplicateSubElement(i));
+ }
+ sb.append("");
+ return sb.toString();
+
+ }
+
+ public static String getSampleWithLineBreak(){
+ StringBuilder sb = new StringBuilder();
+ sb.append("");
+ for(int i = 1; i < 4; i++){
+ sb.append(getTypicalElement(i));
+ }
+ sb.append("" +
+ "With line\n break" +
+ "Book title 4" +
+ "2010-05-26" +
+ "");
+ sb.append(getTypicalElement(5));
+ sb.append(getTypicalElement(6));
+ sb.append("");
+ return sb.toString();
+ }
+
+ public static String getSampleWithVaryingStructure(){
+ StringBuilder sb = new StringBuilder();
+ sb.append("");
+ for(int i = 1; i < 6; i++){
+ sb.append(getTypicalElement(i));
+ }
+ sb.append("" +
+ "Author 6, The" +
+ "Book title 6" +
+ "New element not seen in other records" +
+ "2010-05-26" +
+ "");
+ sb.append("");
+ return sb.toString();
+ }
+
+ public static String getSampleWithTreeStructure(){
+ StringBuilder sb = new StringBuilder();
+ sb.append("");
+ for(int i = 1; i < 7; i++){
+ sb.append("" +
+ "Author " + i + ", The" +
+ "1950-0" + i + "-15" +
+ "Book title " + i + "" +
+ "2010-05-26" +
+ "");
+ }
+ sb.append("");
+ return sb.toString();
+ }
+
+ private void RunTest(String testString){
+ try {
+ inputStream = new ByteArrayInputStream( testString.getBytes( "UTF-8" ) );
+ } catch (UnsupportedEncodingException e1) {
+ Assert.fail();
+ }
+
+ try {
+ SUT.read(inputStream, project, options);
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+
+}
diff --git a/main/tests/server/src/log4j.properties b/main/tests/server/src/log4j.properties
index 7ccee1634..adaedaa02 100644
--- a/main/tests/server/src/log4j.properties
+++ b/main/tests/server/src/log4j.properties
@@ -1,4 +1,7 @@
log4j.rootLogger=ERROR, console
+log4j.logger.org.apache.http.headers=WARN
+log4j.logger.org.apache.http.impl=WARN
+log4j.logger.org.apache.http.client=WARN
log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.layout=com.metaweb.util.logging.IndentingLayout
+log4j.appender.console.layout=com.metaweb.gridworks.logging.IndentingLayout
diff --git a/main/webapp/WEB-INF/web.xml b/main/webapp/WEB-INF/web.xml
index 7c9d17f30..d3f38fef9 100644
--- a/main/webapp/WEB-INF/web.xml
+++ b/main/webapp/WEB-INF/web.xml
@@ -8,7 +8,6 @@
gridworks
com.metaweb.gridworks.GridworksServlet
- 1
diff --git a/server/IDEs/eclipse/Gridworks.launch b/server/IDEs/eclipse/Gridworks.launch
index 003f3a3c0..bde9d67c1 100644
--- a/server/IDEs/eclipse/Gridworks.launch
+++ b/server/IDEs/eclipse/Gridworks.launch
@@ -1,12 +1,13 @@
-
+
-
-
+
+
+
diff --git a/server/src/com/metaweb/gridworks/Gridworks.java b/server/src/com/metaweb/gridworks/Gridworks.java
index f49b3a06d..011ba1286 100644
--- a/server/src/com/metaweb/gridworks/Gridworks.java
+++ b/server/src/com/metaweb/gridworks/Gridworks.java
@@ -24,11 +24,14 @@ import org.apache.log4j.Level;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
+import org.mortbay.jetty.servlet.ServletHolder;
import org.mortbay.jetty.webapp.WebAppContext;
import org.mortbay.util.Scanner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.codeberry.jdatapath.DataPath;
+import com.codeberry.jdatapath.JDataPathSystem;
import com.metaweb.util.threads.ThreadPoolExecutorAdapter;
/**
@@ -37,38 +40,15 @@ import com.metaweb.util.threads.ThreadPoolExecutorAdapter;
*/
public class Gridworks {
- static private final String VERSION = "1.0";
static private final String DEFAULT_HOST = "127.0.0.1";
static private final int DEFAULT_PORT = 3333;
- static private final int MAX_UPLOAD_SIZE = 1024 * 1024 * 1024;
-
- static private File tempDir;
+
static private int port;
static private String host;
final static Logger logger = LoggerFactory.getLogger("gridworks");
-
- public static String getVersion() {
- return VERSION;
- }
-
- public static File getTempFile(String name) {
- return new File(tempDir, name);
- }
-
- public static File getTempDir() {
- return tempDir;
- }
-
- public static int getMaxUploadSize() {
- return Configurations.getInteger("gridworks.max_upload_size",MAX_UPLOAD_SIZE);
- }
-
- public static String getFullHost() {
- return host + ":" + port;
- }
-
- public static void main(String[] args) throws Exception {
+
+ public static void main(String[] args) throws Exception {
// tell jetty to use SLF4J for logging instead of its own stuff
System.setProperty("VERBOSE","false");
@@ -89,9 +69,6 @@ public class Gridworks {
// set the log verbosity level
org.apache.log4j.Logger.getRootLogger().setLevel(Level.toLevel(Configurations.get("gridworks.verbosity","info")));
- tempDir = new File(Configurations.get("gridworks.temp","temp"));
- if (!tempDir.exists()) tempDir.mkdirs();
-
port = Configurations.getInteger("gridworks.port",DEFAULT_PORT);
host = Configurations.get("gridworks.host",DEFAULT_HOST);
@@ -117,7 +94,8 @@ public class Gridworks {
// hook up the signal handlers
Runtime.getRuntime().addShutdownHook(
- new Thread(new ShutdownSignalHandler(server)));
+ new Thread(new ShutdownSignalHandler(server))
+ );
server.join();
}
@@ -178,7 +156,15 @@ class GridworksServer extends Server {
scanForUpdates(contextRoot, context);
}
+ // start the server
this.start();
+
+ // inject configuration parameters in the servlets
+ // NOTE: this is done *after* starting the server because jetty might override the init
+ // parameters if we set them in the webapp context upon reading the web.xml file
+ ServletHolder servlet = context.getServletHandler().getServlet("gridworks");
+ servlet.setInitParameter("gridworks.data", getDataDir());
+ servlet.doStart();
}
@Override
@@ -194,7 +180,7 @@ class GridworksServer extends Server {
}
}
- private void scanForUpdates(final File contextRoot, final WebAppContext context) {
+ static private void scanForUpdates(final File contextRoot, final WebAppContext context) {
List scanList = new ArrayList();
scanList.add(new File(contextRoot, "WEB-INF/web.xml"));
@@ -225,7 +211,7 @@ class GridworksServer extends Server {
scanner.start();
}
- private void findFiles(final String extension, File baseDir, final Collection found) {
+ static private void findFiles(final String extension, File baseDir, final Collection found) {
baseDir.listFiles(new FileFilter() {
public boolean accept(File pathname) {
if (pathname.isDirectory()) {
@@ -238,6 +224,103 @@ class GridworksServer extends Server {
});
}
+ static private String getDataDir() {
+
+ String data_dir = Configurations.get("gridworks.data_dir");
+ if (data_dir != null) {
+ return data_dir;
+ }
+
+ String os = System.getProperty("os.name").toLowerCase();
+ if (os.contains("windows")) {
+ try {
+ // NOTE(SM): finding the "local data app" in windows from java is actually a PITA
+ // see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
+ // so we're using a library that uses JNI to ask directly the win32 APIs,
+ // it's not elegant but it's the safest bet.
+
+ DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
+ File data = new File(fixWindowsUnicodePath(localDataPath.getPath()));
+ data.mkdirs();
+ return data.getAbsolutePath();
+ } catch (Error e) {
+ /*
+ * The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll
+ * we include is compiled for 32-bit. In this case, we just have to dig up
+ * environment variables and try our best to find a user-specific path.
+ */
+
+ logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables");
+
+ File parentDir = null;
+ String appData = System.getenv("APPDATA");
+ if (appData != null && appData.length() > 0) {
+ // e.g., C:\Users\[userid]\AppData\Roaming
+ parentDir = new File(appData);
+ } else {
+ String userProfile = System.getenv("USERPROFILE");
+ if (userProfile != null && userProfile.length() > 0) {
+ // e.g., C:\Users\[userid]
+ parentDir = new File(userProfile);
+ }
+ }
+
+ if (parentDir == null) {
+ parentDir = new File(".");
+ }
+
+ File data = new File(parentDir, "Gridworks");
+ data.mkdirs();
+
+ return data.getAbsolutePath();
+ }
+ } else if (os.contains("mac os x")) {
+ // on macosx, use "~/Library/Application Support"
+ String home = System.getProperty("user.home");
+ String data_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks";
+ File data = new File(data_home);
+ data.mkdirs();
+ return data.getAbsolutePath();
+ } else { // most likely a UNIX flavor
+ // start with the XDG environment
+ // see http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+ String data_home = System.getenv("XDG_DATA_HOME");
+ if (data_home == null) { // if not found, default back to ~/.local/share
+ String home = System.getProperty("user.home");
+ if (home == null) home = ".";
+ data_home = home + "/.local/share";
+ }
+ File data = new File(data_home + "/gridworks");
+ data.mkdirs();
+ return data.getAbsolutePath();
+ }
+ }
+
+ /**
+ * For Windows file paths that contain user IDs with non ASCII characters,
+ * those characters might get replaced with ?. We need to use the environment
+ * APPDATA value to substitute back the original user ID.
+ */
+ static private String fixWindowsUnicodePath(String path) {
+ int q = path.indexOf('?');
+ if (q < 0) {
+ return path;
+ }
+ int pathSep = path.indexOf(File.separatorChar, q);
+
+ String goodPath = System.getenv("APPDATA");
+ if (goodPath == null || goodPath.length() == 0) {
+ goodPath = System.getenv("USERPROFILE");
+ if (!goodPath.endsWith(File.separator)) {
+ goodPath = goodPath + File.separator;
+ }
+ }
+
+ int goodPathSep = goodPath.indexOf(File.separatorChar, q);
+
+ return path.substring(0, q) + goodPath.substring(q, goodPathSep) + path.substring(pathSep);
+ }
+
}
/* -------------- Gridworks Client ----------------- */
diff --git a/server/src/log4j.properties b/server/src/log4j.properties
index 6d09a1788..fe5307d2f 100644
--- a/server/src/log4j.properties
+++ b/server/src/log4j.properties
@@ -1,9 +1,6 @@
log4j.rootLogger=INFO, console
log4j.logger.org.mortbay.log=WARN
log4j.logger.org.mortbay.jetty=ERROR
-log4j.logger.org.apache.http.headers=WARN
-log4j.logger.org.apache.http.impl=WARN
-log4j.logger.org.apache.http.client=WARN
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=com.metaweb.util.logging.IndentingLayout