diff --git a/.gitignore b/.gitignore index 1f2288b5d..16d995959 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ +.DS_Store build/ -src/main/webapp/WEB-INF/classes/ -tests/java/classes/ +server/classes/ +main/webapp/WEB-INF/classes/ +main/tests/server/classes/ +main/test-output/ +appengine/classes/ tools/ diff --git a/build.xml b/build.xml index 82097d3ea..2dc4a6b82 100644 --- a/build.xml +++ b/build.xml @@ -118,6 +118,7 @@ + @@ -125,6 +126,7 @@ + diff --git a/main/IDEs/eclipse/GridworksTests.launch b/main/IDEs/eclipse/GridworksTests.launch index 20ebd70ff..0cc57b89a 100644 --- a/main/IDEs/eclipse/GridworksTests.launch +++ b/main/IDEs/eclipse/GridworksTests.launch @@ -6,7 +6,6 @@ - @@ -15,7 +14,7 @@ - + diff --git a/main/src/com/metaweb/gridworks/GridworksServlet.java b/main/src/com/metaweb/gridworks/GridworksServlet.java index 46aef38b1..d884d5ef6 100644 --- a/main/src/com/metaweb/gridworks/GridworksServlet.java +++ b/main/src/com/metaweb/gridworks/GridworksServlet.java @@ -1,11 +1,13 @@ package com.metaweb.gridworks; +import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Timer; import java.util.TimerTask; +import javax.servlet.ServletConfig; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; @@ -18,8 +20,12 @@ import com.metaweb.gridworks.commands.Command; public class GridworksServlet extends HttpServlet { + static private final String VERSION = "1.0"; + private static final long serialVersionUID = 2386057901503517403L; + private static final String JAVAX_SERVLET_CONTEXT_TEMPDIR = "javax.servlet.context.tempdir"; + static final private Map commands = new HashMap(); // timer for periodically saving projects @@ -104,11 +110,12 @@ public class GridworksServlet extends HttpServlet { {"mqlwrite", "com.metaweb.gridworks.commands.freebase.MQLWriteCommand"}, }; - static { - registerCommands(commandNames); + public static String getVersion() { + return VERSION; } final static protected long s_autoSavePeriod = 1000 * 60 * 5; // 5 minutes + static protected class AutoSaveTimerTask extends TimerTask { public void run() { try { @@ -121,12 +128,21 @@ public class GridworksServlet extends HttpServlet { } } + protected ServletConfig config; + @Override public void init() throws ServletException { - super.init(); logger.trace("> initialize"); - ProjectManager.initialize(); + String data = getInitParameter("gridworks.data"); + + if (data == null) { + throw new ServletException("can't find servlet init config 'gridworks.data', I have to give up initializing"); + } + + registerCommands(commandNames); + + ProjectManager.initialize(new File(data)); if (_timer == null) { _timer = new Timer("autosave"); @@ -149,6 +165,8 @@ public class GridworksServlet extends HttpServlet { ProjectManager.singleton.save(true); // complete save ProjectManager.singleton = null; } + + this.config = null; super.destroy(); @@ -188,6 +206,26 @@ public class GridworksServlet extends HttpServlet { return slash > 0 ? commandName.substring(0, slash) : commandName; } + private File tempDir = null; + + public File getTempDir() { + if (tempDir == null) { + File tempDir = (File) this.config.getServletContext().getAttribute(JAVAX_SERVLET_CONTEXT_TEMPDIR); + if (tempDir == null) { + throw new RuntimeException("This app server doesn't support temp directories"); + } + } + return tempDir; + } + + public File getTempFile(String name) { + return new File(getTempDir(), name); + } + + public String getConfiguration(String name, String def) { + return null; + } + /** * Register an array of commands * @@ -198,7 +236,7 @@ public class GridworksServlet extends HttpServlet { * the second. * @return false if any commands failed to load */ - static public boolean registerCommands(String[][] commands) { + private boolean registerCommands(String[][] commands) { boolean status = true; for (String[] command : commandNames) { String commandName = command[0]; @@ -206,8 +244,8 @@ public class GridworksServlet extends HttpServlet { logger.debug("Loading command " + commandName + " class: " + className); Command cmd; try { - // TODO: May need to use the servlet container's class loader here - cmd = (Command) Class.forName(className).newInstance(); + cmd = (Command) this.getClass().getClassLoader().loadClass(className).newInstance(); + cmd.init(this); } catch (InstantiationException e) { logger.error("Failed to load command class " + className, e); status = false; @@ -235,8 +273,7 @@ public class GridworksServlet extends HttpServlet { * object implementing the command * @return true if command was loaded and registered successfully */ - static public boolean registerCommand(String name, - Command commandObject) { + protected boolean registerCommand(String name, Command commandObject) { if (commands.containsKey(name)) { return false; } @@ -245,7 +282,7 @@ public class GridworksServlet extends HttpServlet { } // Currently only for test purposes - static protected boolean unregisterCommand(String verb) { + protected boolean unregisterCommand(String verb) { return commands.remove(verb) != null; } } diff --git a/main/src/com/metaweb/gridworks/ProjectManager.java b/main/src/com/metaweb/gridworks/ProjectManager.java index a5885b819..19d0a32df 100644 --- a/main/src/com/metaweb/gridworks/ProjectManager.java +++ b/main/src/com/metaweb/gridworks/ProjectManager.java @@ -22,8 +22,6 @@ import org.json.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.codeberry.jdatapath.DataPath; -import com.codeberry.jdatapath.JDataPathSystem; import com.metaweb.gridworks.model.Project; import com.metaweb.gridworks.util.JSONUtilities; @@ -58,112 +56,13 @@ public class ProjectManager { static public ProjectManager singleton; - static public synchronized void initialize() { + static public synchronized void initialize(File dir) { if (singleton == null) { - File dir = getProjectLocation(); logger.info("Using workspace directory: {}", dir.getAbsolutePath()); - singleton = new ProjectManager(dir); } } - - static protected File getProjectLocation() { - String data_dir = Configurations.get("gridworks.data_dir"); - if (data_dir != null) { - return new File(data_dir); - } - String os = Configurations.get("os.name").toLowerCase(); - if (os.contains("windows")) { - try { - // NOTE(SM): finding the "local data app" in windows from java is actually a PITA - // see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java - // so we're using a library that uses JNI to ask directly the win32 APIs, - // it's not elegant but it's the safest bet. - - DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks"); - File data = new File(fixWindowsUnicodePath(localDataPath.getPath())); - data.mkdirs(); - return data; - } catch (Error e) { - /* - * The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll - * we include is compiled for 32-bit. In this case, we just have to dig up - * environment variables and try our best to find a user-specific path. - */ - - logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables"); - - File parentDir = null; - { - String appData = System.getenv("APPDATA"); - if (appData != null && appData.length() > 0) { - // e.g., C:\Users\[userid]\AppData\Roaming - parentDir = new File(appData); - } else { - String userProfile = System.getenv("USERPROFILE"); - if (userProfile != null && userProfile.length() > 0) { - // e.g., C:\Users\[userid] - parentDir = new File(userProfile); - } - } - } - if (parentDir == null) { - parentDir = new File("."); - } - - File data = new File(parentDir, "Gridworks"); - data.mkdirs(); - - return data; - } - } else if (os.contains("mac os x")) { - // on macosx, use "~/Library/Application Support" - String home = System.getProperty("user.home"); - String data_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks"; - File data = new File(data_home); - data.mkdirs(); - return data; - } else { // most likely a UNIX flavor - // start with the XDG environment - // see http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html - String data_home = System.getenv("XDG_DATA_HOME"); - if (data_home == null) { // if not found, default back to ~/.local/share - String home = System.getProperty("user.home"); - if (home == null) home = "."; - data_home = home + "/.local/share"; - } - File data = new File(data_home + "/gridworks"); - data.mkdirs(); - return data; - } - } - - /** - * For Windows file paths that contain user IDs with non ASCII characters, - * those characters might get replaced with ?. We need to use the environment - * APPDATA value to substitute back the original user ID. - */ - static protected String fixWindowsUnicodePath(String path) { - int q = path.indexOf('?'); - if (q < 0) { - return path; - } - int pathSep = path.indexOf(File.separatorChar, q); - - String goodPath = System.getenv("APPDATA"); - if (goodPath == null || goodPath.length() == 0) { - goodPath = System.getenv("USERPROFILE"); - if (!goodPath.endsWith(File.separator)) { - goodPath = goodPath + File.separator; - } - } - - int goodPathSep = goodPath.indexOf(File.separatorChar, q); - - return path.substring(0, q) + goodPath.substring(q, goodPathSep) + path.substring(pathSep); - } - private ProjectManager(File dir) { _workspaceDir = dir; _workspaceDir.mkdirs(); diff --git a/main/src/com/metaweb/gridworks/commands/Command.java b/main/src/com/metaweb/gridworks/commands/Command.java index 349eea62c..d998c4ed1 100644 --- a/main/src/com/metaweb/gridworks/commands/Command.java +++ b/main/src/com/metaweb/gridworks/commands/Command.java @@ -16,6 +16,7 @@ import org.json.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.metaweb.gridworks.GridworksServlet; import com.metaweb.gridworks.Jsonizable; import com.metaweb.gridworks.ProjectManager; import com.metaweb.gridworks.ProjectMetadata; @@ -33,6 +34,12 @@ public abstract class Command { final static protected Logger logger = LoggerFactory.getLogger("command"); + protected GridworksServlet servlet; + + public void init(GridworksServlet servlet) { + this.servlet = servlet; + } + public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { diff --git a/main/src/com/metaweb/gridworks/commands/auth/AuthorizeCommand.java b/main/src/com/metaweb/gridworks/commands/auth/AuthorizeCommand.java index 2c1ea053e..a5d8709a1 100644 --- a/main/src/com/metaweb/gridworks/commands/auth/AuthorizeCommand.java +++ b/main/src/com/metaweb/gridworks/commands/auth/AuthorizeCommand.java @@ -2,6 +2,8 @@ package com.metaweb.gridworks.commands.auth; import java.io.IOException; import java.io.PrintWriter; +import java.net.URI; +import java.net.URISyntaxException; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -10,13 +12,12 @@ import javax.servlet.http.HttpServletResponse; import oauth.signpost.OAuthConsumer; import oauth.signpost.OAuthProvider; -import com.metaweb.gridworks.Gridworks; import com.metaweb.gridworks.commands.Command; import com.metaweb.gridworks.oauth.Credentials; import com.metaweb.gridworks.oauth.OAuthUtilities; import com.metaweb.gridworks.oauth.Provider; -public class AuthorizeCommand extends Command { +public class AuthorizeCommand extends Command { private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier"; @@ -124,7 +125,21 @@ public class AuthorizeCommand extends Command { private String getBaseURL(HttpServletRequest request, Provider provider) { String host = request.getHeader("host"); - if (host == null) host = Gridworks.getFullHost(); + if (host == null) { + String referrer = request.getHeader("referer"); + if (referrer != null) { + URI url; + try { + url = new URI(referrer); + int port = url.getPort(); + host = url.getHost() + ((port > -1) ? ":" + url.getPort() : ""); + } catch (URISyntaxException e) { + throw new RuntimeException("referrer '" + referrer + "' can't be parsed as a URL"); + } + } else { + throw new RuntimeException("neither the 'host' nor 'referer' headers were present in the HTTP response, I can't determine what URL gridworks is listening to."); + } + } return "http://" + host + "/command/authorize/" + provider.getHost(); } } diff --git a/main/src/com/metaweb/gridworks/commands/project/CreateProjectCommand.java b/main/src/com/metaweb/gridworks/commands/project/CreateProjectCommand.java index c4e567ff4..36f87ec0a 100644 --- a/main/src/com/metaweb/gridworks/commands/project/CreateProjectCommand.java +++ b/main/src/com/metaweb/gridworks/commands/project/CreateProjectCommand.java @@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory; import com.ibm.icu.text.CharsetDetector; import com.ibm.icu.text.CharsetMatch; -import com.metaweb.gridworks.Gridworks; import com.metaweb.gridworks.ProjectManager; import com.metaweb.gridworks.ProjectMetadata; import com.metaweb.gridworks.commands.Command; @@ -367,7 +366,7 @@ public class CreateProjectCommand extends Command { } private File save(InputStream is) throws IOException { - File temp = Gridworks.getTempFile(Long.toString(System.currentTimeMillis())); + File temp = this.servlet.getTempFile(Long.toString(System.currentTimeMillis())); temp.deleteOnExit(); IOUtils.copy(is,temp); is.close(); diff --git a/main/src/com/metaweb/gridworks/exporters/CsvExporter.java b/main/src/com/metaweb/gridworks/exporters/CsvExporter.java index 2e6548b6e..f001233cf 100644 --- a/main/src/com/metaweb/gridworks/exporters/CsvExporter.java +++ b/main/src/com/metaweb/gridworks/exporters/CsvExporter.java @@ -40,66 +40,66 @@ public class CsvExporter implements Exporter{ @Override public void export(Project project, Properties options, Engine engine, Writer writer) throws IOException { - { - boolean printColumnHeader = true; - if(options != null) - printColumnHeader = options.getProperty("printColumnHeader")=="false"?false:true; + boolean printColumnHeader = true; - RowVisitor visitor = new RowVisitor() { - CSVWriter csvWriter; - boolean printColumnHeader = true; - boolean isFirstRow = true; //the first row should also add the column headers - - public RowVisitor init(CSVWriter writer, boolean printColumnHeader){ - this.csvWriter = writer; - this.printColumnHeader = printColumnHeader; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row) { - String[] cols = new String[project.columnModel.columns.size()]; - String[] vals = new String[row.cells.size()]; - - int i = 0; - for(Column col : project.columnModel.columns){ - int cellIndex = col.getCellIndex(); - cols[i] = col.getName(); - - Cell cell = row.cells.get(cellIndex); - if(cell != null){ - vals[i] = cell.value.toString(); - } - i++; - } - - if( printColumnHeader && isFirstRow ){ - csvWriter.writeNext(cols,false); - isFirstRow = false; //switch off flag - } - csvWriter.writeNext(vals,false); - - return false; - } - - @Override - public void start(Project project) { - // nothing to do - } - - @Override - public void end(Project project) { - try { - csvWriter.close(); - } catch (IOException e) { - logger.error("CsvExporter could not close writer : " + e.getMessage()); - } - } - - }.init(new CSVWriter(writer, separator), printColumnHeader); - - FilteredRows filteredRows = engine.getAllFilteredRows(); - filteredRows.accept(project, visitor); + if (options != null) { + printColumnHeader = Boolean.parseBoolean(options.getProperty("printColumnHeader")); } + + RowVisitor visitor = new RowVisitor() { + CSVWriter csvWriter; + boolean printColumnHeader = true; + boolean isFirstRow = true; //the first row should also add the column headers + + public RowVisitor init(CSVWriter writer, boolean printColumnHeader){ + this.csvWriter = writer; + this.printColumnHeader = printColumnHeader; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row) { + String[] cols = new String[project.columnModel.columns.size()]; + String[] vals = new String[row.cells.size()]; + + int i = 0; + for(Column col : project.columnModel.columns){ + int cellIndex = col.getCellIndex(); + cols[i] = col.getName(); + + Cell cell = row.cells.get(cellIndex); + if(cell != null){ + vals[i] = cell.value.toString(); + } + i++; + } + + if( printColumnHeader && isFirstRow ){ + csvWriter.writeNext(cols,false); + isFirstRow = false; //switch off flag + } + csvWriter.writeNext(vals,false); + + return false; + } + + @Override + public void start(Project project) { + // nothing to do + } + + @Override + public void end(Project project) { + try { + csvWriter.close(); + } catch (IOException e) { + logger.error("CsvExporter could not close writer : " + e.getMessage()); + } + } + + }.init(new CSVWriter(writer, separator), printColumnHeader); + + FilteredRows filteredRows = engine.getAllFilteredRows(); + filteredRows.accept(project, visitor); } @Override diff --git a/main/src/com/metaweb/gridworks/expr/JythonEvaluable.java b/main/src/com/metaweb/gridworks/expr/JythonEvaluable.java index 1125409d5..5264db879 100644 --- a/main/src/com/metaweb/gridworks/expr/JythonEvaluable.java +++ b/main/src/com/metaweb/gridworks/expr/JythonEvaluable.java @@ -19,7 +19,7 @@ public class JythonEvaluable implements Evaluable { private static PythonInterpreter _engine; static { - File libPath = new File("lib/jython"); + File libPath = new File("webapp/WEB-INF/lib/jython"); if (libPath.exists()) { Properties props = new Properties(); props.setProperty("python.path", libPath.getAbsolutePath()); diff --git a/main/src/com/metaweb/gridworks/history/History.java b/main/src/com/metaweb/gridworks/history/History.java index bc600bd87..52789b412 100644 --- a/main/src/com/metaweb/gridworks/history/History.java +++ b/main/src/com/metaweb/gridworks/history/History.java @@ -16,7 +16,7 @@ import java.util.Properties; import org.json.JSONException; import org.json.JSONWriter; -import com.metaweb.gridworks.Gridworks; +import com.metaweb.gridworks.GridworksServlet; import com.metaweb.gridworks.Jsonizable; import com.metaweb.gridworks.ProjectManager; import com.metaweb.gridworks.model.Project; @@ -68,7 +68,7 @@ public class History implements Jsonizable { } static public void writeOneChange(Writer writer, Change change, Properties options) throws IOException { - writer.write(Gridworks.getVersion()); writer.write('\n'); + writer.write(GridworksServlet.getVersion()); writer.write('\n'); writer.write(change.getClass().getName()); writer.write('\n'); change.save(writer, options); diff --git a/main/src/com/metaweb/gridworks/importers/XmlImportUtilities.java b/main/src/com/metaweb/gridworks/importers/XmlImportUtilities.java index 89de297a2..78ab92ebe 100644 --- a/main/src/com/metaweb/gridworks/importers/XmlImportUtilities.java +++ b/main/src/com/metaweb/gridworks/importers/XmlImportUtilities.java @@ -72,8 +72,6 @@ public class XmlImportUtilities { } static public String[] detectPathFromTag(InputStream inputStream, String tag) { - //List candidates = new ArrayList(); - try { XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(inputStream); @@ -99,6 +97,8 @@ public class XmlImportUtilities { } static protected List detectRecordElement(XMLStreamReader parser, String tag) throws XMLStreamException { + if(parser.getEventType() == XMLStreamConstants.START_DOCUMENT) + parser.next(); String localName = parser.getLocalName(); String fullName = composeName(parser.getPrefix(), localName); if (tag.equals(parser.getLocalName()) || tag.equals(fullName)) { @@ -327,6 +327,10 @@ public class XmlImportUtilities { int pathIndex, ImportColumnGroup rootColumnGroup ) throws XMLStreamException { + if(parser.getEventType() == XMLStreamConstants.START_DOCUMENT){ + logger.warn("Cannot use findRecord method for START_DOCUMENT event"); + return; + } String tagName = parser.getLocalName(); if (tagName.equals(recordPath[pathIndex])) { if (pathIndex < recordPath.length - 1) { @@ -466,7 +470,7 @@ public class XmlImportUtilities { ImportRecord record, String columnLocalName, String text, - int commonStaringRowIndex + int commonStartingRowIndex ) { if (text == null || ((String) text).isEmpty()) { return; @@ -478,7 +482,7 @@ public class XmlImportUtilities { int cellIndex = column.cellIndex; while (cellIndex >= record.columnEmptyRowIndices.size()) { - record.columnEmptyRowIndices.add(commonStaringRowIndex); + record.columnEmptyRowIndices.add(commonStartingRowIndex); } int rowIndex = record.columnEmptyRowIndices.get(cellIndex); @@ -491,7 +495,9 @@ public class XmlImportUtilities { row.add(null); } - row.set(cellIndex, new Cell(value, null)); + logger.trace("Adding cell with value : " + value + " to row : " + rowIndex + " at cell index : " + (cellIndex-1)); + + row.set(cellIndex-1, new Cell(value, null)); record.columnEmptyRowIndices.set(cellIndex, rowIndex + 1); diff --git a/main/src/com/metaweb/gridworks/importers/XmlImporter.java b/main/src/com/metaweb/gridworks/importers/XmlImporter.java index 23c9a6369..6a2f42001 100644 --- a/main/src/com/metaweb/gridworks/importers/XmlImporter.java +++ b/main/src/com/metaweb/gridworks/importers/XmlImporter.java @@ -57,6 +57,9 @@ public class XmlImporter implements Importer { } } + if(recordPath == null) + return; + ImportColumnGroup rootColumnGroup = new ImportColumnGroup(); XmlImportUtilities.importXml(pis, project, recordPath, rootColumnGroup); diff --git a/main/src/com/metaweb/gridworks/logging/IndentingLayout.java b/main/src/com/metaweb/gridworks/logging/IndentingLayout.java new file mode 100644 index 000000000..4886431e4 --- /dev/null +++ b/main/src/com/metaweb/gridworks/logging/IndentingLayout.java @@ -0,0 +1,143 @@ +package com.metaweb.gridworks.logging; + +/* + * Copyright (c) Massachusetts Institute of Technology, 2007 + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Original code: http://simile.mit.edu/repository/tracer/trunk/ + */ + +import java.util.Calendar; +import java.util.Date; + +import org.apache.log4j.Layout; +import org.apache.log4j.spi.LoggingEvent; + +/** + * This is a special Log4j log formatter that is capable of reacting on special log messages + * and 'indent' the logs accordingly. This is very useful to visually inspect a debug log + * and see what calls what. An example of logs are "> method()" and "< method()" where > and < + * are used to indicate respectively "entering" and "exiting". + */ +public class IndentingLayout extends Layout { + + protected static final int CONTEXT_SIZE = 25; + protected static final long MAX_DELTA = 10000; + + protected Calendar calendar = Calendar.getInstance(); + protected long previousTime = 0; + protected int indentation = 0; + + public void activateOptions() { + // no options at this time + } + + public String format(LoggingEvent event) { + String message = event.getRenderedMessage(); + if (message == null) return ""; + if (message.length() < 2) return message; + + char leader = message.charAt(0); + char secondLeader = message.charAt(1); + if ((leader == '<') && (secondLeader == ' ') && (this.indentation > 0)) this.indentation--; + + // Reset buf + StringBuffer buf = new StringBuffer(256); + + Date date = new Date(); + long now = date.getTime(); + calendar.setTime(date); + + long delta = 0; + if (previousTime > 0) { + delta = now - previousTime; + } + previousTime = now; + +// if ((previousTime == 0) || (delta > MAX_DELTA)) { +// buf.append('\n'); +// indentation = 0; // reset indentation after a while, as we might +// // have runaway/unmatched log entries +// } + + int hour = calendar.get(Calendar.HOUR_OF_DAY); + if (hour < 10) buf.append('0'); + buf.append(hour); + buf.append(':'); + + int mins = calendar.get(Calendar.MINUTE); + if (mins < 10) buf.append('0'); + buf.append(mins); + buf.append(':'); + + int secs = calendar.get(Calendar.SECOND); + if (secs < 10) buf.append('0'); + buf.append(secs); + buf.append('.'); + + int millis = (int) (now % 1000); + if (millis < 100) buf.append('0'); + if (millis < 10) buf.append('0'); + buf.append(millis); + + buf.append(" ["); + String context = ((String) event.getMDC("LogEvent")); + if (context == null) { + context = event.getLoggerName(); + } + if (context.length() < CONTEXT_SIZE) { + pad(buf, CONTEXT_SIZE - context.length(), ' '); + buf.append(context); + } else { + buf.append(".."); + buf.append(context.substring(context.length() - CONTEXT_SIZE + 2)); + } + buf.append("] "); + + pad(buf, indentation, ' '); + + buf.append(message); + + buf.append(" ("); + buf.append(delta); + buf.append("ms)\n"); + + if ((leader == '>') && (secondLeader == ' ')) indentation++; + + return buf.toString(); + } + + private void pad(StringBuffer buffer, int pads, char padchar) { + for (int i = 0; i < pads; i++) { + buffer.append(padchar); + } + } + + public boolean ignoresThrowable() { + return true; + } +} diff --git a/main/src/com/metaweb/gridworks/model/Project.java b/main/src/com/metaweb/gridworks/model/Project.java index 6a22e51d3..d73bee522 100644 --- a/main/src/com/metaweb/gridworks/model/Project.java +++ b/main/src/com/metaweb/gridworks/model/Project.java @@ -19,7 +19,7 @@ import java.util.zip.ZipOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.metaweb.gridworks.Gridworks; +import com.metaweb.gridworks.GridworksServlet; import com.metaweb.gridworks.ProjectManager; import com.metaweb.gridworks.ProjectMetadata; import com.metaweb.gridworks.history.History; @@ -128,7 +128,7 @@ public class Project { } protected void saveToWriter(Writer writer, Properties options) throws IOException { - writer.write(Gridworks.getVersion()); writer.write('\n'); + writer.write(GridworksServlet.getVersion()); writer.write('\n'); writer.write("columnModel=\n"); columnModel.save(writer, options); writer.write("history=\n"); history.save(writer, options); diff --git a/main/src/com/metaweb/gridworks/util/FreebaseUtils.java b/main/src/com/metaweb/gridworks/util/FreebaseUtils.java index 50c56ceca..dfc93510f 100644 --- a/main/src/com/metaweb/gridworks/util/FreebaseUtils.java +++ b/main/src/com/metaweb/gridworks/util/FreebaseUtils.java @@ -27,7 +27,7 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import com.metaweb.gridworks.Gridworks; +import com.metaweb.gridworks.GridworksServlet; import com.metaweb.gridworks.oauth.Credentials; import com.metaweb.gridworks.oauth.OAuthUtilities; import com.metaweb.gridworks.oauth.Provider; @@ -59,7 +59,7 @@ public class FreebaseUtils { OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion()); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion()); // this is required by the Metaweb API to avoid XSS httpRequest.setHeader("X-Requested-With", "1"); @@ -102,7 +102,7 @@ public class FreebaseUtils { UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion()); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion()); httpRequest.setEntity(entity); // this is required by the Metaweb API to avoid XSS @@ -128,7 +128,7 @@ public class FreebaseUtils { UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion()); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion()); httpRequest.setEntity(entity); // this is required by the Metaweb API to avoid XSS @@ -176,7 +176,7 @@ public class FreebaseUtils { UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); HttpPost httpRequest = new HttpPost(FREEQ_URL); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion()); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion()); httpRequest.setEntity(entity); HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST)); diff --git a/main/src/com/metaweb/gridworks/util/Pool.java b/main/src/com/metaweb/gridworks/util/Pool.java index ecc24d7ca..6f4893521 100644 --- a/main/src/com/metaweb/gridworks/util/Pool.java +++ b/main/src/com/metaweb/gridworks/util/Pool.java @@ -15,7 +15,7 @@ import java.util.Map.Entry; import org.json.JSONException; import org.json.JSONWriter; -import com.metaweb.gridworks.Gridworks; +import com.metaweb.gridworks.GridworksServlet; import com.metaweb.gridworks.Jsonizable; import com.metaweb.gridworks.model.Recon; import com.metaweb.gridworks.model.ReconCandidate; @@ -59,7 +59,7 @@ public class Pool implements Jsonizable { } public void save(Writer writer) throws IOException { - writer.write(Gridworks.getVersion()); writer.write('\n'); + writer.write(GridworksServlet.getVersion()); writer.write('\n'); Properties options = new Properties(); options.setProperty("mode", "save"); diff --git a/main/src/log4j.properties b/main/src/log4j.properties new file mode 100644 index 000000000..fea32e42d --- /dev/null +++ b/main/src/log4j.properties @@ -0,0 +1,7 @@ +log4j.rootLogger=INFO, console +log4j.logger.org.apache.http.headers=WARN +log4j.logger.org.apache.http.impl=WARN +log4j.logger.org.apache.http.client=WARN + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=com.metaweb.gridworks.logging.IndentingLayout diff --git a/main/tests/client/js/facets.js b/main/tests/client/src/facets.js similarity index 100% rename from main/tests/client/js/facets.js rename to main/tests/client/src/facets.js diff --git a/main/tests/client/js/initialize.js b/main/tests/client/src/initialize.js similarity index 100% rename from main/tests/client/js/initialize.js rename to main/tests/client/src/initialize.js diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletStub.java b/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletStub.java index 351f8fd71..0e1bf1045 100644 --- a/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletStub.java +++ b/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletStub.java @@ -36,7 +36,7 @@ public class GridworksServletStub extends GridworksServlet { * @param commandName * @param command */ - static public void InsertCommand( String commandName, Command command ){ + public void insertCommand( String commandName, Command command ){ registerCommand(commandName, command); } @@ -44,7 +44,7 @@ public class GridworksServletStub extends GridworksServlet { * Helper method for clearing up after testing * @param commandName */ - static public void RemoveCommand( String commandName ){ + public void removeCommand( String commandName ){ unregisterCommand(commandName); } } diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletTests.java b/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletTests.java index f40f42053..e3a089184 100644 --- a/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletTests.java +++ b/main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletTests.java @@ -40,21 +40,22 @@ public class GridworksServletTests { @BeforeMethod public void SetUp() { - SUT = new GridworksServletStub(); request = mock(HttpServletRequest.class); response = mock(HttpServletResponse.class); command = mock(Command.class); - GridworksServletStub.InsertCommand(TEST_COMMAND_NAME,command); //inject mock into command container + SUT = new GridworksServletStub(); + SUT.insertCommand(TEST_COMMAND_NAME,command); //inject mock into command container } @AfterMethod public void TearDown() { + SUT.removeCommand(TEST_COMMAND_NAME); //remove mock to clean command container SUT = null; + request = null; response = null; command = null; - GridworksServletStub.RemoveCommand(TEST_COMMAND_NAME); //remove mock to clean command container } //-------------------AutoSaveTimerTask tests----------- diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/importers/RdfTripleImporterTests.java b/main/tests/server/src/com/metaweb/gridworks/tests/importers/RdfTripleImporterTests.java index 694521e80..91a4f5059 100644 --- a/main/tests/server/src/com/metaweb/gridworks/tests/importers/RdfTripleImporterTests.java +++ b/main/tests/server/src/com/metaweb/gridworks/tests/importers/RdfTripleImporterTests.java @@ -1,153 +1,153 @@ -package com.metaweb.gridworks.tests.importers; - -import java.io.StringReader; -import java.util.Properties; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.Assert; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import com.metaweb.gridworks.importers.RdfTripleImporter; -import com.metaweb.gridworks.model.Project; - - -public class RdfTripleImporterTests { - // logging - final static protected Logger logger = LoggerFactory.getLogger("RdfImporterTests"); - - //System Under Test - RdfTripleImporter SUT = null; - Project project = null; - Properties options = null; - - @BeforeMethod - public void SetUp(){ - SUT = new RdfTripleImporter(); - project = new Project(); - options = new Properties(); - options.put("base-url", "http://rdf.freebase.com"); - } - - @Test(enabled=false) - public void CanParseSingleLineTriple(){ - String sampleRdf = " ."; - StringReader reader = new StringReader(sampleRdf); - - try { - SUT.read(reader, project, options); - project.update(); - } catch (Exception e) { - Assert.fail(); - } - - Assert.assertEquals(project.columnModel.columns.size(), 2); - Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); - Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album"); - Assert.assertEquals(project.rows.size(), 1); - Assert.assertEquals(project.rows.get(0).cells.size(), 2); - Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); - Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks"); - } - - @Test - public void CanParseMultiLineTriple(){ - String sampleRdf = " .\n" + - " .\n" + - " ."; - StringReader reader = new StringReader(sampleRdf); - - try { - SUT.read(reader, project, options); - project.update(); - } catch (Exception e) { - Assert.fail(); - } - - //columns - Assert.assertEquals(project.columnModel.columns.size(), 2); - Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); - Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album"); - - //rows - Assert.assertEquals(project.rows.size(), 3); - - //row0 - Assert.assertEquals(project.rows.get(0).cells.size(), 2); - Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); - Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks"); - - //row1 - Assert.assertEquals(project.rows.get(1).cells.size(), 2); - Assert.assertNull(project.rows.get(1).cells.get(0)); - Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); //NB triples aren't created in order they were input - Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0); - Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0); - - //row2 - Assert.assertEquals(project.rows.get(2).cells.size(), 2); - Assert.assertNull(project.rows.get(2).cells.get(0)); - Assert.assertEquals(project.rows.get(2).cells.get(1).value, "http://rdf.freebase.com/ns/en.under_the_red_sky"); //NB triples aren't created in order they were input - Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].rowIndex, 0); - Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].cellIndex, 0); - } - - @Test - public void CanParseMultiLineMultiPredicatesTriple(){ - String sampleRdf = " .\n" + - " .\n" + - " ."; - StringReader reader = new StringReader(sampleRdf); - - try { - SUT.read(reader, project, options); - project.update(); - } catch (Exception e) { - Assert.fail(); - } - - //columns - Assert.assertEquals(project.columnModel.columns.size(), 3); - Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); - Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album"); - Assert.assertEquals(project.columnModel.columns.get(2).getName(), "http://rdf.freebase.com/ns/music.artist.genre"); - - //rows - Assert.assertEquals(project.rows.size(), 2); - - //row0 - Assert.assertEquals(project.rows.get(0).cells.size(), 3); - Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); - Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks"); - Assert.assertEquals(project.rows.get(0).cells.get(2).value, "http://rdf.freebase.com/ns/en.folk_rock"); - - //row1 - Assert.assertEquals(project.rows.get(1).cells.size(), 2); - Assert.assertNull(project.rows.get(1).cells.get(0)); - Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); - Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0); - Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0); - } - - @Test - public void CanParseTripleWithValue(){ - String sampleRdf = " \"Robert Zimmerman\"@en."; - StringReader reader = new StringReader(sampleRdf); - - try { - SUT.read(reader, project, options); - project.update(); - } catch (Exception e) { - Assert.fail(); - } - - Assert.assertEquals(project.columnModel.columns.size(), 2); - Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); - Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/common.topic.alias"); - Assert.assertEquals(project.rows.size(), 1); - Assert.assertEquals(project.rows.get(0).cells.size(), 2); - Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); - Assert.assertEquals(project.rows.get(0).cells.get(1).value, "\"Robert Zimmerman\"@en"); - } -} +package com.metaweb.gridworks.tests.importers; + +import java.io.StringReader; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.metaweb.gridworks.importers.RdfTripleImporter; +import com.metaweb.gridworks.model.Project; + + +public class RdfTripleImporterTests { + // logging + final static protected Logger logger = LoggerFactory.getLogger("RdfImporterTests"); + + //System Under Test + RdfTripleImporter SUT = null; + Project project = null; + Properties options = null; + + @BeforeMethod + public void SetUp(){ + SUT = new RdfTripleImporter(); + project = new Project(); + options = new Properties(); + options.put("base-url", "http://rdf.freebase.com"); + } + + @Test(enabled=false) + public void CanParseSingleLineTriple(){ + String sampleRdf = " ."; + StringReader reader = new StringReader(sampleRdf); + + try { + SUT.read(reader, project, options); + project.update(); + } catch (Exception e) { + Assert.fail(); + } + + Assert.assertEquals(project.columnModel.columns.size(), 2); + Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); + Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album"); + Assert.assertEquals(project.rows.size(), 1); + Assert.assertEquals(project.rows.get(0).cells.size(), 2); + Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); + Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks"); + } + + @Test + public void CanParseMultiLineTriple(){ + String sampleRdf = " .\n" + + " .\n" + + " ."; + StringReader reader = new StringReader(sampleRdf); + + try { + SUT.read(reader, project, options); + project.update(); + } catch (Exception e) { + Assert.fail(); + } + + //columns + Assert.assertEquals(project.columnModel.columns.size(), 2); + Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); + Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album"); + + //rows + Assert.assertEquals(project.rows.size(), 3); + + //row0 + Assert.assertEquals(project.rows.get(0).cells.size(), 2); + Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); + Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks"); + + //row1 + Assert.assertEquals(project.rows.get(1).cells.size(), 2); + Assert.assertNull(project.rows.get(1).cells.get(0)); + Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); //NB triples aren't created in order they were input + Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0); + Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0); + + //row2 + Assert.assertEquals(project.rows.get(2).cells.size(), 2); + Assert.assertNull(project.rows.get(2).cells.get(0)); + Assert.assertEquals(project.rows.get(2).cells.get(1).value, "http://rdf.freebase.com/ns/en.under_the_red_sky"); //NB triples aren't created in order they were input + Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].rowIndex, 0); + Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].cellIndex, 0); + } + + @Test + public void CanParseMultiLineMultiPredicatesTriple(){ + String sampleRdf = " .\n" + + " .\n" + + " ."; + StringReader reader = new StringReader(sampleRdf); + + try { + SUT.read(reader, project, options); + project.update(); + } catch (Exception e) { + Assert.fail(); + } + + //columns + Assert.assertEquals(project.columnModel.columns.size(), 3); + Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); + Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album"); + Assert.assertEquals(project.columnModel.columns.get(2).getName(), "http://rdf.freebase.com/ns/music.artist.genre"); + + //rows + Assert.assertEquals(project.rows.size(), 2); + + //row0 + Assert.assertEquals(project.rows.get(0).cells.size(), 3); + Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); + Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks"); + Assert.assertEquals(project.rows.get(0).cells.get(2).value, "http://rdf.freebase.com/ns/en.folk_rock"); + + //row1 + Assert.assertEquals(project.rows.get(1).cells.size(), 2); + Assert.assertNull(project.rows.get(1).cells.get(0)); + Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); + Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0); + Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0); + } + + @Test + public void CanParseTripleWithValue(){ + String sampleRdf = " \"Robert Zimmerman\"@en."; + StringReader reader = new StringReader(sampleRdf); + + try { + SUT.read(reader, project, options); + project.update(); + } catch (Exception e) { + Assert.fail(); + } + + Assert.assertEquals(project.columnModel.columns.size(), 2); + Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); + Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/common.topic.alias"); + Assert.assertEquals(project.rows.size(), 1); + Assert.assertEquals(project.rows.get(0).cells.size(), 2); + Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); + Assert.assertEquals(project.rows.get(0).cells.get(1).value, "\"Robert Zimmerman\"@en"); + } +} diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/importers/TestTools.java b/main/tests/server/src/com/metaweb/gridworks/tests/importers/TestTools.java new file mode 100644 index 000000000..68fb8321a --- /dev/null +++ b/main/tests/server/src/com/metaweb/gridworks/tests/importers/TestTools.java @@ -0,0 +1,50 @@ +package com.metaweb.gridworks.tests.importers; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; + +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +/** + * Helper methods for Importer testing + * + */ +public class TestTools { + final static Logger logger = LoggerFactory.getLogger("Common"); + + public static void AssertGridCreated(Project project, int numCols, int numRows){ + Assert.assertNotNull(project); + Assert.assertNotNull(project.columnModel); + Assert.assertNotNull(project.columnModel.columns); + Assert.assertEquals(project.columnModel.columns.size(), numCols); + Assert.assertNotNull(project.rows); + Assert.assertEquals(project.rows.size(), numRows); + } + + public static void PrintProject(Project project){ + //some quick and dirty debugging + StringBuilder sb = new StringBuilder(); + for(Column c : project.columnModel.columns){ + sb.append(c.getName()); + sb.append("; "); + } + logger.info(sb.toString()); + for(Row r : project.rows){ + sb = new StringBuilder(); + for(int i = 0; i < r.cells.size(); i++){ + Cell c = r.getCell(i); + if(c != null){ + sb.append(c.value); + sb.append("; "); + }else{ + sb.append("null; "); + } + } + logger.info(sb.toString()); + } + } +} diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesStub.java b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesStub.java new file mode 100644 index 000000000..a35cdb6e7 --- /dev/null +++ b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesStub.java @@ -0,0 +1,32 @@ +package com.metaweb.gridworks.tests.importers; + +import java.util.List; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +import com.metaweb.gridworks.importers.XmlImportUtilities; +import com.metaweb.gridworks.model.Project; + + +public class XmlImportUtilitiesStub extends XmlImportUtilities{ + public List detectRecordElementWrapper(XMLStreamReader parser, String tag) throws XMLStreamException{ + return super.detectRecordElement(parser, tag); + } + + public void ProcessSubRecordWrapper(Project project, XMLStreamReader parser, ImportColumnGroup columnGroup, ImportRecord record) throws XMLStreamException{ + super.processSubRecord(project, parser, columnGroup, record); + } + + public void findRecordWrapper(Project project, XMLStreamReader parser, String[] recordPath, int pathIndex, ImportColumnGroup rootColumnGroup) throws XMLStreamException{ + super.findRecord(project, parser, recordPath, pathIndex, rootColumnGroup); + } + + public void processRecordWrapper(Project project, XMLStreamReader parser, ImportColumnGroup rootColumnGroup) throws XMLStreamException{ + super.processRecord(project, parser, rootColumnGroup); + } + + public void addCellWrapper(Project project, ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, String text, int commonStartingRowIndex){ + super.addCell(project, columnGroup, record, columnLocalName, text, commonStartingRowIndex); + } +} diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesTests.java b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesTests.java new file mode 100644 index 000000000..c21c94314 --- /dev/null +++ b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesTests.java @@ -0,0 +1,336 @@ +package com.metaweb.gridworks.tests.importers; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.stream.FactoryConfigurationError; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +import org.apache.log4j.Level; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.metaweb.gridworks.importers.XmlImportUtilities.ImportColumn; +import com.metaweb.gridworks.importers.XmlImportUtilities.ImportColumnGroup; +import com.metaweb.gridworks.importers.XmlImportUtilities.ImportRecord; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + + +public class XmlImportUtilitiesTests { + final static Logger logger = LoggerFactory.getLogger("XmlImporterUtilitiesTests"); + + //dependencies + Project project; + XMLStreamReader parser; + ImportColumnGroup columnGroup; + ImportRecord record; + ByteArrayInputStream inputStream; + + //System Under Test + XmlImportUtilitiesStub SUT; + + @BeforeMethod + public void SetUp(){ + org.apache.log4j.Logger.getRootLogger().setLevel(Level.toLevel("trace")); + SUT = new XmlImportUtilitiesStub(); + project = new Project(); + columnGroup = new ImportColumnGroup(); + record = new ImportRecord(); + } + + @AfterMethod + public void TearDown() throws IOException{ + SUT = null; + project = null; + parser = null; + columnGroup = null; + record = null; + if(inputStream != null) + inputStream.close(); + inputStream = null; + } + + @Test + public void detectPathFromTagTest(){ + loadXml("author1genre1"); + String tag = "library"; + + String[] response = XmlImportUtilitiesStub.detectPathFromTag(inputStream, tag); + Assert.assertNotNull(response); + Assert.assertEquals(response.length, 1); + Assert.assertEquals(response[0], "library"); + } + + @Test + public void detectPathFromTagWithNestedElement(){ + loadXml("author1genre1"); + String tag = "book"; + String[] response = XmlImportUtilitiesStub.detectPathFromTag(inputStream, tag); + Assert.assertNotNull(response); + Assert.assertEquals(response.length, 2); + Assert.assertEquals(response[0], "library"); + Assert.assertEquals(response[1], "book"); + } + + @Test + public void detectRecordElementTest(){ + loadXml("author1genre1"); + createParser(); + String tag="library"; + + List response = new ArrayList(); + try { + response = SUT.detectRecordElementWrapper(parser, tag); + } catch (XMLStreamException e) { + Assert.fail(); + } + Assert.assertNotNull(response); + Assert.assertEquals(response.size(), 1); + Assert.assertEquals(response.get(0), "library"); + } + + @Test + public void detectRecordElementCanHandleWithNestedElements(){ + loadXml("author1genre1"); + createParser(); + String tag="book"; + + List response = new ArrayList(); + try { + response = SUT.detectRecordElementWrapper(parser, tag); + } catch (XMLStreamException e) { + Assert.fail(); + } + Assert.assertNotNull(response); + Assert.assertEquals(response.size(), 2); + Assert.assertEquals(response.get(0), "library"); + Assert.assertEquals(response.get(1), "book"); + } + + @Test + public void detectRecordElementIsNullForUnfoundTag(){ + loadXml("author1genre1"); + createParser(); + String tag=""; + + List response = new ArrayList(); + try { + response = SUT.detectRecordElementWrapper(parser, tag); + } catch (XMLStreamException e) { + Assert.fail(); + } + Assert.assertNull(response); + } + + @Test + public void detectRecordElementRegressionTest(){ + loadSampleXml(); + + String[] path = XmlImportUtilitiesStub.detectRecordElement(inputStream); + Assert.assertNotNull(path); + Assert.assertEquals(path.length, 2); + Assert.assertEquals(path[0], "library"); + Assert.assertEquals(path[1], "book"); + } + + @Test + public void importXmlTest(){ + loadSampleXml(); + + String[] recordPath = new String[]{"library","book"}; + XmlImportUtilitiesStub.importXml(inputStream, project, recordPath, columnGroup ); + + TestTools.PrintProject(project); + TestTools.AssertGridCreated(project, 0, 6); + Assert.assertEquals(project.rows.get(0).cells.size(), 4); + //TODO + } + + @Test + public void createColumnsFromImportTest(){ + + ImportColumnGroup columnGroup = new ImportColumnGroup(); + ImportColumn ic1 = new ImportColumn(); + ic1.name = "hello"; + ImportColumn ic2 = new ImportColumn(); + ic2.name = "world"; + ImportColumnGroup subGroup = new ImportColumnGroup(); + ImportColumn ic3 = new ImportColumn(); + ic3.name = "foo"; + ImportColumn ic4 = new ImportColumn(); + ic4.name = "bar"; + subGroup.columns.put("c", ic3); + subGroup.columns.put("d", ic4); + columnGroup.columns.put("a", ic1); + columnGroup.columns.put("b", ic2); + columnGroup.subgroups.put("e", subGroup); + XmlImportUtilitiesStub.createColumnsFromImport(project, columnGroup); + TestTools.PrintProject(project); + TestTools.AssertGridCreated(project, 4, 0); + Assert.assertEquals(project.columnModel.columns.get(0).getName(), "world"); + Assert.assertEquals(project.columnModel.columns.get(1).getName(), "hello"); + Assert.assertEquals(project.columnModel.columns.get(2).getName(), "bar"); + Assert.assertEquals(project.columnModel.columns.get(3).getName(), "foo"); + Assert.assertEquals(project.columnModel.columnGroups.get(0).keyColumnIndex, 2); + Assert.assertEquals(project.columnModel.columnGroups.get(0).startColumnIndex, 2); + Assert.assertEquals(project.columnModel.columnGroups.get(0).columnSpan, 2); + } + + @Test + public void findRecordTest(){ + loadSampleXml(); + createParser(); + ParserSkip(); + + String[] recordPath = new String[]{"library","book"}; + int pathIndex = 0; + + try { + SUT.findRecordWrapper(project, parser, recordPath, pathIndex, columnGroup); + } catch (XMLStreamException e) { + Assert.fail(); + } + + TestTools.PrintProject(project); + TestTools.AssertGridCreated(project, 0, 6); + Assert.assertEquals(project.rows.get(0).cells.size(), 4); + //TODO + } + + @Test + public void processRecordTest(){ + loadXml("author1genre1"); + createParser(); + ParserSkip(); + + try { + SUT.processRecordWrapper(project, parser, columnGroup); + } catch (XMLStreamException e) { + Assert.fail(); + } + TestTools.PrintProject(project); + Assert.assertNotNull(project.rows); + Assert.assertEquals(project.rows.size(), 1); + Row row = project.rows.get(0); + Assert.assertNotNull(row); + Assert.assertNotNull(row.getCell(1)); + Assert.assertEquals(row.getCell(1).value, "author1"); + + } + + @Test + public void processRecordTestDuplicateColumns(){ + loadXml("author1author2genre1"); + createParser(); + ParserSkip(); + + try { + SUT.processRecordWrapper(project, parser, columnGroup); + } catch (XMLStreamException e) { + Assert.fail(); + } + TestTools.PrintProject(project); + Assert.assertNotNull(project.rows); + Assert.assertEquals(project.rows.size(), 2); + Row row = project.rows.get(0); + Assert.assertNotNull(row); + Assert.assertEquals(row.cells.size(), 3); + Assert.assertNotNull(row.getCell(1)); + Assert.assertEquals(row.getCell(1).value, "author1"); + row = project.rows.get(1); + Assert.assertEquals(row.getCell(1).value, "author2"); + } + + @Test + public void processRecordTestNestedElement(){ + loadXml("author1a dategenre1"); + createParser(); + ParserSkip(); + + try { + SUT.processRecordWrapper(project, parser, columnGroup); + } catch (XMLStreamException e) { + Assert.fail(); + } + TestTools.PrintProject(project); + Assert.assertNotNull(project.rows); + Assert.assertEquals(project.rows.size(), 1); + Row row = project.rows.get(0); + Assert.assertNotNull(row); + Assert.assertEquals(row.cells.size(), 4); + Assert.assertNotNull(row.getCell(1)); + Assert.assertEquals(row.getCell(1).value, "author1"); + Assert.assertNotNull(row.getCell(2)); + Assert.assertEquals(row.getCell(2).value, "a date"); + } + + + @Test(groups={"broken"}) + public void processSubRecordTest(){ + loadXml("author1genre1"); + createParser(); + ParserSkip(); + + try { + SUT.ProcessSubRecordWrapper(project, parser, columnGroup, record); + } catch (XMLStreamException e) { + Assert.fail(); + } + TestTools.PrintProject(project); + Assert.fail(); + //TODO need to verify 'record' was set correctly which we can't do as ImportRecord is an internal class + } + + @Test(groups={"broken"}) + public void addCellTest(){ + String columnLocalName = "author"; + String text = "Author1, The"; + int commonStartingRowIndex = 0; + project.rows.add(new Row(0)); + SUT.addCellWrapper(project, columnGroup, record, columnLocalName, text, commonStartingRowIndex); + + Assert.fail(); + //TODO need to verify 'record' was set correctly which we can't do as ImportRecord is an internal class + } + + //----------------helpers------------- + public void loadSampleXml(){ + loadXml( XmlImporterTests.getSample() ); + } + + public void loadXml(String xml){ + try { + inputStream = new ByteArrayInputStream( xml.getBytes( "UTF-8" ) ); + } catch (UnsupportedEncodingException e1) { + Assert.fail(); + } + } + + public void ParserSkip(){ + try { + parser.next(); //move parser forward once e.g. skip the START_DOCUMENT parser event + } catch (XMLStreamException e1) { + Assert.fail(); + } + } + + public void createParser(){ + try { + parser = XMLInputFactory.newInstance().createXMLStreamReader(inputStream); + } catch (XMLStreamException e1) { + Assert.fail(); + } catch (FactoryConfigurationError e1) { + Assert.fail(); + } + } +} diff --git a/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImporterTests.java b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImporterTests.java index e4272b6bf..613d5c80d 100644 --- a/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImporterTests.java +++ b/main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImporterTests.java @@ -1,207 +1,225 @@ -package com.metaweb.gridworks.tests.importers; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.Properties; - -import static org.mockito.Mockito.mock; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import com.metaweb.gridworks.importers.XmlImporter; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - - -public class XmlImporterTests { - final static Logger logger = LoggerFactory.getLogger("XmlImporterTests"); - - //dependencies - Project project = null; - Properties options = null; - ByteArrayInputStream inputStream = null; - - //System Under Test - XmlImporter SUT = null; - - - @BeforeMethod - public void SetUp(){ - SUT = new XmlImporter(); - project = new Project(); - options = mock(Properties.class); - } - - @AfterMethod - public void TearDown(){ - SUT = null; - project = null; - options = null; - } - - @Test - public void canParseSample(){ - - RunTest(getSample()); - - AssertGridCreate(project, 4, 6); - PrintProject(project); - - Row row = project.rows.get(0); - Assert.assertNotNull(row); - Assert.assertNotNull(row.cells); - Assert.assertNotNull(row.cells.get(2)); - Assert.assertEquals(row.cells.get(2).value, "Author 1, The"); - - - } - - @Test - public void testCanParseLineBreak(){ - - RunTest(getSampleWithLineBreak()); - - AssertGridCreate(project, 4, 6); - PrintProject(project); - - Row row = project.rows.get(3); - Assert.assertNotNull(row); - Assert.assertNotNull(row.cells); - Assert.assertNotNull(row.cells.get(2)); - Assert.assertEquals(row.cells.get(2).value, "With line\n break"); - } - - @Test(groups={"broken"}) - public void testElementsWithVaryingStructure(){ - - - RunTest(getSampleWithVaryingStructure()); - - AssertGridCreate(project, 5, 6); - PrintProject(project); - - Row row0 = project.rows.get(0); - Assert.assertNotNull(row0); - Assert.assertNotNull(row0.cells); - Assert.assertEquals(row0.cells.size(),6); - - Row row5 = project.rows.get(5); - Assert.assertNotNull(row5); - Assert.assertNotNull(row5.cells); - Assert.assertEquals(row5.cells.size(),6); - - - } - - //------------helper methods--------------- - - protected String getTypicalElement(int id){ - return "" + - "Author " + id + ", The" + - "Book title " + id + "" + - "2010-05-26" + - ""; - } - - protected String getSample(){ - StringBuilder sb = new StringBuilder(); - sb.append(""); - for(int i = 1; i < 7; i++){ - sb.append(getTypicalElement(i)); - } - sb.append(""); - return sb.toString(); - } - - protected String getSampleWithLineBreak(){ - StringBuilder sb = new StringBuilder(); - sb.append(""); - for(int i = 1; i < 4; i++){ - sb.append(getTypicalElement(i)); - } - sb.append("" + - "With line\n break" + - "Book title 4" + - "2010-05-26" + - ""); - sb.append(getTypicalElement(5)); - sb.append(getTypicalElement(6)); - sb.append(""); - return sb.toString(); - } - - protected String getSampleWithVaryingStructure(){ - StringBuilder sb = new StringBuilder(); - sb.append(""); - for(int i = 1; i < 6; i++){ - sb.append(getTypicalElement(i)); - } - sb.append("" + - "With line\n break" + - "Book title 6" + - "New element not seen in other records" + - "2010-05-26" + - ""); - sb.append(""); - return sb.toString(); - } - - private void RunTest(String testString){ - try { - inputStream = new ByteArrayInputStream( testString.getBytes( "UTF-8" ) ); - } catch (UnsupportedEncodingException e1) { - Assert.fail(); - } - - try { - SUT.read(inputStream, project, options); - } catch (Exception e) { - Assert.fail(); - } - - try { - inputStream.close(); - } catch (IOException e) { - Assert.fail(); - } - } - - private void AssertGridCreate(Project project, int numCols, int numRows){ - Assert.assertNotNull(project); - Assert.assertNotNull(project.columnModel); - Assert.assertNotNull(project.columnModel.columns); - Assert.assertEquals(project.columnModel.columns.size(), numCols); - Assert.assertNotNull(project.rows); - Assert.assertEquals(project.rows.size(), numRows); - } - - private void PrintProject(Project project){ - //some quick and dirty debugging - StringBuilder sb = new StringBuilder(); - for(Column c : project.columnModel.columns){ - sb.append(c.getName()); - sb.append("; "); - } - logger.info(sb.toString()); - for(Row r : project.rows){ - sb = new StringBuilder(); - for(Cell c : r.cells){ - if(c != null){ - sb.append(c.value); - sb.append("; "); - }else{ - sb.append("null; "); - } - } - logger.info(sb.toString()); - } - } -} +package com.metaweb.gridworks.tests.importers; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.Properties; + +import static org.mockito.Mockito.mock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.metaweb.gridworks.importers.XmlImporter; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + + +public class XmlImporterTests { + final static Logger logger = LoggerFactory.getLogger("XmlImporterTests"); + + //dependencies + Project project = null; + Properties options = null; + ByteArrayInputStream inputStream = null; + + //System Under Test + XmlImporter SUT = null; + + + @BeforeMethod + public void SetUp(){ + SUT = new XmlImporter(); + project = new Project(); + options = mock(Properties.class); + } + + @AfterMethod + public void TearDown() throws IOException{ + SUT = null; + project = null; + options = null; + inputStream.close(); + inputStream = null; + } + + @Test + public void canParseSample(){ + RunTest(getSample()); + + TestTools.AssertGridCreated(project, 4, 6); + TestTools.PrintProject(project); + + Row row = project.rows.get(0); + Assert.assertNotNull(row); + Assert.assertNotNull(row.getCell(1)); + Assert.assertEquals(row.getCell(1).value, "Author 1, The"); + } + + @Test + public void canParseSampleWithDuplicateNestedElements(){ + RunTest(getSampleWithDuplicateNestedElements()); + + TestTools.PrintProject(project); + TestTools.AssertGridCreated(project, 4, 12); + + Row row = project.rows.get(0); + Assert.assertNotNull(row); + Assert.assertEquals(row.cells.size(), 4); + Assert.assertNotNull(row.getCell(2)); + Assert.assertEquals(row.getCell(1).value, "Author 1, The"); + Assert.assertEquals(project.rows.get(1).getCell(1).value, "Author 1, Another"); + } + + @Test + public void testCanParseLineBreak(){ + + RunTest(getSampleWithLineBreak()); + + TestTools.AssertGridCreated(project, 4, 6); + TestTools.PrintProject(project); + + Row row = project.rows.get(3); + Assert.assertNotNull(row); + Assert.assertEquals(row.cells.size(), 4); + Assert.assertNotNull(row.getCell(1)); + Assert.assertEquals(row.getCell(1).value, "With line\n break"); + } + + @Test(groups={"broken"}) + public void testElementsWithVaryingStructure(){ + RunTest(getSampleWithVaryingStructure()); + + TestTools.AssertGridCreated(project, 5, 6); + TestTools.PrintProject(project); + + Row row0 = project.rows.get(0); + Assert.assertNotNull(row0); + Assert.assertEquals(row0.cells.size(),6); + + Row row5 = project.rows.get(5); + Assert.assertNotNull(row5); + Assert.assertEquals(row5.cells.size(),6); + } + + @Test + public void testElementWithNestedTree(){ + RunTest(getSampleWithTreeStructure()); + TestTools.AssertGridCreated(project, 5, 6); + TestTools.PrintProject(project); + Assert.assertEquals(project.columnModel.columnGroups.size(),1); + Assert.assertEquals(project.columnModel.columnGroups.get(0).keyColumnIndex, 2); + Assert.assertEquals(project.columnModel.columnGroups.get(0).startColumnIndex, 2); + Assert.assertNull(project.columnModel.columnGroups.get(0).parentGroup); + Assert.assertEquals(project.columnModel.columnGroups.get(0).subgroups.size(),0); + Assert.assertEquals(project.columnModel.columnGroups.get(0).columnSpan,2); + } + + //------------helper methods--------------- + + public static String getTypicalElement(int id){ + return "" + + "Author " + id + ", The" + + "Book title " + id + "" + + "2010-05-26" + + ""; + } + + public static String getElementWithDuplicateSubElement(int id){ + return "" + + "Author " + id + ", The" + + "Author " + id + ", Another" + + "Book title " + id + "" + + "2010-05-26" + + ""; + } + + public static String getSample(){ + StringBuilder sb = new StringBuilder(); + sb.append(""); + for(int i = 1; i < 7; i++){ + sb.append(getTypicalElement(i)); + } + sb.append(""); + return sb.toString(); + } + + public static String getSampleWithDuplicateNestedElements(){ + StringBuilder sb = new StringBuilder(); + sb.append(""); + for(int i = 1; i < 7; i++){ + sb.append(getElementWithDuplicateSubElement(i)); + } + sb.append(""); + return sb.toString(); + + } + + public static String getSampleWithLineBreak(){ + StringBuilder sb = new StringBuilder(); + sb.append(""); + for(int i = 1; i < 4; i++){ + sb.append(getTypicalElement(i)); + } + sb.append("" + + "With line\n break" + + "Book title 4" + + "2010-05-26" + + ""); + sb.append(getTypicalElement(5)); + sb.append(getTypicalElement(6)); + sb.append(""); + return sb.toString(); + } + + public static String getSampleWithVaryingStructure(){ + StringBuilder sb = new StringBuilder(); + sb.append(""); + for(int i = 1; i < 6; i++){ + sb.append(getTypicalElement(i)); + } + sb.append("" + + "Author 6, The" + + "Book title 6" + + "New element not seen in other records" + + "2010-05-26" + + ""); + sb.append(""); + return sb.toString(); + } + + public static String getSampleWithTreeStructure(){ + StringBuilder sb = new StringBuilder(); + sb.append(""); + for(int i = 1; i < 7; i++){ + sb.append("" + + "Author " + i + ", The" + + "1950-0" + i + "-15" + + "Book title " + i + "" + + "2010-05-26" + + ""); + } + sb.append(""); + return sb.toString(); + } + + private void RunTest(String testString){ + try { + inputStream = new ByteArrayInputStream( testString.getBytes( "UTF-8" ) ); + } catch (UnsupportedEncodingException e1) { + Assert.fail(); + } + + try { + SUT.read(inputStream, project, options); + } catch (Exception e) { + Assert.fail(); + } + } + + +} diff --git a/main/tests/server/src/log4j.properties b/main/tests/server/src/log4j.properties index 7ccee1634..adaedaa02 100644 --- a/main/tests/server/src/log4j.properties +++ b/main/tests/server/src/log4j.properties @@ -1,4 +1,7 @@ log4j.rootLogger=ERROR, console +log4j.logger.org.apache.http.headers=WARN +log4j.logger.org.apache.http.impl=WARN +log4j.logger.org.apache.http.client=WARN log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.layout=com.metaweb.util.logging.IndentingLayout +log4j.appender.console.layout=com.metaweb.gridworks.logging.IndentingLayout diff --git a/main/webapp/WEB-INF/web.xml b/main/webapp/WEB-INF/web.xml index 7c9d17f30..d3f38fef9 100644 --- a/main/webapp/WEB-INF/web.xml +++ b/main/webapp/WEB-INF/web.xml @@ -8,7 +8,6 @@ gridworks com.metaweb.gridworks.GridworksServlet - 1 diff --git a/server/IDEs/eclipse/Gridworks.launch b/server/IDEs/eclipse/Gridworks.launch index 003f3a3c0..bde9d67c1 100644 --- a/server/IDEs/eclipse/Gridworks.launch +++ b/server/IDEs/eclipse/Gridworks.launch @@ -1,12 +1,13 @@ - + - - + + + diff --git a/server/src/com/metaweb/gridworks/Gridworks.java b/server/src/com/metaweb/gridworks/Gridworks.java index f49b3a06d..011ba1286 100644 --- a/server/src/com/metaweb/gridworks/Gridworks.java +++ b/server/src/com/metaweb/gridworks/Gridworks.java @@ -24,11 +24,14 @@ import org.apache.log4j.Level; import org.mortbay.jetty.Connector; import org.mortbay.jetty.Server; import org.mortbay.jetty.bio.SocketConnector; +import org.mortbay.jetty.servlet.ServletHolder; import org.mortbay.jetty.webapp.WebAppContext; import org.mortbay.util.Scanner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.codeberry.jdatapath.DataPath; +import com.codeberry.jdatapath.JDataPathSystem; import com.metaweb.util.threads.ThreadPoolExecutorAdapter; /** @@ -37,38 +40,15 @@ import com.metaweb.util.threads.ThreadPoolExecutorAdapter; */ public class Gridworks { - static private final String VERSION = "1.0"; static private final String DEFAULT_HOST = "127.0.0.1"; static private final int DEFAULT_PORT = 3333; - static private final int MAX_UPLOAD_SIZE = 1024 * 1024 * 1024; - - static private File tempDir; + static private int port; static private String host; final static Logger logger = LoggerFactory.getLogger("gridworks"); - - public static String getVersion() { - return VERSION; - } - - public static File getTempFile(String name) { - return new File(tempDir, name); - } - - public static File getTempDir() { - return tempDir; - } - - public static int getMaxUploadSize() { - return Configurations.getInteger("gridworks.max_upload_size",MAX_UPLOAD_SIZE); - } - - public static String getFullHost() { - return host + ":" + port; - } - - public static void main(String[] args) throws Exception { + + public static void main(String[] args) throws Exception { // tell jetty to use SLF4J for logging instead of its own stuff System.setProperty("VERBOSE","false"); @@ -89,9 +69,6 @@ public class Gridworks { // set the log verbosity level org.apache.log4j.Logger.getRootLogger().setLevel(Level.toLevel(Configurations.get("gridworks.verbosity","info"))); - tempDir = new File(Configurations.get("gridworks.temp","temp")); - if (!tempDir.exists()) tempDir.mkdirs(); - port = Configurations.getInteger("gridworks.port",DEFAULT_PORT); host = Configurations.get("gridworks.host",DEFAULT_HOST); @@ -117,7 +94,8 @@ public class Gridworks { // hook up the signal handlers Runtime.getRuntime().addShutdownHook( - new Thread(new ShutdownSignalHandler(server))); + new Thread(new ShutdownSignalHandler(server)) + ); server.join(); } @@ -178,7 +156,15 @@ class GridworksServer extends Server { scanForUpdates(contextRoot, context); } + // start the server this.start(); + + // inject configuration parameters in the servlets + // NOTE: this is done *after* starting the server because jetty might override the init + // parameters if we set them in the webapp context upon reading the web.xml file + ServletHolder servlet = context.getServletHandler().getServlet("gridworks"); + servlet.setInitParameter("gridworks.data", getDataDir()); + servlet.doStart(); } @Override @@ -194,7 +180,7 @@ class GridworksServer extends Server { } } - private void scanForUpdates(final File contextRoot, final WebAppContext context) { + static private void scanForUpdates(final File contextRoot, final WebAppContext context) { List scanList = new ArrayList(); scanList.add(new File(contextRoot, "WEB-INF/web.xml")); @@ -225,7 +211,7 @@ class GridworksServer extends Server { scanner.start(); } - private void findFiles(final String extension, File baseDir, final Collection found) { + static private void findFiles(final String extension, File baseDir, final Collection found) { baseDir.listFiles(new FileFilter() { public boolean accept(File pathname) { if (pathname.isDirectory()) { @@ -238,6 +224,103 @@ class GridworksServer extends Server { }); } + static private String getDataDir() { + + String data_dir = Configurations.get("gridworks.data_dir"); + if (data_dir != null) { + return data_dir; + } + + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("windows")) { + try { + // NOTE(SM): finding the "local data app" in windows from java is actually a PITA + // see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java + // so we're using a library that uses JNI to ask directly the win32 APIs, + // it's not elegant but it's the safest bet. + + DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks"); + File data = new File(fixWindowsUnicodePath(localDataPath.getPath())); + data.mkdirs(); + return data.getAbsolutePath(); + } catch (Error e) { + /* + * The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll + * we include is compiled for 32-bit. In this case, we just have to dig up + * environment variables and try our best to find a user-specific path. + */ + + logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables"); + + File parentDir = null; + String appData = System.getenv("APPDATA"); + if (appData != null && appData.length() > 0) { + // e.g., C:\Users\[userid]\AppData\Roaming + parentDir = new File(appData); + } else { + String userProfile = System.getenv("USERPROFILE"); + if (userProfile != null && userProfile.length() > 0) { + // e.g., C:\Users\[userid] + parentDir = new File(userProfile); + } + } + + if (parentDir == null) { + parentDir = new File("."); + } + + File data = new File(parentDir, "Gridworks"); + data.mkdirs(); + + return data.getAbsolutePath(); + } + } else if (os.contains("mac os x")) { + // on macosx, use "~/Library/Application Support" + String home = System.getProperty("user.home"); + String data_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks"; + File data = new File(data_home); + data.mkdirs(); + return data.getAbsolutePath(); + } else { // most likely a UNIX flavor + // start with the XDG environment + // see http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html + String data_home = System.getenv("XDG_DATA_HOME"); + if (data_home == null) { // if not found, default back to ~/.local/share + String home = System.getProperty("user.home"); + if (home == null) home = "."; + data_home = home + "/.local/share"; + } + File data = new File(data_home + "/gridworks"); + data.mkdirs(); + return data.getAbsolutePath(); + } + } + + /** + * For Windows file paths that contain user IDs with non ASCII characters, + * those characters might get replaced with ?. We need to use the environment + * APPDATA value to substitute back the original user ID. + */ + static private String fixWindowsUnicodePath(String path) { + int q = path.indexOf('?'); + if (q < 0) { + return path; + } + int pathSep = path.indexOf(File.separatorChar, q); + + String goodPath = System.getenv("APPDATA"); + if (goodPath == null || goodPath.length() == 0) { + goodPath = System.getenv("USERPROFILE"); + if (!goodPath.endsWith(File.separator)) { + goodPath = goodPath + File.separator; + } + } + + int goodPathSep = goodPath.indexOf(File.separatorChar, q); + + return path.substring(0, q) + goodPath.substring(q, goodPathSep) + path.substring(pathSep); + } + } /* -------------- Gridworks Client ----------------- */ diff --git a/server/src/log4j.properties b/server/src/log4j.properties index 6d09a1788..fe5307d2f 100644 --- a/server/src/log4j.properties +++ b/server/src/log4j.properties @@ -1,9 +1,6 @@ log4j.rootLogger=INFO, console log4j.logger.org.mortbay.log=WARN log4j.logger.org.mortbay.jetty=ERROR -log4j.logger.org.apache.http.headers=WARN -log4j.logger.org.apache.http.impl=WARN -log4j.logger.org.apache.http.client=WARN log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.layout=com.metaweb.util.logging.IndentingLayout