s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

bringing the refactor branch up to speed with trunk

Browse Source 
(everything works like in trunk for now, although some tests still fail)


git-svn-id: http://google-refine.googlecode.com/svn/branches/split-refactor@915 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Stefano Mazzocchi 2010-05-30 18:18:59 +00:00
parent 081df24a97
commit e3fc7ab603
32 changed files with 1242 additions and 601 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.gitignore vendored
View File

@ -1,4 +1,8 @@
.DS_Store
build/
src/main/webapp/WEB-INF/classes/
tests/java/classes/
server/classes/
main/webapp/WEB-INF/classes/
main/tests/server/classes/
main/test-output/
appengine/classes/
tools/

2
build.xml
View File

@ -118,6 +118,7 @@
<src path="${webapp.src.dir}"/>
<classpath refid="webapp.class.path" />
</javac>
<copy file="${webapp.src.dir}/log4j.properties" tofile="${webapp.classes.dir}/log4j.properties"/>
</target>
<target name="build_tests" depends="build_webapp">
@ -125,6 +126,7 @@
<javac srcdir="${server.tests.src.dir}" destdir="${server.tests.classes.dir}" debug="true" includeAntRuntime="no">
<classpath refid="tests.class.path" />
</javac>
<copy file="${server.tests.src.dir}/log4j.properties" tofile="${server.tests.classes.dir}/log4j.properties"/>
</target>
<target name="build" depends="build_server, build_webapp"/>

3
main/IDEs/eclipse/GridworksTests.launch
View File

@ -6,7 +6,6 @@
</listAttribute>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.testng.remote.RemoteTestNG"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="gridworks"/>
<stringAttribute key="org.eclipse.jdt.launching.WORKING_DIRECTORY" value="${workspace_loc:gridworks/build}"/>
<mapAttribute key="org.testng.eclipse.ALL_CLASS_METHODS"/>
<listAttribute key="org.testng.eclipse.CLASS_TEST_LIST"/>
<stringAttribute key="org.testng.eclipse.COMPLIANCE_LEVEL" value="JDK"/>
@ -15,7 +14,7 @@
<stringAttribute key="org.testng.eclipse.LOG_LEVEL" value="2"/>
<listAttribute key="org.testng.eclipse.PACKAGE_TEST_LIST"/>
<listAttribute key="org.testng.eclipse.SUITE_TEST_LIST">
<listEntry value="tests/java/conf/tests.xml"/>
<listEntry value="tests/server/conf/tests.xml"/>
</listAttribute>
<intAttribute key="org.testng.eclipse.TYPE" value="3"/>
</launchConfiguration>

57
main/src/com/metaweb/gridworks/GridworksServlet.java
View File

@ -1,11 +1,13 @@
package com.metaweb.gridworks;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
@ -18,8 +20,12 @@ import com.metaweb.gridworks.commands.Command;
public class GridworksServlet extends HttpServlet {
static private final String VERSION = "1.0";
private static final long serialVersionUID = 2386057901503517403L;
private static final String JAVAX_SERVLET_CONTEXT_TEMPDIR = "javax.servlet.context.tempdir";
static final private Map<String, Command> commands = new HashMap<String, Command>();
// timer for periodically saving projects
@ -104,11 +110,12 @@ public class GridworksServlet extends HttpServlet {
{"mqlwrite", "com.metaweb.gridworks.commands.freebase.MQLWriteCommand"},
};
static {
registerCommands(commandNames);
public static String getVersion() {
return VERSION;
}
final static protected long s_autoSavePeriod = 1000 * 60 * 5; // 5 minutes
static protected class AutoSaveTimerTask extends TimerTask {
public void run() {
try {
@ -121,12 +128,21 @@ public class GridworksServlet extends HttpServlet {
}
}
protected ServletConfig config;
@Override
public void init() throws ServletException {
super.init();
logger.trace("> initialize");
ProjectManager.initialize();
String data = getInitParameter("gridworks.data");
if (data == null) {
throw new ServletException("can't find servlet init config 'gridworks.data', I have to give up initializing");
}
registerCommands(commandNames);
ProjectManager.initialize(new File(data));
if (_timer == null) {
_timer = new Timer("autosave");
@ -150,6 +166,8 @@ public class GridworksServlet extends HttpServlet {
ProjectManager.singleton = null;
}
this.config = null;
super.destroy();
logger.trace("< destroy");
@ -188,6 +206,26 @@ public class GridworksServlet extends HttpServlet {
return slash > 0 ? commandName.substring(0, slash) : commandName;
}
private File tempDir = null;
public File getTempDir() {
if (tempDir == null) {
File tempDir = (File) this.config.getServletContext().getAttribute(JAVAX_SERVLET_CONTEXT_TEMPDIR);
if (tempDir == null) {
throw new RuntimeException("This app server doesn't support temp directories");
}
}
return tempDir;
}
public File getTempFile(String name) {
return new File(getTempDir(), name);
}
public String getConfiguration(String name, String def) {
return null;
}
/**
* Register an array of commands
*
@ -198,7 +236,7 @@ public class GridworksServlet extends HttpServlet {
* the second.
* @return false if any commands failed to load
*/
static public boolean registerCommands(String[][] commands) {
private boolean registerCommands(String[][] commands) {
boolean status = true;
for (String[] command : commandNames) {
String commandName = command[0];
@ -206,8 +244,8 @@ public class GridworksServlet extends HttpServlet {
logger.debug("Loading command " + commandName + " class: " + className);
Command cmd;
try {
// TODO: May need to use the servlet container's class loader here
cmd = (Command) Class.forName(className).newInstance();
cmd = (Command) this.getClass().getClassLoader().loadClass(className).newInstance();
cmd.init(this);
} catch (InstantiationException e) {
logger.error("Failed to load command class " + className, e);
status = false;
@ -235,8 +273,7 @@ public class GridworksServlet extends HttpServlet {
* object implementing the command
* @return true if command was loaded and registered successfully
*/
static public boolean registerCommand(String name,
Command commandObject) {
protected boolean registerCommand(String name, Command commandObject) {
if (commands.containsKey(name)) {
return false;
}
@ -245,7 +282,7 @@ public class GridworksServlet extends HttpServlet {
}
// Currently only for test purposes
static protected boolean unregisterCommand(String verb) {
protected boolean unregisterCommand(String verb) {
return commands.remove(verb) != null;
}
}

103
main/src/com/metaweb/gridworks/ProjectManager.java
View File

@ -22,8 +22,6 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.codeberry.jdatapath.DataPath;
import com.codeberry.jdatapath.JDataPathSystem;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.util.JSONUtilities;
@ -58,112 +56,13 @@ public class ProjectManager {
static public ProjectManager singleton;
static public synchronized void initialize() {
static public synchronized void initialize(File dir) {
if (singleton == null) {
File dir = getProjectLocation();
logger.info("Using workspace directory: {}", dir.getAbsolutePath());
singleton = new ProjectManager(dir);
}
}
static protected File getProjectLocation() {
String data_dir = Configurations.get("gridworks.data_dir");
if (data_dir != null) {
return new File(data_dir);
}
String os = Configurations.get("os.name").toLowerCase();
if (os.contains("windows")) {
try {
// NOTE(SM): finding the "local data app" in windows from java is actually a PITA
// see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
// so we're using a library that uses JNI to ask directly the win32 APIs,
// it's not elegant but it's the safest bet.
DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
File data = new File(fixWindowsUnicodePath(localDataPath.getPath()));
data.mkdirs();
return data;
} catch (Error e) {
/*
* The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll
* we include is compiled for 32-bit. In this case, we just have to dig up
* environment variables and try our best to find a user-specific path.
*/
logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables");
File parentDir = null;
{
String appData = System.getenv("APPDATA");
if (appData != null && appData.length() > 0) {
// e.g., C:\Users\[userid]\AppData\Roaming
parentDir = new File(appData);
} else {
String userProfile = System.getenv("USERPROFILE");
if (userProfile != null && userProfile.length() > 0) {
// e.g., C:\Users\[userid]
parentDir = new File(userProfile);
}
}
}
if (parentDir == null) {
parentDir = new File(".");
}
File data = new File(parentDir, "Gridworks");
data.mkdirs();
return data;
}
} else if (os.contains("mac os x")) {
// on macosx, use "~/Library/Application Support"
String home = System.getProperty("user.home");
String data_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks";
File data = new File(data_home);
data.mkdirs();
return data;
} else { // most likely a UNIX flavor
// start with the XDG environment
// see http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
String data_home = System.getenv("XDG_DATA_HOME");
if (data_home == null) { // if not found, default back to ~/.local/share
String home = System.getProperty("user.home");
if (home == null) home = ".";
data_home = home + "/.local/share";
}
File data = new File(data_home + "/gridworks");
data.mkdirs();
return data;
}
}
/**
* For Windows file paths that contain user IDs with non ASCII characters,
* those characters might get replaced with ?. We need to use the environment
* APPDATA value to substitute back the original user ID.
*/
static protected String fixWindowsUnicodePath(String path) {
int q = path.indexOf('?');
if (q < 0) {
return path;
}
int pathSep = path.indexOf(File.separatorChar, q);
String goodPath = System.getenv("APPDATA");
if (goodPath == null || goodPath.length() == 0) {
goodPath = System.getenv("USERPROFILE");
if (!goodPath.endsWith(File.separator)) {
goodPath = goodPath + File.separator;
}
}
int goodPathSep = goodPath.indexOf(File.separatorChar, q);
return path.substring(0, q) + goodPath.substring(q, goodPathSep) + path.substring(pathSep);
}
private ProjectManager(File dir) {
_workspaceDir = dir;
_workspaceDir.mkdirs();

7
main/src/com/metaweb/gridworks/commands/Command.java
View File

@ -16,6 +16,7 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata;
@ -33,6 +34,12 @@ public abstract class Command {
final static protected Logger logger = LoggerFactory.getLogger("command");
protected GridworksServlet servlet;
public void init(GridworksServlet servlet) {
this.servlet = servlet;
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {

21
main/src/com/metaweb/gridworks/commands/auth/AuthorizeCommand.java
View File

@ -2,6 +2,8 @@ package com.metaweb.gridworks.commands.auth;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URI;
import java.net.URISyntaxException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@ -10,13 +12,12 @@ import javax.servlet.http.HttpServletResponse;
import oauth.signpost.OAuthConsumer;
import oauth.signpost.OAuthProvider;
import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.oauth.Credentials;
import com.metaweb.gridworks.oauth.OAuthUtilities;
import com.metaweb.gridworks.oauth.Provider;
public class AuthorizeCommand extends Command {
public class AuthorizeCommand extends Command {
private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier";
@ -124,7 +125,21 @@ public class AuthorizeCommand extends Command {
private String getBaseURL(HttpServletRequest request, Provider provider) {
String host = request.getHeader("host");
if (host == null) host = Gridworks.getFullHost();
if (host == null) {
String referrer = request.getHeader("referer");
if (referrer != null) {
URI url;
try {
url = new URI(referrer);
int port = url.getPort();
host = url.getHost() + ((port > -1) ? ":" + url.getPort() : "");
} catch (URISyntaxException e) {
throw new RuntimeException("referrer '" + referrer + "' can't be parsed as a URL");
}
} else {
throw new RuntimeException("neither the 'host' nor 'referer' headers were present in the HTTP response, I can't determine what URL gridworks is listening to.");
}
}
return "http://" + host + "/command/authorize/" + provider.getHost();
}
}

3
main/src/com/metaweb/gridworks/commands/project/CreateProjectCommand.java
View File

@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata;
import com.metaweb.gridworks.commands.Command;
@ -367,7 +366,7 @@ public class CreateProjectCommand extends Command {
}
private File save(InputStream is) throws IOException {
File temp = Gridworks.getTempFile(Long.toString(System.currentTimeMillis()));
File temp = this.servlet.getTempFile(Long.toString(System.currentTimeMillis()));
temp.deleteOnExit();
IOUtils.copy(is,temp);
is.close();

116
main/src/com/metaweb/gridworks/exporters/CsvExporter.java
View File

@ -40,66 +40,66 @@ public class CsvExporter implements Exporter{
@Override
public void export(Project project, Properties options, Engine engine, Writer writer) throws IOException {
{
boolean printColumnHeader = true;
if(options != null)
printColumnHeader = options.getProperty("printColumnHeader")=="false"?false:true;
boolean printColumnHeader = true;
RowVisitor visitor = new RowVisitor() {
CSVWriter csvWriter;
boolean printColumnHeader = true;
boolean isFirstRow = true; //the first row should also add the column headers
public RowVisitor init(CSVWriter writer, boolean printColumnHeader){
this.csvWriter = writer;
this.printColumnHeader = printColumnHeader;
return this;
}
public boolean visit(Project project, int rowIndex, Row row) {
String[] cols = new String[project.columnModel.columns.size()];
String[] vals = new String[row.cells.size()];
int i = 0;
for(Column col : project.columnModel.columns){
int cellIndex = col.getCellIndex();
cols[i] = col.getName();
Cell cell = row.cells.get(cellIndex);
if(cell != null){
vals[i] = cell.value.toString();
}
i++;
}
if( printColumnHeader && isFirstRow ){
csvWriter.writeNext(cols,false);
isFirstRow = false; //switch off flag
}
csvWriter.writeNext(vals,false);
return false;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
try {
csvWriter.close();
} catch (IOException e) {
logger.error("CsvExporter could not close writer : " + e.getMessage());
}
}
}.init(new CSVWriter(writer, separator), printColumnHeader);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, visitor);
if (options != null) {
printColumnHeader = Boolean.parseBoolean(options.getProperty("printColumnHeader"));
}
RowVisitor visitor = new RowVisitor() {
CSVWriter csvWriter;
boolean printColumnHeader = true;
boolean isFirstRow = true; //the first row should also add the column headers
public RowVisitor init(CSVWriter writer, boolean printColumnHeader){
this.csvWriter = writer;
this.printColumnHeader = printColumnHeader;
return this;
}
public boolean visit(Project project, int rowIndex, Row row) {
String[] cols = new String[project.columnModel.columns.size()];
String[] vals = new String[row.cells.size()];
int i = 0;
for(Column col : project.columnModel.columns){
int cellIndex = col.getCellIndex();
cols[i] = col.getName();
Cell cell = row.cells.get(cellIndex);
if(cell != null){
vals[i] = cell.value.toString();
}
i++;
}
if( printColumnHeader && isFirstRow ){
csvWriter.writeNext(cols,false);
isFirstRow = false; //switch off flag
}
csvWriter.writeNext(vals,false);
return false;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
try {
csvWriter.close();
} catch (IOException e) {
logger.error("CsvExporter could not close writer : " + e.getMessage());
}
}
}.init(new CSVWriter(writer, separator), printColumnHeader);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, visitor);
}
@Override

2
main/src/com/metaweb/gridworks/expr/JythonEvaluable.java
View File

@ -19,7 +19,7 @@ public class JythonEvaluable implements Evaluable {
private static PythonInterpreter _engine;
static {
File libPath = new File("lib/jython");
File libPath = new File("webapp/WEB-INF/lib/jython");
if (libPath.exists()) {
Properties props = new Properties();
props.setProperty("python.path", libPath.getAbsolutePath());

4
main/src/com/metaweb/gridworks/history/History.java
View File

@ -16,7 +16,7 @@ import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.model.Project;
@ -68,7 +68,7 @@ public class History implements Jsonizable {
}
static public void writeOneChange(Writer writer, Change change, Properties options) throws IOException {
writer.write(Gridworks.getVersion()); writer.write('\n');
writer.write(GridworksServlet.getVersion()); writer.write('\n');
writer.write(change.getClass().getName()); writer.write('\n');
change.save(writer, options);

16
main/src/com/metaweb/gridworks/importers/XmlImportUtilities.java
View File

@ -72,8 +72,6 @@ public class XmlImportUtilities {
}
static public String[] detectPathFromTag(InputStream inputStream, String tag) {
//List<RecordElementCandidate> candidates = new ArrayList<RecordElementCandidate>();
try {
XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(inputStream);
@ -99,6 +97,8 @@ public class XmlImportUtilities {
}
static protected List<String> detectRecordElement(XMLStreamReader parser, String tag) throws XMLStreamException {
if(parser.getEventType() == XMLStreamConstants.START_DOCUMENT)
parser.next();
String localName = parser.getLocalName();
String fullName = composeName(parser.getPrefix(), localName);
if (tag.equals(parser.getLocalName()) || tag.equals(fullName)) {
@ -327,6 +327,10 @@ public class XmlImportUtilities {
int pathIndex,
ImportColumnGroup rootColumnGroup
) throws XMLStreamException {
if(parser.getEventType() == XMLStreamConstants.START_DOCUMENT){
logger.warn("Cannot use findRecord method for START_DOCUMENT event");
return;
}
String tagName = parser.getLocalName();
if (tagName.equals(recordPath[pathIndex])) {
if (pathIndex < recordPath.length - 1) {
@ -466,7 +470,7 @@ public class XmlImportUtilities {
ImportRecord record,
String columnLocalName,
String text,
int commonStaringRowIndex
int commonStartingRowIndex
) {
if (text == null || ((String) text).isEmpty()) {
return;
@ -478,7 +482,7 @@ public class XmlImportUtilities {
int cellIndex = column.cellIndex;
while (cellIndex >= record.columnEmptyRowIndices.size()) {
record.columnEmptyRowIndices.add(commonStaringRowIndex);
record.columnEmptyRowIndices.add(commonStartingRowIndex);
}
int rowIndex = record.columnEmptyRowIndices.get(cellIndex);
@ -491,7 +495,9 @@ public class XmlImportUtilities {
row.add(null);
}
row.set(cellIndex, new Cell(value, null));
logger.trace("Adding cell with value : " + value + " to row : " + rowIndex + " at cell index : " + (cellIndex-1));
row.set(cellIndex-1, new Cell(value, null));
record.columnEmptyRowIndices.set(cellIndex, rowIndex + 1);

3
main/src/com/metaweb/gridworks/importers/XmlImporter.java
View File

@ -57,6 +57,9 @@ public class XmlImporter implements Importer {
}
}
if(recordPath == null)
return;
ImportColumnGroup rootColumnGroup = new ImportColumnGroup();
XmlImportUtilities.importXml(pis, project, recordPath, rootColumnGroup);

143
main/src/com/metaweb/gridworks/logging/IndentingLayout.java Normal file
View File

@ -0,0 +1,143 @@
package com.metaweb.gridworks.logging;
/*
* Copyright (c) Massachusetts Institute of Technology, 2007
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* Original code: http://simile.mit.edu/repository/tracer/trunk/
*/
import java.util.Calendar;
import java.util.Date;
import org.apache.log4j.Layout;
import org.apache.log4j.spi.LoggingEvent;
/**
* This is a special Log4j log formatter that is capable of reacting on special log messages
* and 'indent' the logs accordingly. This is very useful to visually inspect a debug log
* and see what calls what. An example of logs are "> method()" and "< method()" where > and <
* are used to indicate respectively "entering" and "exiting".
*/
public class IndentingLayout extends Layout {
protected static final int CONTEXT_SIZE = 25;
protected static final long MAX_DELTA = 10000;
protected Calendar calendar = Calendar.getInstance();
protected long previousTime = 0;
protected int indentation = 0;
public void activateOptions() {
// no options at this time
}
public String format(LoggingEvent event) {
String message = event.getRenderedMessage();
if (message == null) return "";
if (message.length() < 2) return message;
char leader = message.charAt(0);
char secondLeader = message.charAt(1);
if ((leader == '<') && (secondLeader == ' ') && (this.indentation > 0)) this.indentation--;
// Reset buf
StringBuffer buf = new StringBuffer(256);
Date date = new Date();
long now = date.getTime();
calendar.setTime(date);
long delta = 0;
if (previousTime > 0) {
delta = now - previousTime;
}
previousTime = now;
// if ((previousTime == 0) || (delta > MAX_DELTA)) {
// buf.append('\n');
// indentation = 0; // reset indentation after a while, as we might
// // have runaway/unmatched log entries
// }
int hour = calendar.get(Calendar.HOUR_OF_DAY);
if (hour < 10) buf.append('0');
buf.append(hour);
buf.append(':');
int mins = calendar.get(Calendar.MINUTE);
if (mins < 10) buf.append('0');
buf.append(mins);
buf.append(':');
int secs = calendar.get(Calendar.SECOND);
if (secs < 10) buf.append('0');
buf.append(secs);
buf.append('.');
int millis = (int) (now % 1000);
if (millis < 100) buf.append('0');
if (millis < 10) buf.append('0');
buf.append(millis);
buf.append(" [");
String context = ((String) event.getMDC("LogEvent"));
if (context == null) {
context = event.getLoggerName();
}
if (context.length() < CONTEXT_SIZE) {
pad(buf, CONTEXT_SIZE - context.length(), ' ');
buf.append(context);
} else {
buf.append("..");
buf.append(context.substring(context.length() - CONTEXT_SIZE + 2));
}
buf.append("] ");
pad(buf, indentation, ' ');
buf.append(message);
buf.append(" (");
buf.append(delta);
buf.append("ms)\n");
if ((leader == '>') && (secondLeader == ' ')) indentation++;
return buf.toString();
}
private void pad(StringBuffer buffer, int pads, char padchar) {
for (int i = 0; i < pads; i++) {
buffer.append(padchar);
}
}
public boolean ignoresThrowable() {
return true;
}
}

4
main/src/com/metaweb/gridworks/model/Project.java
View File

@ -19,7 +19,7 @@ import java.util.zip.ZipOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata;
import com.metaweb.gridworks.history.History;
@ -128,7 +128,7 @@ public class Project {
}
protected void saveToWriter(Writer writer, Properties options) throws IOException {
writer.write(Gridworks.getVersion()); writer.write('\n');
writer.write(GridworksServlet.getVersion()); writer.write('\n');
writer.write("columnModel=\n"); columnModel.save(writer, options);
writer.write("history=\n"); history.save(writer, options);

10
main/src/com/metaweb/gridworks/util/FreebaseUtils.java
View File

@ -27,7 +27,7 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.oauth.Credentials;
import com.metaweb.gridworks.oauth.OAuthUtilities;
import com.metaweb.gridworks.oauth.Provider;
@ -59,7 +59,7 @@ public class FreebaseUtils {
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion());
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
// this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1");
@ -102,7 +102,7 @@ public class FreebaseUtils {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion());
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS
@ -128,7 +128,7 @@ public class FreebaseUtils {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion());
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS
@ -176,7 +176,7 @@ public class FreebaseUtils {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(FREEQ_URL);
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion());
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
httpRequest.setEntity(entity);
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));

4
main/src/com/metaweb/gridworks/util/Pool.java
View File

@ -15,7 +15,7 @@ import java.util.Map.Entry;
import org.json.JSONException;
import org.json.JSONWriter;
import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.ReconCandidate;
@ -59,7 +59,7 @@ public class Pool implements Jsonizable {
}
public void save(Writer writer) throws IOException {
writer.write(Gridworks.getVersion()); writer.write('\n');
writer.write(GridworksServlet.getVersion()); writer.write('\n');
Properties options = new Properties();
options.setProperty("mode", "save");

7
main/src/log4j.properties Normal file
View File

@ -0,0 +1,7 @@
log4j.rootLogger=INFO, console
log4j.logger.org.apache.http.headers=WARN
log4j.logger.org.apache.http.impl=WARN
log4j.logger.org.apache.http.client=WARN
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=com.metaweb.gridworks.logging.IndentingLayout

0
main/tests/client/js/facets.js → main/tests/client/src/facets.js
View File

0
main/tests/client/js/initialize.js → main/tests/client/src/initialize.js
View File

4
main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletStub.java
View File

@ -36,7 +36,7 @@ public class GridworksServletStub extends GridworksServlet {
* @param commandName
* @param command
*/
static public void InsertCommand( String commandName, Command command ){
public void insertCommand( String commandName, Command command ){
registerCommand(commandName, command);
}
@ -44,7 +44,7 @@ public class GridworksServletStub extends GridworksServlet {
* Helper method for clearing up after testing
* @param commandName
*/
static public void RemoveCommand( String commandName ){
public void removeCommand( String commandName ){
unregisterCommand(commandName);
}
}

7
main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletTests.java
View File

@ -40,21 +40,22 @@ public class GridworksServletTests {
@BeforeMethod
public void SetUp() {
SUT = new GridworksServletStub();
request = mock(HttpServletRequest.class);
response = mock(HttpServletResponse.class);
command = mock(Command.class);
GridworksServletStub.InsertCommand(TEST_COMMAND_NAME,command); //inject mock into command container
SUT = new GridworksServletStub();
SUT.insertCommand(TEST_COMMAND_NAME,command); //inject mock into command container
}
@AfterMethod
public void TearDown() {
SUT.removeCommand(TEST_COMMAND_NAME); //remove mock to clean command container
SUT = null;
request = null;
response = null;
command = null;
GridworksServletStub.RemoveCommand(TEST_COMMAND_NAME); //remove mock to clean command container
}
//-------------------AutoSaveTimerTask tests-----------

50
main/tests/server/src/com/metaweb/gridworks/tests/importers/TestTools.java Normal file
View File

@ -0,0 +1,50 @@
package com.metaweb.gridworks.tests.importers;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
/**
* Helper methods for Importer testing
*
*/
public class TestTools {
final static Logger logger = LoggerFactory.getLogger("Common");
public static void AssertGridCreated(Project project, int numCols, int numRows){
Assert.assertNotNull(project);
Assert.assertNotNull(project.columnModel);
Assert.assertNotNull(project.columnModel.columns);
Assert.assertEquals(project.columnModel.columns.size(), numCols);
Assert.assertNotNull(project.rows);
Assert.assertEquals(project.rows.size(), numRows);
}
public static void PrintProject(Project project){
//some quick and dirty debugging
StringBuilder sb = new StringBuilder();
for(Column c : project.columnModel.columns){
sb.append(c.getName());
sb.append("; ");
}
logger.info(sb.toString());
for(Row r : project.rows){
sb = new StringBuilder();
for(int i = 0; i < r.cells.size(); i++){
Cell c = r.getCell(i);
if(c != null){
sb.append(c.value);
sb.append("; ");
}else{
sb.append("null; ");
}
}
logger.info(sb.toString());
}
}
}

32
main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesStub.java Normal file
View File

@ -0,0 +1,32 @@
package com.metaweb.gridworks.tests.importers;
import java.util.List;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import com.metaweb.gridworks.importers.XmlImportUtilities;
import com.metaweb.gridworks.model.Project;
public class XmlImportUtilitiesStub extends XmlImportUtilities{
public List<String> detectRecordElementWrapper(XMLStreamReader parser, String tag) throws XMLStreamException{
return super.detectRecordElement(parser, tag);
}
public void ProcessSubRecordWrapper(Project project, XMLStreamReader parser, ImportColumnGroup columnGroup, ImportRecord record) throws XMLStreamException{
super.processSubRecord(project, parser, columnGroup, record);
}
public void findRecordWrapper(Project project, XMLStreamReader parser, String[] recordPath, int pathIndex, ImportColumnGroup rootColumnGroup) throws XMLStreamException{
super.findRecord(project, parser, recordPath, pathIndex, rootColumnGroup);
}
public void processRecordWrapper(Project project, XMLStreamReader parser, ImportColumnGroup rootColumnGroup) throws XMLStreamException{
super.processRecord(project, parser, rootColumnGroup);
}
public void addCellWrapper(Project project, ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, String text, int commonStartingRowIndex){
super.addCell(project, columnGroup, record, columnLocalName, text, commonStartingRowIndex);
}
}

336
main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesTests.java Normal file
View File

@ -0,0 +1,336 @@
package com.metaweb.gridworks.tests.importers;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import javax.xml.stream.FactoryConfigurationError;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import org.apache.log4j.Level;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.metaweb.gridworks.importers.XmlImportUtilities.ImportColumn;
import com.metaweb.gridworks.importers.XmlImportUtilities.ImportColumnGroup;
import com.metaweb.gridworks.importers.XmlImportUtilities.ImportRecord;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
public class XmlImportUtilitiesTests {
final static Logger logger = LoggerFactory.getLogger("XmlImporterUtilitiesTests");
//dependencies
Project project;
XMLStreamReader parser;
ImportColumnGroup columnGroup;
ImportRecord record;
ByteArrayInputStream inputStream;
//System Under Test
XmlImportUtilitiesStub SUT;
@BeforeMethod
public void SetUp(){
org.apache.log4j.Logger.getRootLogger().setLevel(Level.toLevel("trace"));
SUT = new XmlImportUtilitiesStub();
project = new Project();
columnGroup = new ImportColumnGroup();
record = new ImportRecord();
}
@AfterMethod
public void TearDown() throws IOException{
SUT = null;
project = null;
parser = null;
columnGroup = null;
record = null;
if(inputStream != null)
inputStream.close();
inputStream = null;
}
@Test
public void detectPathFromTagTest(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
String tag = "library";
String[] response = XmlImportUtilitiesStub.detectPathFromTag(inputStream, tag);
Assert.assertNotNull(response);
Assert.assertEquals(response.length, 1);
Assert.assertEquals(response[0], "library");
}
@Test
public void detectPathFromTagWithNestedElement(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
String tag = "book";
String[] response = XmlImportUtilitiesStub.detectPathFromTag(inputStream, tag);
Assert.assertNotNull(response);
Assert.assertEquals(response.length, 2);
Assert.assertEquals(response[0], "library");
Assert.assertEquals(response[1], "book");
}
@Test
public void detectRecordElementTest(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
createParser();
String tag="library";
List<String> response = new ArrayList<String>();
try {
response = SUT.detectRecordElementWrapper(parser, tag);
} catch (XMLStreamException e) {
Assert.fail();
}
Assert.assertNotNull(response);
Assert.assertEquals(response.size(), 1);
Assert.assertEquals(response.get(0), "library");
}
@Test
public void detectRecordElementCanHandleWithNestedElements(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
createParser();
String tag="book";
List<String> response = new ArrayList<String>();
try {
response = SUT.detectRecordElementWrapper(parser, tag);
} catch (XMLStreamException e) {
Assert.fail();
}
Assert.assertNotNull(response);
Assert.assertEquals(response.size(), 2);
Assert.assertEquals(response.get(0), "library");
Assert.assertEquals(response.get(1), "book");
}
@Test
public void detectRecordElementIsNullForUnfoundTag(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
createParser();
String tag="";
List<String> response = new ArrayList<String>();
try {
response = SUT.detectRecordElementWrapper(parser, tag);
} catch (XMLStreamException e) {
Assert.fail();
}
Assert.assertNull(response);
}
@Test
public void detectRecordElementRegressionTest(){
loadSampleXml();
String[] path = XmlImportUtilitiesStub.detectRecordElement(inputStream);
Assert.assertNotNull(path);
Assert.assertEquals(path.length, 2);
Assert.assertEquals(path[0], "library");
Assert.assertEquals(path[1], "book");
}
@Test
public void importXmlTest(){
loadSampleXml();
String[] recordPath = new String[]{"library","book"};
XmlImportUtilitiesStub.importXml(inputStream, project, recordPath, columnGroup );
TestTools.PrintProject(project);
TestTools.AssertGridCreated(project, 0, 6);
Assert.assertEquals(project.rows.get(0).cells.size(), 4);
//TODO
}
@Test
public void createColumnsFromImportTest(){
ImportColumnGroup columnGroup = new ImportColumnGroup();
ImportColumn ic1 = new ImportColumn();
ic1.name = "hello";
ImportColumn ic2 = new ImportColumn();
ic2.name = "world";
ImportColumnGroup subGroup = new ImportColumnGroup();
ImportColumn ic3 = new ImportColumn();
ic3.name = "foo";
ImportColumn ic4 = new ImportColumn();
ic4.name = "bar";
subGroup.columns.put("c", ic3);
subGroup.columns.put("d", ic4);
columnGroup.columns.put("a", ic1);
columnGroup.columns.put("b", ic2);
columnGroup.subgroups.put("e", subGroup);
XmlImportUtilitiesStub.createColumnsFromImport(project, columnGroup);
TestTools.PrintProject(project);
TestTools.AssertGridCreated(project, 4, 0);
Assert.assertEquals(project.columnModel.columns.get(0).getName(), "world");
Assert.assertEquals(project.columnModel.columns.get(1).getName(), "hello");
Assert.assertEquals(project.columnModel.columns.get(2).getName(), "bar");
Assert.assertEquals(project.columnModel.columns.get(3).getName(), "foo");
Assert.assertEquals(project.columnModel.columnGroups.get(0).keyColumnIndex, 2);
Assert.assertEquals(project.columnModel.columnGroups.get(0).startColumnIndex, 2);
Assert.assertEquals(project.columnModel.columnGroups.get(0).columnSpan, 2);
}
@Test
public void findRecordTest(){
loadSampleXml();
createParser();
ParserSkip();
String[] recordPath = new String[]{"library","book"};
int pathIndex = 0;
try {
SUT.findRecordWrapper(project, parser, recordPath, pathIndex, columnGroup);
} catch (XMLStreamException e) {
Assert.fail();
}
TestTools.PrintProject(project);
TestTools.AssertGridCreated(project, 0, 6);
Assert.assertEquals(project.rows.get(0).cells.size(), 4);
//TODO
}
@Test
public void processRecordTest(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
createParser();
ParserSkip();
try {
SUT.processRecordWrapper(project, parser, columnGroup);
} catch (XMLStreamException e) {
Assert.fail();
}
TestTools.PrintProject(project);
Assert.assertNotNull(project.rows);
Assert.assertEquals(project.rows.size(), 1);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "author1");
}
@Test
public void processRecordTestDuplicateColumns(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><author>author2</author><genre>genre1</genre></book></library>");
createParser();
ParserSkip();
try {
SUT.processRecordWrapper(project, parser, columnGroup);
} catch (XMLStreamException e) {
Assert.fail();
}
TestTools.PrintProject(project);
Assert.assertNotNull(project.rows);
Assert.assertEquals(project.rows.size(), 2);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
Assert.assertEquals(row.cells.size(), 3);
Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "author1");
row = project.rows.get(1);
Assert.assertEquals(row.getCell(1).value, "author2");
}
@Test
public void processRecordTestNestedElement(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author><author-name>author1</author-name><author-dob>a date</author-dob></author><genre>genre1</genre></book></library>");
createParser();
ParserSkip();
try {
SUT.processRecordWrapper(project, parser, columnGroup);
} catch (XMLStreamException e) {
Assert.fail();
}
TestTools.PrintProject(project);
Assert.assertNotNull(project.rows);
Assert.assertEquals(project.rows.size(), 1);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
Assert.assertEquals(row.cells.size(), 4);
Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "author1");
Assert.assertNotNull(row.getCell(2));
Assert.assertEquals(row.getCell(2).value, "a date");
}
@Test(groups={"broken"})
public void processSubRecordTest(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
createParser();
ParserSkip();
try {
SUT.ProcessSubRecordWrapper(project, parser, columnGroup, record);
} catch (XMLStreamException e) {
Assert.fail();
}
TestTools.PrintProject(project);
Assert.fail();
//TODO need to verify 'record' was set correctly which we can't do as ImportRecord is an internal class
}
@Test(groups={"broken"})
public void addCellTest(){
String columnLocalName = "author";
String text = "Author1, The";
int commonStartingRowIndex = 0;
project.rows.add(new Row(0));
SUT.addCellWrapper(project, columnGroup, record, columnLocalName, text, commonStartingRowIndex);
Assert.fail();
//TODO need to verify 'record' was set correctly which we can't do as ImportRecord is an internal class
}
//----------------helpers-------------
public void loadSampleXml(){
loadXml( XmlImporterTests.getSample() );
}
public void loadXml(String xml){
try {
inputStream = new ByteArrayInputStream( xml.getBytes( "UTF-8" ) );
} catch (UnsupportedEncodingException e1) {
Assert.fail();
}
}
public void ParserSkip(){
try {
parser.next(); //move parser forward once e.g. skip the START_DOCUMENT parser event
} catch (XMLStreamException e1) {
Assert.fail();
}
}
public void createParser(){
try {
parser = XMLInputFactory.newInstance().createXMLStreamReader(inputStream);
} catch (XMLStreamException e1) {
Assert.fail();
} catch (FactoryConfigurationError e1) {
Assert.fail();
}
}
}

140
main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImporterTests.java
View File

@ -15,8 +15,6 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.metaweb.gridworks.importers.XmlImporter;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
@ -41,27 +39,40 @@ public class XmlImporterTests {
}
@AfterMethod
public void TearDown(){
public void TearDown() throws IOException{
SUT = null;
project = null;
options = null;
inputStream.close();
inputStream = null;
}
@Test
public void canParseSample(){
RunTest(getSample());
AssertGridCreate(project, 4, 6);
PrintProject(project);
TestTools.AssertGridCreated(project, 4, 6);
TestTools.PrintProject(project);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
Assert.assertNotNull(row.cells);
Assert.assertNotNull(row.cells.get(2));
Assert.assertEquals(row.cells.get(2).value, "Author 1, The");
Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "Author 1, The");
}
@Test
public void canParseSampleWithDuplicateNestedElements(){
RunTest(getSampleWithDuplicateNestedElements());
TestTools.PrintProject(project);
TestTools.AssertGridCreated(project, 4, 12);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
Assert.assertEquals(row.cells.size(), 4);
Assert.assertNotNull(row.getCell(2));
Assert.assertEquals(row.getCell(1).value, "Author 1, The");
Assert.assertEquals(project.rows.get(1).getCell(1).value, "Author 1, Another");
}
@Test
@ -69,41 +80,48 @@ public class XmlImporterTests {
RunTest(getSampleWithLineBreak());
AssertGridCreate(project, 4, 6);
PrintProject(project);
TestTools.AssertGridCreated(project, 4, 6);
TestTools.PrintProject(project);
Row row = project.rows.get(3);
Assert.assertNotNull(row);
Assert.assertNotNull(row.cells);
Assert.assertNotNull(row.cells.get(2));
Assert.assertEquals(row.cells.get(2).value, "With line\n break");
Assert.assertEquals(row.cells.size(), 4);
Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "With line\n break");
}
@Test(groups={"broken"})
public void testElementsWithVaryingStructure(){
RunTest(getSampleWithVaryingStructure());
AssertGridCreate(project, 5, 6);
PrintProject(project);
TestTools.AssertGridCreated(project, 5, 6);
TestTools.PrintProject(project);
Row row0 = project.rows.get(0);
Assert.assertNotNull(row0);
Assert.assertNotNull(row0.cells);
Assert.assertEquals(row0.cells.size(),6);
Row row5 = project.rows.get(5);
Assert.assertNotNull(row5);
Assert.assertNotNull(row5.cells);
Assert.assertEquals(row5.cells.size(),6);
}
@Test
public void testElementWithNestedTree(){
RunTest(getSampleWithTreeStructure());
TestTools.AssertGridCreated(project, 5, 6);
TestTools.PrintProject(project);
Assert.assertEquals(project.columnModel.columnGroups.size(),1);
Assert.assertEquals(project.columnModel.columnGroups.get(0).keyColumnIndex, 2);
Assert.assertEquals(project.columnModel.columnGroups.get(0).startColumnIndex, 2);
Assert.assertNull(project.columnModel.columnGroups.get(0).parentGroup);
Assert.assertEquals(project.columnModel.columnGroups.get(0).subgroups.size(),0);
Assert.assertEquals(project.columnModel.columnGroups.get(0).columnSpan,2);
}
//------------helper methods---------------
protected String getTypicalElement(int id){
public static String getTypicalElement(int id){
return "<book id=\"" + id + "\">" +
"<author>Author " + id + ", The</author>" +
"<title>Book title " + id + "</title>" +
@ -111,7 +129,16 @@ public class XmlImporterTests {
"</book>";
}
protected String getSample(){
public static String getElementWithDuplicateSubElement(int id){
return "<book id=\"" + id + "\">" +
"<author>Author " + id + ", The</author>" +
"<author>Author " + id + ", Another</author>" +
"<title>Book title " + id + "</title>" +
"<publish_date>2010-05-26</publish_date>" +
"</book>";
}
public static String getSample(){
StringBuilder sb = new StringBuilder();
sb.append("<?xml version=\"1.0\"?><library>");
for(int i = 1; i < 7; i++){
@ -121,7 +148,18 @@ public class XmlImporterTests {
return sb.toString();
}
protected String getSampleWithLineBreak(){
public static String getSampleWithDuplicateNestedElements(){
StringBuilder sb = new StringBuilder();
sb.append("<?xml version=\"1.0\"?><library>");
for(int i = 1; i < 7; i++){
sb.append(getElementWithDuplicateSubElement(i));
}
sb.append("</library>");
return sb.toString();
}
public static String getSampleWithLineBreak(){
StringBuilder sb = new StringBuilder();
sb.append("<?xml version=\"1.0\"?><library>");
for(int i = 1; i < 4; i++){
@ -138,14 +176,14 @@ public class XmlImporterTests {
return sb.toString();
}
protected String getSampleWithVaryingStructure(){
public static String getSampleWithVaryingStructure(){
StringBuilder sb = new StringBuilder();
sb.append("<?xml version=\"1.0\"?><library>");
for(int i = 1; i < 6; i++){
sb.append(getTypicalElement(i));
}
sb.append("<book id=\"6\">" +
"<author>With line\n break</author>" +
"<author>Author 6, The</author>" +
"<title>Book title 6</title>" +
"<genre>New element not seen in other records</genre>" +
"<publish_date>2010-05-26</publish_date>" +
@ -154,6 +192,21 @@ public class XmlImporterTests {
return sb.toString();
}
public static String getSampleWithTreeStructure(){
StringBuilder sb = new StringBuilder();
sb.append("<?xml version=\"1.0\"?><library>");
for(int i = 1; i < 7; i++){
sb.append("<book id=\"" + i + "\">" +
"<author><author-name>Author " + i + ", The</author-name>" +
"<author-dob>1950-0" + i + "-15</author-dob></author>" +
"<title>Book title " + i + "</title>" +
"<publish_date>2010-05-26</publish_date>" +
"</book>");
}
sb.append("</library>");
return sb.toString();
}
private void RunTest(String testString){
try {
inputStream = new ByteArrayInputStream( testString.getBytes( "UTF-8" ) );
@ -166,42 +219,7 @@ public class XmlImporterTests {
} catch (Exception e) {
Assert.fail();
}
try {
inputStream.close();
} catch (IOException e) {
Assert.fail();
}
}
private void AssertGridCreate(Project project, int numCols, int numRows){
Assert.assertNotNull(project);
Assert.assertNotNull(project.columnModel);
Assert.assertNotNull(project.columnModel.columns);
Assert.assertEquals(project.columnModel.columns.size(), numCols);
Assert.assertNotNull(project.rows);
Assert.assertEquals(project.rows.size(), numRows);
}
private void PrintProject(Project project){
//some quick and dirty debugging
StringBuilder sb = new StringBuilder();
for(Column c : project.columnModel.columns){
sb.append(c.getName());
sb.append("; ");
}
logger.info(sb.toString());
for(Row r : project.rows){
sb = new StringBuilder();
for(Cell c : r.cells){
if(c != null){
sb.append(c.value);
sb.append("; ");
}else{
sb.append("null; ");
}
}
logger.info(sb.toString());
}
}
}

5
main/tests/server/src/log4j.properties
View File

@ -1,4 +1,7 @@
log4j.rootLogger=ERROR, console
log4j.logger.org.apache.http.headers=WARN
log4j.logger.org.apache.http.impl=WARN
log4j.logger.org.apache.http.client=WARN
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=com.metaweb.util.logging.IndentingLayout
log4j.appender.console.layout=com.metaweb.gridworks.logging.IndentingLayout

1
main/webapp/WEB-INF/web.xml
View File

@ -8,7 +8,6 @@
<servlet>
<servlet-name>gridworks</servlet-name>
<servlet-class>com.metaweb.gridworks.GridworksServlet</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>

7
server/IDEs/eclipse/Gridworks.launch
View File

@ -1,12 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
<listEntry value="/gridworks/src/server/java/com/metaweb/gridworks/Gridworks.java"/>
<listEntry value="/gridworks server/src/com/metaweb/gridworks/Gridworks.java"/>
</listAttribute>
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
<listEntry value="1"/>
</listAttribute>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="com.metaweb.gridworks.Gridworks"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="gridworks"/>
<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Dgridworks.autoreloading=true -Dgridworks.webapp=src/main/webapp"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="gridworks server"/>
<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Dgridworks.autoreloading=true"/>
<stringAttribute key="org.eclipse.jdt.launching.WORKING_DIRECTORY" value="${workspace_loc:gridworks}"/>
</launchConfiguration>

143
server/src/com/metaweb/gridworks/Gridworks.java
View File

@ -24,11 +24,14 @@ import org.apache.log4j.Level;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.servlet.ServletHolder;
import org.mortbay.jetty.webapp.WebAppContext;
import org.mortbay.util.Scanner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.codeberry.jdatapath.DataPath;
import com.codeberry.jdatapath.JDataPathSystem;
import com.metaweb.util.threads.ThreadPoolExecutorAdapter;
/**
@ -37,38 +40,15 @@ import com.metaweb.util.threads.ThreadPoolExecutorAdapter;
*/
public class Gridworks {
static private final String VERSION = "1.0";
static private final String DEFAULT_HOST = "127.0.0.1";
static private final int DEFAULT_PORT = 3333;
static private final int MAX_UPLOAD_SIZE = 1024 * 1024 * 1024;
static private File tempDir;
static private int port;
static private String host;
final static Logger logger = LoggerFactory.getLogger("gridworks");
public static String getVersion() {
return VERSION;
}
public static File getTempFile(String name) {
return new File(tempDir, name);
}
public static File getTempDir() {
return tempDir;
}
public static int getMaxUploadSize() {
return Configurations.getInteger("gridworks.max_upload_size",MAX_UPLOAD_SIZE);
}
public static String getFullHost() {
return host + ":" + port;
}
public static void main(String[] args) throws Exception {
public static void main(String[] args) throws Exception {
// tell jetty to use SLF4J for logging instead of its own stuff
System.setProperty("VERBOSE","false");
@ -89,9 +69,6 @@ public class Gridworks {
// set the log verbosity level
org.apache.log4j.Logger.getRootLogger().setLevel(Level.toLevel(Configurations.get("gridworks.verbosity","info")));
tempDir = new File(Configurations.get("gridworks.temp","temp"));
if (!tempDir.exists()) tempDir.mkdirs();
port = Configurations.getInteger("gridworks.port",DEFAULT_PORT);
host = Configurations.get("gridworks.host",DEFAULT_HOST);
@ -117,7 +94,8 @@ public class Gridworks {
// hook up the signal handlers
Runtime.getRuntime().addShutdownHook(
new Thread(new ShutdownSignalHandler(server)));
new Thread(new ShutdownSignalHandler(server))
);
server.join();
}
@ -178,7 +156,15 @@ class GridworksServer extends Server {
scanForUpdates(contextRoot, context);
}
// start the server
this.start();
// inject configuration parameters in the servlets
// NOTE: this is done *after* starting the server because jetty might override the init
// parameters if we set them in the webapp context upon reading the web.xml file
ServletHolder servlet = context.getServletHandler().getServlet("gridworks");
servlet.setInitParameter("gridworks.data", getDataDir());
servlet.doStart();
}
@Override
@ -194,7 +180,7 @@ class GridworksServer extends Server {
}
}
private void scanForUpdates(final File contextRoot, final WebAppContext context) {
static private void scanForUpdates(final File contextRoot, final WebAppContext context) {
List<File> scanList = new ArrayList<File>();
scanList.add(new File(contextRoot, "WEB-INF/web.xml"));
@ -225,7 +211,7 @@ class GridworksServer extends Server {
scanner.start();
}
private void findFiles(final String extension, File baseDir, final Collection<File> found) {
static private void findFiles(final String extension, File baseDir, final Collection<File> found) {
baseDir.listFiles(new FileFilter() {
public boolean accept(File pathname) {
if (pathname.isDirectory()) {
@ -238,6 +224,103 @@ class GridworksServer extends Server {
});
}
static private String getDataDir() {
String data_dir = Configurations.get("gridworks.data_dir");
if (data_dir != null) {
return data_dir;
}
String os = System.getProperty("os.name").toLowerCase();
if (os.contains("windows")) {
try {
// NOTE(SM): finding the "local data app" in windows from java is actually a PITA
// see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
// so we're using a library that uses JNI to ask directly the win32 APIs,
// it's not elegant but it's the safest bet.
DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
File data = new File(fixWindowsUnicodePath(localDataPath.getPath()));
data.mkdirs();
return data.getAbsolutePath();
} catch (Error e) {
/*
* The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll
* we include is compiled for 32-bit. In this case, we just have to dig up
* environment variables and try our best to find a user-specific path.
*/
logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables");
File parentDir = null;
String appData = System.getenv("APPDATA");
if (appData != null && appData.length() > 0) {
// e.g., C:\Users\[userid]\AppData\Roaming
parentDir = new File(appData);
} else {
String userProfile = System.getenv("USERPROFILE");
if (userProfile != null && userProfile.length() > 0) {
// e.g., C:\Users\[userid]
parentDir = new File(userProfile);
}
}
if (parentDir == null) {
parentDir = new File(".");
}
File data = new File(parentDir, "Gridworks");
data.mkdirs();
return data.getAbsolutePath();
}
} else if (os.contains("mac os x")) {
// on macosx, use "~/Library/Application Support"
String home = System.getProperty("user.home");
String data_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks";
File data = new File(data_home);
data.mkdirs();
return data.getAbsolutePath();
} else { // most likely a UNIX flavor
// start with the XDG environment
// see http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
String data_home = System.getenv("XDG_DATA_HOME");
if (data_home == null) { // if not found, default back to ~/.local/share
String home = System.getProperty("user.home");
if (home == null) home = ".";
data_home = home + "/.local/share";
}
File data = new File(data_home + "/gridworks");
data.mkdirs();
return data.getAbsolutePath();
}
}
/**
* For Windows file paths that contain user IDs with non ASCII characters,
* those characters might get replaced with ?. We need to use the environment
* APPDATA value to substitute back the original user ID.
*/
static private String fixWindowsUnicodePath(String path) {
int q = path.indexOf('?');
if (q < 0) {
return path;
}
int pathSep = path.indexOf(File.separatorChar, q);
String goodPath = System.getenv("APPDATA");
if (goodPath == null || goodPath.length() == 0) {
goodPath = System.getenv("USERPROFILE");
if (!goodPath.endsWith(File.separator)) {
goodPath = goodPath + File.separator;
}
}
int goodPathSep = goodPath.indexOf(File.separatorChar, q);
return path.substring(0, q) + goodPath.substring(q, goodPathSep) + path.substring(pathSep);
}
}
/* -------------- Gridworks Client ----------------- */

3
server/src/log4j.properties
View File

@ -1,9 +1,6 @@
log4j.rootLogger=INFO, console
log4j.logger.org.mortbay.log=WARN
log4j.logger.org.mortbay.jetty=ERROR
log4j.logger.org.apache.http.headers=WARN
log4j.logger.org.apache.http.impl=WARN
log4j.logger.org.apache.http.client=WARN
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=com.metaweb.util.logging.IndentingLayout