s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

bringing the refactor branch up to speed with trunk

Browse Source 
(everything works like in trunk for now, although some tests still fail)


git-svn-id: http://google-refine.googlecode.com/svn/branches/split-refactor@915 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Stefano Mazzocchi 2010-05-30 18:18:59 +00:00
parent 081df24a97
commit e3fc7ab603
32 changed files with 1242 additions and 601 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.gitignore vendored
View File

@ -1,4 +1,8 @@
.DS_Store
build/ build/
src/main/webapp/WEB-INF/classes/ server/classes/
tests/java/classes/ main/webapp/WEB-INF/classes/
main/tests/server/classes/
main/test-output/
appengine/classes/
tools/ tools/

2
build.xml
View File

@ -118,6 +118,7 @@
<src path="${webapp.src.dir}"/> <src path="${webapp.src.dir}"/>
<classpath refid="webapp.class.path" /> <classpath refid="webapp.class.path" />
</javac> </javac>
<copy file="${webapp.src.dir}/log4j.properties" tofile="${webapp.classes.dir}/log4j.properties"/>
</target> </target>
<target name="build_tests" depends="build_webapp"> <target name="build_tests" depends="build_webapp">
@ -125,6 +126,7 @@
<javac srcdir="${server.tests.src.dir}" destdir="${server.tests.classes.dir}" debug="true" includeAntRuntime="no"> <javac srcdir="${server.tests.src.dir}" destdir="${server.tests.classes.dir}" debug="true" includeAntRuntime="no">
<classpath refid="tests.class.path" /> <classpath refid="tests.class.path" />
</javac> </javac>
<copy file="${server.tests.src.dir}/log4j.properties" tofile="${server.tests.classes.dir}/log4j.properties"/>
</target> </target>
<target name="build" depends="build_server, build_webapp"/> <target name="build" depends="build_server, build_webapp"/>

3
main/IDEs/eclipse/GridworksTests.launch
View File

@ -6,7 +6,6 @@
</listAttribute> </listAttribute>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.testng.remote.RemoteTestNG"/> <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.testng.remote.RemoteTestNG"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="gridworks"/> <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="gridworks"/>
<stringAttribute key="org.eclipse.jdt.launching.WORKING_DIRECTORY" value="${workspace_loc:gridworks/build}"/>
<mapAttribute key="org.testng.eclipse.ALL_CLASS_METHODS"/> <mapAttribute key="org.testng.eclipse.ALL_CLASS_METHODS"/>
<listAttribute key="org.testng.eclipse.CLASS_TEST_LIST"/> <listAttribute key="org.testng.eclipse.CLASS_TEST_LIST"/>
<stringAttribute key="org.testng.eclipse.COMPLIANCE_LEVEL" value="JDK"/> <stringAttribute key="org.testng.eclipse.COMPLIANCE_LEVEL" value="JDK"/>
@ -15,7 +14,7 @@
<stringAttribute key="org.testng.eclipse.LOG_LEVEL" value="2"/> <stringAttribute key="org.testng.eclipse.LOG_LEVEL" value="2"/>
<listAttribute key="org.testng.eclipse.PACKAGE_TEST_LIST"/> <listAttribute key="org.testng.eclipse.PACKAGE_TEST_LIST"/>
<listAttribute key="org.testng.eclipse.SUITE_TEST_LIST"> <listAttribute key="org.testng.eclipse.SUITE_TEST_LIST">
<listEntry value="tests/java/conf/tests.xml"/> <listEntry value="tests/server/conf/tests.xml"/>
</listAttribute> </listAttribute>
<intAttribute key="org.testng.eclipse.TYPE" value="3"/> <intAttribute key="org.testng.eclipse.TYPE" value="3"/>
</launchConfiguration> </launchConfiguration>

57
main/src/com/metaweb/gridworks/GridworksServlet.java
View File

@ -1,11 +1,13 @@
package com.metaweb.gridworks; package com.metaweb.gridworks;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Timer; import java.util.Timer;
import java.util.TimerTask; import java.util.TimerTask;
import javax.servlet.ServletConfig;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
@ -18,8 +20,12 @@ import com.metaweb.gridworks.commands.Command;
public class GridworksServlet extends HttpServlet { public class GridworksServlet extends HttpServlet {
static private final String VERSION = "1.0";
private static final long serialVersionUID = 2386057901503517403L; private static final long serialVersionUID = 2386057901503517403L;
private static final String JAVAX_SERVLET_CONTEXT_TEMPDIR = "javax.servlet.context.tempdir";
static final private Map<String, Command> commands = new HashMap<String, Command>(); static final private Map<String, Command> commands = new HashMap<String, Command>();
// timer for periodically saving projects // timer for periodically saving projects
@ -104,11 +110,12 @@ public class GridworksServlet extends HttpServlet {
{"mqlwrite", "com.metaweb.gridworks.commands.freebase.MQLWriteCommand"}, {"mqlwrite", "com.metaweb.gridworks.commands.freebase.MQLWriteCommand"},
}; };
static { public static String getVersion() {
registerCommands(commandNames); return VERSION;
} }
final static protected long s_autoSavePeriod = 1000 * 60 * 5; // 5 minutes final static protected long s_autoSavePeriod = 1000 * 60 * 5; // 5 minutes
static protected class AutoSaveTimerTask extends TimerTask { static protected class AutoSaveTimerTask extends TimerTask {
public void run() { public void run() {
try { try {
@ -121,12 +128,21 @@ public class GridworksServlet extends HttpServlet {
} }
} }
protected ServletConfig config;
@Override @Override
public void init() throws ServletException { public void init() throws ServletException {
super.init();
logger.trace("> initialize"); logger.trace("> initialize");
ProjectManager.initialize(); String data = getInitParameter("gridworks.data");
if (data == null) {
throw new ServletException("can't find servlet init config 'gridworks.data', I have to give up initializing");
}
registerCommands(commandNames);
ProjectManager.initialize(new File(data));
if (_timer == null) { if (_timer == null) {
_timer = new Timer("autosave"); _timer = new Timer("autosave");
@ -149,6 +165,8 @@ public class GridworksServlet extends HttpServlet {
ProjectManager.singleton.save(true); // complete save ProjectManager.singleton.save(true); // complete save
ProjectManager.singleton = null; ProjectManager.singleton = null;
} }
this.config = null;
super.destroy(); super.destroy();
@ -188,6 +206,26 @@ public class GridworksServlet extends HttpServlet {
return slash > 0 ? commandName.substring(0, slash) : commandName; return slash > 0 ? commandName.substring(0, slash) : commandName;
} }
private File tempDir = null;
public File getTempDir() {
if (tempDir == null) {
File tempDir = (File) this.config.getServletContext().getAttribute(JAVAX_SERVLET_CONTEXT_TEMPDIR);
if (tempDir == null) {
throw new RuntimeException("This app server doesn't support temp directories");
}
}
return tempDir;
}
public File getTempFile(String name) {
return new File(getTempDir(), name);
}
public String getConfiguration(String name, String def) {
return null;
}
/** /**
* Register an array of commands * Register an array of commands
* *
@ -198,7 +236,7 @@ public class GridworksServlet extends HttpServlet {
* the second. * the second.
* @return false if any commands failed to load * @return false if any commands failed to load
*/ */
static public boolean registerCommands(String[][] commands) { private boolean registerCommands(String[][] commands) {
boolean status = true; boolean status = true;
for (String[] command : commandNames) { for (String[] command : commandNames) {
String commandName = command[0]; String commandName = command[0];
@ -206,8 +244,8 @@ public class GridworksServlet extends HttpServlet {
logger.debug("Loading command " + commandName + " class: " + className); logger.debug("Loading command " + commandName + " class: " + className);
Command cmd; Command cmd;
try { try {
// TODO: May need to use the servlet container's class loader here cmd = (Command) this.getClass().getClassLoader().loadClass(className).newInstance();
cmd = (Command) Class.forName(className).newInstance(); cmd.init(this);
} catch (InstantiationException e) { } catch (InstantiationException e) {
logger.error("Failed to load command class " + className, e); logger.error("Failed to load command class " + className, e);
status = false; status = false;
@ -235,8 +273,7 @@ public class GridworksServlet extends HttpServlet {
* object implementing the command * object implementing the command
* @return true if command was loaded and registered successfully * @return true if command was loaded and registered successfully
*/ */
static public boolean registerCommand(String name, protected boolean registerCommand(String name, Command commandObject) {
Command commandObject) {
if (commands.containsKey(name)) { if (commands.containsKey(name)) {
return false; return false;
} }
@ -245,7 +282,7 @@ public class GridworksServlet extends HttpServlet {
} }
// Currently only for test purposes // Currently only for test purposes
static protected boolean unregisterCommand(String verb) { protected boolean unregisterCommand(String verb) {
return commands.remove(verb) != null; return commands.remove(verb) != null;
} }
} }

103
main/src/com/metaweb/gridworks/ProjectManager.java
View File

@ -22,8 +22,6 @@ import org.json.JSONWriter;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.codeberry.jdatapath.DataPath;
import com.codeberry.jdatapath.JDataPathSystem;
import com.metaweb.gridworks.model.Project; import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.util.JSONUtilities; import com.metaweb.gridworks.util.JSONUtilities;
@ -58,112 +56,13 @@ public class ProjectManager {
static public ProjectManager singleton; static public ProjectManager singleton;
static public synchronized void initialize() { static public synchronized void initialize(File dir) {
if (singleton == null) { if (singleton == null) {
File dir = getProjectLocation();
logger.info("Using workspace directory: {}", dir.getAbsolutePath()); logger.info("Using workspace directory: {}", dir.getAbsolutePath());
singleton = new ProjectManager(dir); singleton = new ProjectManager(dir);
} }
} }
static protected File getProjectLocation() {
String data_dir = Configurations.get("gridworks.data_dir");
if (data_dir != null) {
return new File(data_dir);
}
String os = Configurations.get("os.name").toLowerCase();
if (os.contains("windows")) {
try {
// NOTE(SM): finding the "local data app" in windows from java is actually a PITA
// see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
// so we're using a library that uses JNI to ask directly the win32 APIs,
// it's not elegant but it's the safest bet.
DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
File data = new File(fixWindowsUnicodePath(localDataPath.getPath()));
data.mkdirs();
return data;
} catch (Error e) {
/*
* The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll
* we include is compiled for 32-bit. In this case, we just have to dig up
* environment variables and try our best to find a user-specific path.
*/
logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables");
File parentDir = null;
{
String appData = System.getenv("APPDATA");
if (appData != null && appData.length() > 0) {
// e.g., C:\Users\[userid]\AppData\Roaming
parentDir = new File(appData);
} else {
String userProfile = System.getenv("USERPROFILE");
if (userProfile != null && userProfile.length() > 0) {
// e.g., C:\Users\[userid]
parentDir = new File(userProfile);
}
}
}
if (parentDir == null) {
parentDir = new File(".");
}
File data = new File(parentDir, "Gridworks");
data.mkdirs();
return data;
}
} else if (os.contains("mac os x")) {
// on macosx, use "~/Library/Application Support"
String home = System.getProperty("user.home");
String data_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks";
File data = new File(data_home);
data.mkdirs();
return data;
} else { // most likely a UNIX flavor
// start with the XDG environment
// see http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
String data_home = System.getenv("XDG_DATA_HOME");
if (data_home == null) { // if not found, default back to ~/.local/share
String home = System.getProperty("user.home");
if (home == null) home = ".";
data_home = home + "/.local/share";
}
File data = new File(data_home + "/gridworks");
data.mkdirs();
return data;
}
}
/**
* For Windows file paths that contain user IDs with non ASCII characters,
* those characters might get replaced with ?. We need to use the environment
* APPDATA value to substitute back the original user ID.
*/
static protected String fixWindowsUnicodePath(String path) {
int q = path.indexOf('?');
if (q < 0) {
return path;
}
int pathSep = path.indexOf(File.separatorChar, q);
String goodPath = System.getenv("APPDATA");
if (goodPath == null || goodPath.length() == 0) {
goodPath = System.getenv("USERPROFILE");
if (!goodPath.endsWith(File.separator)) {
goodPath = goodPath + File.separator;
}
}
int goodPathSep = goodPath.indexOf(File.separatorChar, q);
return path.substring(0, q) + goodPath.substring(q, goodPathSep) + path.substring(pathSep);
}
private ProjectManager(File dir) { private ProjectManager(File dir) {
_workspaceDir = dir; _workspaceDir = dir;
_workspaceDir.mkdirs(); _workspaceDir.mkdirs();

7
main/src/com/metaweb/gridworks/commands/Command.java
View File

@ -16,6 +16,7 @@ import org.json.JSONWriter;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.Jsonizable; import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.ProjectManager; import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata; import com.metaweb.gridworks.ProjectMetadata;
@ -33,6 +34,12 @@ public abstract class Command {
final static protected Logger logger = LoggerFactory.getLogger("command"); final static protected Logger logger = LoggerFactory.getLogger("command");
protected GridworksServlet servlet;
public void init(GridworksServlet servlet) {
this.servlet = servlet;
}
public void doPost(HttpServletRequest request, HttpServletResponse response) public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException { throws ServletException, IOException {

21
main/src/com/metaweb/gridworks/commands/auth/AuthorizeCommand.java
View File

@ -2,6 +2,8 @@ package com.metaweb.gridworks.commands.auth;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.net.URI;
import java.net.URISyntaxException;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
@ -10,13 +12,12 @@ import javax.servlet.http.HttpServletResponse;
import oauth.signpost.OAuthConsumer; import oauth.signpost.OAuthConsumer;
import oauth.signpost.OAuthProvider; import oauth.signpost.OAuthProvider;
import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.commands.Command; import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.oauth.Credentials; import com.metaweb.gridworks.oauth.Credentials;
import com.metaweb.gridworks.oauth.OAuthUtilities; import com.metaweb.gridworks.oauth.OAuthUtilities;
import com.metaweb.gridworks.oauth.Provider; import com.metaweb.gridworks.oauth.Provider;
public class AuthorizeCommand extends Command { public class AuthorizeCommand extends Command {
private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier"; private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier";
@ -124,7 +125,21 @@ public class AuthorizeCommand extends Command {
private String getBaseURL(HttpServletRequest request, Provider provider) { private String getBaseURL(HttpServletRequest request, Provider provider) {
String host = request.getHeader("host"); String host = request.getHeader("host");
if (host == null) host = Gridworks.getFullHost(); if (host == null) {
String referrer = request.getHeader("referer");
if (referrer != null) {
URI url;
try {
url = new URI(referrer);
int port = url.getPort();
host = url.getHost() + ((port > -1) ? ":" + url.getPort() : "");
} catch (URISyntaxException e) {
throw new RuntimeException("referrer '" + referrer + "' can't be parsed as a URL");
}
} else {
throw new RuntimeException("neither the 'host' nor 'referer' headers were present in the HTTP response, I can't determine what URL gridworks is listening to.");
}
}
return "http://" + host + "/command/authorize/" + provider.getHost(); return "http://" + host + "/command/authorize/" + provider.getHost();
} }
} }

3
main/src/com/metaweb/gridworks/commands/project/CreateProjectCommand.java
View File

@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory;
import com.ibm.icu.text.CharsetDetector; import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch; import com.ibm.icu.text.CharsetMatch;
import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.ProjectManager; import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata; import com.metaweb.gridworks.ProjectMetadata;
import com.metaweb.gridworks.commands.Command; import com.metaweb.gridworks.commands.Command;
@ -367,7 +366,7 @@ public class CreateProjectCommand extends Command {
} }
private File save(InputStream is) throws IOException { private File save(InputStream is) throws IOException {
File temp = Gridworks.getTempFile(Long.toString(System.currentTimeMillis())); File temp = this.servlet.getTempFile(Long.toString(System.currentTimeMillis()));
temp.deleteOnExit(); temp.deleteOnExit();
IOUtils.copy(is,temp); IOUtils.copy(is,temp);
is.close(); is.close();

116
main/src/com/metaweb/gridworks/exporters/CsvExporter.java
View File

@ -40,66 +40,66 @@ public class CsvExporter implements Exporter{
@Override @Override
public void export(Project project, Properties options, Engine engine, Writer writer) throws IOException { public void export(Project project, Properties options, Engine engine, Writer writer) throws IOException {
{ boolean printColumnHeader = true;
boolean printColumnHeader = true;
if(options != null)
printColumnHeader = options.getProperty("printColumnHeader")=="false"?false:true;
RowVisitor visitor = new RowVisitor() { if (options != null) {
CSVWriter csvWriter; printColumnHeader = Boolean.parseBoolean(options.getProperty("printColumnHeader"));
boolean printColumnHeader = true;
boolean isFirstRow = true; //the first row should also add the column headers
public RowVisitor init(CSVWriter writer, boolean printColumnHeader){
this.csvWriter = writer;
this.printColumnHeader = printColumnHeader;
return this;
}
public boolean visit(Project project, int rowIndex, Row row) {
String[] cols = new String[project.columnModel.columns.size()];
String[] vals = new String[row.cells.size()];
int i = 0;
for(Column col : project.columnModel.columns){
int cellIndex = col.getCellIndex();
cols[i] = col.getName();
Cell cell = row.cells.get(cellIndex);
if(cell != null){
vals[i] = cell.value.toString();
}
i++;
}
if( printColumnHeader && isFirstRow ){
csvWriter.writeNext(cols,false);
isFirstRow = false; //switch off flag
}
csvWriter.writeNext(vals,false);
return false;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
try {
csvWriter.close();
} catch (IOException e) {
logger.error("CsvExporter could not close writer : " + e.getMessage());
}
}
}.init(new CSVWriter(writer, separator), printColumnHeader);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, visitor);
} }
RowVisitor visitor = new RowVisitor() {
CSVWriter csvWriter;
boolean printColumnHeader = true;
boolean isFirstRow = true; //the first row should also add the column headers
public RowVisitor init(CSVWriter writer, boolean printColumnHeader){
this.csvWriter = writer;
this.printColumnHeader = printColumnHeader;
return this;
}
public boolean visit(Project project, int rowIndex, Row row) {
String[] cols = new String[project.columnModel.columns.size()];
String[] vals = new String[row.cells.size()];
int i = 0;
for(Column col : project.columnModel.columns){
int cellIndex = col.getCellIndex();
cols[i] = col.getName();
Cell cell = row.cells.get(cellIndex);
if(cell != null){
vals[i] = cell.value.toString();
}
i++;
}
if( printColumnHeader && isFirstRow ){
csvWriter.writeNext(cols,false);
isFirstRow = false; //switch off flag
}
csvWriter.writeNext(vals,false);
return false;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
try {
csvWriter.close();
} catch (IOException e) {
logger.error("CsvExporter could not close writer : " + e.getMessage());
}
}
}.init(new CSVWriter(writer, separator), printColumnHeader);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, visitor);
} }
@Override @Override

2
main/src/com/metaweb/gridworks/expr/JythonEvaluable.java
View File

@ -19,7 +19,7 @@ public class JythonEvaluable implements Evaluable {
private static PythonInterpreter _engine; private static PythonInterpreter _engine;
static { static {
File libPath = new File("lib/jython"); File libPath = new File("webapp/WEB-INF/lib/jython");
if (libPath.exists()) { if (libPath.exists()) {
Properties props = new Properties(); Properties props = new Properties();
props.setProperty("python.path", libPath.getAbsolutePath()); props.setProperty("python.path", libPath.getAbsolutePath());

4
main/src/com/metaweb/gridworks/history/History.java
View File

@ -16,7 +16,7 @@ import java.util.Properties;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONWriter; import org.json.JSONWriter;
import com.metaweb.gridworks.Gridworks; import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.Jsonizable; import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.ProjectManager; import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.model.Project; import com.metaweb.gridworks.model.Project;
@ -68,7 +68,7 @@ public class History implements Jsonizable {
} }
static public void writeOneChange(Writer writer, Change change, Properties options) throws IOException { static public void writeOneChange(Writer writer, Change change, Properties options) throws IOException {
writer.write(Gridworks.getVersion()); writer.write('\n'); writer.write(GridworksServlet.getVersion()); writer.write('\n');
writer.write(change.getClass().getName()); writer.write('\n'); writer.write(change.getClass().getName()); writer.write('\n');
change.save(writer, options); change.save(writer, options);

16
main/src/com/metaweb/gridworks/importers/XmlImportUtilities.java
View File

@ -72,8 +72,6 @@ public class XmlImportUtilities {
} }
static public String[] detectPathFromTag(InputStream inputStream, String tag) { static public String[] detectPathFromTag(InputStream inputStream, String tag) {
//List<RecordElementCandidate> candidates = new ArrayList<RecordElementCandidate>();
try { try {
XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(inputStream); XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(inputStream);
@ -99,6 +97,8 @@ public class XmlImportUtilities {
} }
static protected List<String> detectRecordElement(XMLStreamReader parser, String tag) throws XMLStreamException { static protected List<String> detectRecordElement(XMLStreamReader parser, String tag) throws XMLStreamException {
if(parser.getEventType() == XMLStreamConstants.START_DOCUMENT)
parser.next();
String localName = parser.getLocalName(); String localName = parser.getLocalName();
String fullName = composeName(parser.getPrefix(), localName); String fullName = composeName(parser.getPrefix(), localName);
if (tag.equals(parser.getLocalName()) || tag.equals(fullName)) { if (tag.equals(parser.getLocalName()) || tag.equals(fullName)) {
@ -327,6 +327,10 @@ public class XmlImportUtilities {
int pathIndex, int pathIndex,
ImportColumnGroup rootColumnGroup ImportColumnGroup rootColumnGroup
) throws XMLStreamException { ) throws XMLStreamException {
if(parser.getEventType() == XMLStreamConstants.START_DOCUMENT){
logger.warn("Cannot use findRecord method for START_DOCUMENT event");
return;
}
String tagName = parser.getLocalName(); String tagName = parser.getLocalName();
if (tagName.equals(recordPath[pathIndex])) { if (tagName.equals(recordPath[pathIndex])) {
if (pathIndex < recordPath.length - 1) { if (pathIndex < recordPath.length - 1) {
@ -466,7 +470,7 @@ public class XmlImportUtilities {
ImportRecord record, ImportRecord record,
String columnLocalName, String columnLocalName,
String text, String text,
int commonStaringRowIndex int commonStartingRowIndex
) { ) {
if (text == null || ((String) text).isEmpty()) { if (text == null || ((String) text).isEmpty()) {
return; return;
@ -478,7 +482,7 @@ public class XmlImportUtilities {
int cellIndex = column.cellIndex; int cellIndex = column.cellIndex;
while (cellIndex >= record.columnEmptyRowIndices.size()) { while (cellIndex >= record.columnEmptyRowIndices.size()) {
record.columnEmptyRowIndices.add(commonStaringRowIndex); record.columnEmptyRowIndices.add(commonStartingRowIndex);
} }
int rowIndex = record.columnEmptyRowIndices.get(cellIndex); int rowIndex = record.columnEmptyRowIndices.get(cellIndex);
@ -491,7 +495,9 @@ public class XmlImportUtilities {
row.add(null); row.add(null);
} }
row.set(cellIndex, new Cell(value, null)); logger.trace("Adding cell with value : " + value + " to row : " + rowIndex + " at cell index : " + (cellIndex-1));
row.set(cellIndex-1, new Cell(value, null));
record.columnEmptyRowIndices.set(cellIndex, rowIndex + 1); record.columnEmptyRowIndices.set(cellIndex, rowIndex + 1);

3
main/src/com/metaweb/gridworks/importers/XmlImporter.java
View File

@ -57,6 +57,9 @@ public class XmlImporter implements Importer {
} }
} }
if(recordPath == null)
return;
ImportColumnGroup rootColumnGroup = new ImportColumnGroup(); ImportColumnGroup rootColumnGroup = new ImportColumnGroup();
XmlImportUtilities.importXml(pis, project, recordPath, rootColumnGroup); XmlImportUtilities.importXml(pis, project, recordPath, rootColumnGroup);

143
main/src/com/metaweb/gridworks/logging/IndentingLayout.java Normal file
View File

@ -0,0 +1,143 @@
package com.metaweb.gridworks.logging;
/*
* Copyright (c) Massachusetts Institute of Technology, 2007
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* Original code: http://simile.mit.edu/repository/tracer/trunk/
*/
import java.util.Calendar;
import java.util.Date;
import org.apache.log4j.Layout;
import org.apache.log4j.spi.LoggingEvent;
/**
* This is a special Log4j log formatter that is capable of reacting on special log messages
* and 'indent' the logs accordingly. This is very useful to visually inspect a debug log
* and see what calls what. An example of logs are "> method()" and "< method()" where > and <
* are used to indicate respectively "entering" and "exiting".
*/
public class IndentingLayout extends Layout {
protected static final int CONTEXT_SIZE = 25;
protected static final long MAX_DELTA = 10000;
protected Calendar calendar = Calendar.getInstance();
protected long previousTime = 0;
protected int indentation = 0;
public void activateOptions() {
// no options at this time
}
public String format(LoggingEvent event) {
String message = event.getRenderedMessage();
if (message == null) return "";
if (message.length() < 2) return message;
char leader = message.charAt(0);
char secondLeader = message.charAt(1);
if ((leader == '<') && (secondLeader == ' ') && (this.indentation > 0)) this.indentation--;
// Reset buf
StringBuffer buf = new StringBuffer(256);
Date date = new Date();
long now = date.getTime();
calendar.setTime(date);
long delta = 0;
if (previousTime > 0) {
delta = now - previousTime;
}
previousTime = now;
// if ((previousTime == 0) || (delta > MAX_DELTA)) {
// buf.append('\n');
// indentation = 0; // reset indentation after a while, as we might
// // have runaway/unmatched log entries
// }
int hour = calendar.get(Calendar.HOUR_OF_DAY);
if (hour < 10) buf.append('0');
buf.append(hour);
buf.append(':');
int mins = calendar.get(Calendar.MINUTE);
if (mins < 10) buf.append('0');
buf.append(mins);
buf.append(':');
int secs = calendar.get(Calendar.SECOND);
if (secs < 10) buf.append('0');
buf.append(secs);
buf.append('.');
int millis = (int) (now % 1000);
if (millis < 100) buf.append('0');
if (millis < 10) buf.append('0');
buf.append(millis);
buf.append(" [");
String context = ((String) event.getMDC("LogEvent"));
if (context == null) {
context = event.getLoggerName();
}
if (context.length() < CONTEXT_SIZE) {
pad(buf, CONTEXT_SIZE - context.length(), ' ');
buf.append(context);
} else {
buf.append("..");
buf.append(context.substring(context.length() - CONTEXT_SIZE + 2));
}
buf.append("] ");
pad(buf, indentation, ' ');
buf.append(message);
buf.append(" (");
buf.append(delta);
buf.append("ms)\n");
if ((leader == '>') && (secondLeader == ' ')) indentation++;
return buf.toString();
}
private void pad(StringBuffer buffer, int pads, char padchar) {
for (int i = 0; i < pads; i++) {
buffer.append(padchar);
}
}
public boolean ignoresThrowable() {
return true;
}
}

4
main/src/com/metaweb/gridworks/model/Project.java
View File

@ -19,7 +19,7 @@ import java.util.zip.ZipOutputStream;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.metaweb.gridworks.Gridworks; import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.ProjectManager; import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata; import com.metaweb.gridworks.ProjectMetadata;
import com.metaweb.gridworks.history.History; import com.metaweb.gridworks.history.History;
@ -128,7 +128,7 @@ public class Project {
} }
protected void saveToWriter(Writer writer, Properties options) throws IOException { protected void saveToWriter(Writer writer, Properties options) throws IOException {
writer.write(Gridworks.getVersion()); writer.write('\n'); writer.write(GridworksServlet.getVersion()); writer.write('\n');
writer.write("columnModel=\n"); columnModel.save(writer, options); writer.write("columnModel=\n"); columnModel.save(writer, options);
writer.write("history=\n"); history.save(writer, options); writer.write("history=\n"); history.save(writer, options);

10
main/src/com/metaweb/gridworks/util/FreebaseUtils.java
View File

@ -27,7 +27,7 @@ import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import com.metaweb.gridworks.Gridworks; import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.oauth.Credentials; import com.metaweb.gridworks.oauth.Credentials;
import com.metaweb.gridworks.oauth.OAuthUtilities; import com.metaweb.gridworks.oauth.OAuthUtilities;
import com.metaweb.gridworks.oauth.Provider; import com.metaweb.gridworks.oauth.Provider;
@ -59,7 +59,7 @@ public class FreebaseUtils {
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost())); HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion()); httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
// this is required by the Metaweb API to avoid XSS // this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1"); httpRequest.setHeader("X-Requested-With", "1");
@ -102,7 +102,7 @@ public class FreebaseUtils {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost())); HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion()); httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
httpRequest.setEntity(entity); httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS // this is required by the Metaweb API to avoid XSS
@ -128,7 +128,7 @@ public class FreebaseUtils {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost())); HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion()); httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
httpRequest.setEntity(entity); httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS // this is required by the Metaweb API to avoid XSS
@ -176,7 +176,7 @@ public class FreebaseUtils {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(FREEQ_URL); HttpPost httpRequest = new HttpPost(FREEQ_URL);
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion()); httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
httpRequest.setEntity(entity); httpRequest.setEntity(entity);
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST)); HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));

4
main/src/com/metaweb/gridworks/util/Pool.java
View File

@ -15,7 +15,7 @@ import java.util.Map.Entry;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONWriter; import org.json.JSONWriter;
import com.metaweb.gridworks.Gridworks; import com.metaweb.gridworks.GridworksServlet;
import com.metaweb.gridworks.Jsonizable; import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.model.Recon; import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.ReconCandidate; import com.metaweb.gridworks.model.ReconCandidate;
@ -59,7 +59,7 @@ public class Pool implements Jsonizable {
} }
public void save(Writer writer) throws IOException { public void save(Writer writer) throws IOException {
writer.write(Gridworks.getVersion()); writer.write('\n'); writer.write(GridworksServlet.getVersion()); writer.write('\n');
Properties options = new Properties(); Properties options = new Properties();
options.setProperty("mode", "save"); options.setProperty("mode", "save");

7
main/src/log4j.properties Normal file
View File

@ -0,0 +1,7 @@
log4j.rootLogger=INFO, console
log4j.logger.org.apache.http.headers=WARN
log4j.logger.org.apache.http.impl=WARN
log4j.logger.org.apache.http.client=WARN
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=com.metaweb.gridworks.logging.IndentingLayout

0
main/tests/client/js/facets.js → main/tests/client/src/facets.js
View File

0
main/tests/client/js/initialize.js → main/tests/client/src/initialize.js
View File

4
main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletStub.java
View File

@ -36,7 +36,7 @@ public class GridworksServletStub extends GridworksServlet {
* @param commandName * @param commandName
* @param command * @param command
*/ */
static public void InsertCommand( String commandName, Command command ){ public void insertCommand( String commandName, Command command ){
registerCommand(commandName, command); registerCommand(commandName, command);
} }
@ -44,7 +44,7 @@ public class GridworksServletStub extends GridworksServlet {
* Helper method for clearing up after testing * Helper method for clearing up after testing
* @param commandName * @param commandName
*/ */
static public void RemoveCommand( String commandName ){ public void removeCommand( String commandName ){
unregisterCommand(commandName); unregisterCommand(commandName);
} }
} }

7
main/tests/server/src/com/metaweb/gridworks/tests/GridworksServletTests.java
View File

@ -40,21 +40,22 @@ public class GridworksServletTests {
@BeforeMethod @BeforeMethod
public void SetUp() { public void SetUp() {
SUT = new GridworksServletStub();
request = mock(HttpServletRequest.class); request = mock(HttpServletRequest.class);
response = mock(HttpServletResponse.class); response = mock(HttpServletResponse.class);
command = mock(Command.class); command = mock(Command.class);
GridworksServletStub.InsertCommand(TEST_COMMAND_NAME,command); //inject mock into command container SUT = new GridworksServletStub();
SUT.insertCommand(TEST_COMMAND_NAME,command); //inject mock into command container
} }
@AfterMethod @AfterMethod
public void TearDown() { public void TearDown() {
SUT.removeCommand(TEST_COMMAND_NAME); //remove mock to clean command container
SUT = null; SUT = null;
request = null; request = null;
response = null; response = null;
command = null; command = null;
GridworksServletStub.RemoveCommand(TEST_COMMAND_NAME); //remove mock to clean command container
} }
//-------------------AutoSaveTimerTask tests----------- //-------------------AutoSaveTimerTask tests-----------

306
main/tests/server/src/com/metaweb/gridworks/tests/importers/RdfTripleImporterTests.java
View File

@ -1,153 +1,153 @@
package com.metaweb.gridworks.tests.importers; package com.metaweb.gridworks.tests.importers;
import java.io.StringReader; import java.io.StringReader;
import java.util.Properties; import java.util.Properties;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.metaweb.gridworks.importers.RdfTripleImporter; import com.metaweb.gridworks.importers.RdfTripleImporter;
import com.metaweb.gridworks.model.Project; import com.metaweb.gridworks.model.Project;
public class RdfTripleImporterTests { public class RdfTripleImporterTests {
// logging // logging
final static protected Logger logger = LoggerFactory.getLogger("RdfImporterTests"); final static protected Logger logger = LoggerFactory.getLogger("RdfImporterTests");
//System Under Test //System Under Test
RdfTripleImporter SUT = null; RdfTripleImporter SUT = null;
Project project = null; Project project = null;
Properties options = null; Properties options = null;
@BeforeMethod @BeforeMethod
public void SetUp(){ public void SetUp(){
SUT = new RdfTripleImporter(); SUT = new RdfTripleImporter();
project = new Project(); project = new Project();
options = new Properties(); options = new Properties();
options.put("base-url", "http://rdf.freebase.com"); options.put("base-url", "http://rdf.freebase.com");
} }
@Test(enabled=false) @Test(enabled=false)
public void CanParseSingleLineTriple(){ public void CanParseSingleLineTriple(){
String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.blood_on_the_tracks>."; String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.blood_on_the_tracks>.";
StringReader reader = new StringReader(sampleRdf); StringReader reader = new StringReader(sampleRdf);
try { try {
SUT.read(reader, project, options); SUT.read(reader, project, options);
project.update(); project.update();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
Assert.assertEquals(project.columnModel.columns.size(), 2); Assert.assertEquals(project.columnModel.columns.size(), 2);
Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album"); Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album");
Assert.assertEquals(project.rows.size(), 1); Assert.assertEquals(project.rows.size(), 1);
Assert.assertEquals(project.rows.get(0).cells.size(), 2); Assert.assertEquals(project.rows.get(0).cells.size(), 2);
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks"); Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks");
} }
@Test @Test
public void CanParseMultiLineTriple(){ public void CanParseMultiLineTriple(){
String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.blood_on_the_tracks>.\n" + String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.blood_on_the_tracks>.\n" +
"<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.under_the_red_sky>.\n" + "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.under_the_red_sky>.\n" +
"<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.bringing_it_all_back_home>."; "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.bringing_it_all_back_home>.";
StringReader reader = new StringReader(sampleRdf); StringReader reader = new StringReader(sampleRdf);
try { try {
SUT.read(reader, project, options); SUT.read(reader, project, options);
project.update(); project.update();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
//columns //columns
Assert.assertEquals(project.columnModel.columns.size(), 2); Assert.assertEquals(project.columnModel.columns.size(), 2);
Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album"); Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album");
//rows //rows
Assert.assertEquals(project.rows.size(), 3); Assert.assertEquals(project.rows.size(), 3);
//row0 //row0
Assert.assertEquals(project.rows.get(0).cells.size(), 2); Assert.assertEquals(project.rows.get(0).cells.size(), 2);
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks"); Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks");
//row1 //row1
Assert.assertEquals(project.rows.get(1).cells.size(), 2); Assert.assertEquals(project.rows.get(1).cells.size(), 2);
Assert.assertNull(project.rows.get(1).cells.get(0)); Assert.assertNull(project.rows.get(1).cells.get(0));
Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); //NB triples aren't created in order they were input Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); //NB triples aren't created in order they were input
Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0); Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0);
Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0); Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0);
//row2 //row2
Assert.assertEquals(project.rows.get(2).cells.size(), 2); Assert.assertEquals(project.rows.get(2).cells.size(), 2);
Assert.assertNull(project.rows.get(2).cells.get(0)); Assert.assertNull(project.rows.get(2).cells.get(0));
Assert.assertEquals(project.rows.get(2).cells.get(1).value, "http://rdf.freebase.com/ns/en.under_the_red_sky"); //NB triples aren't created in order they were input Assert.assertEquals(project.rows.get(2).cells.get(1).value, "http://rdf.freebase.com/ns/en.under_the_red_sky"); //NB triples aren't created in order they were input
Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].rowIndex, 0); Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].rowIndex, 0);
Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].cellIndex, 0); Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].cellIndex, 0);
} }
@Test @Test
public void CanParseMultiLineMultiPredicatesTriple(){ public void CanParseMultiLineMultiPredicatesTriple(){
String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.blood_on_the_tracks>.\n" + String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.blood_on_the_tracks>.\n" +
"<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.genre> <http://rdf.freebase.com/ns/en.folk_rock>.\n" + "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.genre> <http://rdf.freebase.com/ns/en.folk_rock>.\n" +
"<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.bringing_it_all_back_home>."; "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.bringing_it_all_back_home>.";
StringReader reader = new StringReader(sampleRdf); StringReader reader = new StringReader(sampleRdf);
try { try {
SUT.read(reader, project, options); SUT.read(reader, project, options);
project.update(); project.update();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
//columns //columns
Assert.assertEquals(project.columnModel.columns.size(), 3); Assert.assertEquals(project.columnModel.columns.size(), 3);
Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album"); Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/music.artist.album");
Assert.assertEquals(project.columnModel.columns.get(2).getName(), "http://rdf.freebase.com/ns/music.artist.genre"); Assert.assertEquals(project.columnModel.columns.get(2).getName(), "http://rdf.freebase.com/ns/music.artist.genre");
//rows //rows
Assert.assertEquals(project.rows.size(), 2); Assert.assertEquals(project.rows.size(), 2);
//row0 //row0
Assert.assertEquals(project.rows.get(0).cells.size(), 3); Assert.assertEquals(project.rows.get(0).cells.size(), 3);
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks"); Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks");
Assert.assertEquals(project.rows.get(0).cells.get(2).value, "http://rdf.freebase.com/ns/en.folk_rock"); Assert.assertEquals(project.rows.get(0).cells.get(2).value, "http://rdf.freebase.com/ns/en.folk_rock");
//row1 //row1
Assert.assertEquals(project.rows.get(1).cells.size(), 2); Assert.assertEquals(project.rows.get(1).cells.size(), 2);
Assert.assertNull(project.rows.get(1).cells.get(0)); Assert.assertNull(project.rows.get(1).cells.get(0));
Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home");
Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0); Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0);
Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0); Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0);
} }
@Test @Test
public void CanParseTripleWithValue(){ public void CanParseTripleWithValue(){
String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/common.topic.alias> \"Robert Zimmerman\"@en."; String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/common.topic.alias> \"Robert Zimmerman\"@en.";
StringReader reader = new StringReader(sampleRdf); StringReader reader = new StringReader(sampleRdf);
try { try {
SUT.read(reader, project, options); SUT.read(reader, project, options);
project.update(); project.update();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
Assert.assertEquals(project.columnModel.columns.size(), 2); Assert.assertEquals(project.columnModel.columns.size(), 2);
Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/common.topic.alias"); Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/common.topic.alias");
Assert.assertEquals(project.rows.size(), 1); Assert.assertEquals(project.rows.size(), 1);
Assert.assertEquals(project.rows.get(0).cells.size(), 2); Assert.assertEquals(project.rows.get(0).cells.size(), 2);
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan"); Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, "\"Robert Zimmerman\"@en"); Assert.assertEquals(project.rows.get(0).cells.get(1).value, "\"Robert Zimmerman\"@en");
} }
} }

50
main/tests/server/src/com/metaweb/gridworks/tests/importers/TestTools.java Normal file
View File

@ -0,0 +1,50 @@
package com.metaweb.gridworks.tests.importers;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
/**
* Helper methods for Importer testing
*
*/
public class TestTools {
final static Logger logger = LoggerFactory.getLogger("Common");
public static void AssertGridCreated(Project project, int numCols, int numRows){
Assert.assertNotNull(project);
Assert.assertNotNull(project.columnModel);
Assert.assertNotNull(project.columnModel.columns);
Assert.assertEquals(project.columnModel.columns.size(), numCols);
Assert.assertNotNull(project.rows);
Assert.assertEquals(project.rows.size(), numRows);
}
public static void PrintProject(Project project){
//some quick and dirty debugging
StringBuilder sb = new StringBuilder();
for(Column c : project.columnModel.columns){
sb.append(c.getName());
sb.append("; ");
}
logger.info(sb.toString());
for(Row r : project.rows){
sb = new StringBuilder();
for(int i = 0; i < r.cells.size(); i++){
Cell c = r.getCell(i);
if(c != null){
sb.append(c.value);
sb.append("; ");
}else{
sb.append("null; ");
}
}
logger.info(sb.toString());
}
}
}

32
main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesStub.java Normal file
View File

@ -0,0 +1,32 @@
package com.metaweb.gridworks.tests.importers;
import java.util.List;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import com.metaweb.gridworks.importers.XmlImportUtilities;
import com.metaweb.gridworks.model.Project;
public class XmlImportUtilitiesStub extends XmlImportUtilities{
public List<String> detectRecordElementWrapper(XMLStreamReader parser, String tag) throws XMLStreamException{
return super.detectRecordElement(parser, tag);
}
public void ProcessSubRecordWrapper(Project project, XMLStreamReader parser, ImportColumnGroup columnGroup, ImportRecord record) throws XMLStreamException{
super.processSubRecord(project, parser, columnGroup, record);
}
public void findRecordWrapper(Project project, XMLStreamReader parser, String[] recordPath, int pathIndex, ImportColumnGroup rootColumnGroup) throws XMLStreamException{
super.findRecord(project, parser, recordPath, pathIndex, rootColumnGroup);
}
public void processRecordWrapper(Project project, XMLStreamReader parser, ImportColumnGroup rootColumnGroup) throws XMLStreamException{
super.processRecord(project, parser, rootColumnGroup);
}
public void addCellWrapper(Project project, ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, String text, int commonStartingRowIndex){
super.addCell(project, columnGroup, record, columnLocalName, text, commonStartingRowIndex);
}
}

336
main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImportUtilitiesTests.java Normal file
View File

@ -0,0 +1,336 @@
package com.metaweb.gridworks.tests.importers;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import javax.xml.stream.FactoryConfigurationError;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import org.apache.log4j.Level;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.metaweb.gridworks.importers.XmlImportUtilities.ImportColumn;
import com.metaweb.gridworks.importers.XmlImportUtilities.ImportColumnGroup;
import com.metaweb.gridworks.importers.XmlImportUtilities.ImportRecord;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
public class XmlImportUtilitiesTests {
final static Logger logger = LoggerFactory.getLogger("XmlImporterUtilitiesTests");
//dependencies
Project project;
XMLStreamReader parser;
ImportColumnGroup columnGroup;
ImportRecord record;
ByteArrayInputStream inputStream;
//System Under Test
XmlImportUtilitiesStub SUT;
@BeforeMethod
public void SetUp(){
org.apache.log4j.Logger.getRootLogger().setLevel(Level.toLevel("trace"));
SUT = new XmlImportUtilitiesStub();
project = new Project();
columnGroup = new ImportColumnGroup();
record = new ImportRecord();
}
@AfterMethod
public void TearDown() throws IOException{
SUT = null;
project = null;
parser = null;
columnGroup = null;
record = null;
if(inputStream != null)
inputStream.close();
inputStream = null;
}
@Test
public void detectPathFromTagTest(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
String tag = "library";
String[] response = XmlImportUtilitiesStub.detectPathFromTag(inputStream, tag);
Assert.assertNotNull(response);
Assert.assertEquals(response.length, 1);
Assert.assertEquals(response[0], "library");
}
@Test
public void detectPathFromTagWithNestedElement(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
String tag = "book";
String[] response = XmlImportUtilitiesStub.detectPathFromTag(inputStream, tag);
Assert.assertNotNull(response);
Assert.assertEquals(response.length, 2);
Assert.assertEquals(response[0], "library");
Assert.assertEquals(response[1], "book");
}
@Test
public void detectRecordElementTest(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
createParser();
String tag="library";
List<String> response = new ArrayList<String>();
try {
response = SUT.detectRecordElementWrapper(parser, tag);
} catch (XMLStreamException e) {
Assert.fail();
}
Assert.assertNotNull(response);
Assert.assertEquals(response.size(), 1);
Assert.assertEquals(response.get(0), "library");
}
@Test
public void detectRecordElementCanHandleWithNestedElements(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
createParser();
String tag="book";
List<String> response = new ArrayList<String>();
try {
response = SUT.detectRecordElementWrapper(parser, tag);
} catch (XMLStreamException e) {
Assert.fail();
}
Assert.assertNotNull(response);
Assert.assertEquals(response.size(), 2);
Assert.assertEquals(response.get(0), "library");
Assert.assertEquals(response.get(1), "book");
}
@Test
public void detectRecordElementIsNullForUnfoundTag(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
createParser();
String tag="";
List<String> response = new ArrayList<String>();
try {
response = SUT.detectRecordElementWrapper(parser, tag);
} catch (XMLStreamException e) {
Assert.fail();
}
Assert.assertNull(response);
}
@Test
public void detectRecordElementRegressionTest(){
loadSampleXml();
String[] path = XmlImportUtilitiesStub.detectRecordElement(inputStream);
Assert.assertNotNull(path);
Assert.assertEquals(path.length, 2);
Assert.assertEquals(path[0], "library");
Assert.assertEquals(path[1], "book");
}
@Test
public void importXmlTest(){
loadSampleXml();
String[] recordPath = new String[]{"library","book"};
XmlImportUtilitiesStub.importXml(inputStream, project, recordPath, columnGroup );
TestTools.PrintProject(project);
TestTools.AssertGridCreated(project, 0, 6);
Assert.assertEquals(project.rows.get(0).cells.size(), 4);
//TODO
}
@Test
public void createColumnsFromImportTest(){
ImportColumnGroup columnGroup = new ImportColumnGroup();
ImportColumn ic1 = new ImportColumn();
ic1.name = "hello";
ImportColumn ic2 = new ImportColumn();
ic2.name = "world";
ImportColumnGroup subGroup = new ImportColumnGroup();
ImportColumn ic3 = new ImportColumn();
ic3.name = "foo";
ImportColumn ic4 = new ImportColumn();
ic4.name = "bar";
subGroup.columns.put("c", ic3);
subGroup.columns.put("d", ic4);
columnGroup.columns.put("a", ic1);
columnGroup.columns.put("b", ic2);
columnGroup.subgroups.put("e", subGroup);
XmlImportUtilitiesStub.createColumnsFromImport(project, columnGroup);
TestTools.PrintProject(project);
TestTools.AssertGridCreated(project, 4, 0);
Assert.assertEquals(project.columnModel.columns.get(0).getName(), "world");
Assert.assertEquals(project.columnModel.columns.get(1).getName(), "hello");
Assert.assertEquals(project.columnModel.columns.get(2).getName(), "bar");
Assert.assertEquals(project.columnModel.columns.get(3).getName(), "foo");
Assert.assertEquals(project.columnModel.columnGroups.get(0).keyColumnIndex, 2);
Assert.assertEquals(project.columnModel.columnGroups.get(0).startColumnIndex, 2);
Assert.assertEquals(project.columnModel.columnGroups.get(0).columnSpan, 2);
}
@Test
public void findRecordTest(){
loadSampleXml();
createParser();
ParserSkip();
String[] recordPath = new String[]{"library","book"};
int pathIndex = 0;
try {
SUT.findRecordWrapper(project, parser, recordPath, pathIndex, columnGroup);
} catch (XMLStreamException e) {
Assert.fail();
}
TestTools.PrintProject(project);
TestTools.AssertGridCreated(project, 0, 6);
Assert.assertEquals(project.rows.get(0).cells.size(), 4);
//TODO
}
@Test
public void processRecordTest(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
createParser();
ParserSkip();
try {
SUT.processRecordWrapper(project, parser, columnGroup);
} catch (XMLStreamException e) {
Assert.fail();
}
TestTools.PrintProject(project);
Assert.assertNotNull(project.rows);
Assert.assertEquals(project.rows.size(), 1);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "author1");
}
@Test
public void processRecordTestDuplicateColumns(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><author>author2</author><genre>genre1</genre></book></library>");
createParser();
ParserSkip();
try {
SUT.processRecordWrapper(project, parser, columnGroup);
} catch (XMLStreamException e) {
Assert.fail();
}
TestTools.PrintProject(project);
Assert.assertNotNull(project.rows);
Assert.assertEquals(project.rows.size(), 2);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
Assert.assertEquals(row.cells.size(), 3);
Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "author1");
row = project.rows.get(1);
Assert.assertEquals(row.getCell(1).value, "author2");
}
@Test
public void processRecordTestNestedElement(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author><author-name>author1</author-name><author-dob>a date</author-dob></author><genre>genre1</genre></book></library>");
createParser();
ParserSkip();
try {
SUT.processRecordWrapper(project, parser, columnGroup);
} catch (XMLStreamException e) {
Assert.fail();
}
TestTools.PrintProject(project);
Assert.assertNotNull(project.rows);
Assert.assertEquals(project.rows.size(), 1);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
Assert.assertEquals(row.cells.size(), 4);
Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "author1");
Assert.assertNotNull(row.getCell(2));
Assert.assertEquals(row.getCell(2).value, "a date");
}
@Test(groups={"broken"})
public void processSubRecordTest(){
loadXml("<?xml version=\"1.0\"?><library><book id=\"1\"><author>author1</author><genre>genre1</genre></book></library>");
createParser();
ParserSkip();
try {
SUT.ProcessSubRecordWrapper(project, parser, columnGroup, record);
} catch (XMLStreamException e) {
Assert.fail();
}
TestTools.PrintProject(project);
Assert.fail();
//TODO need to verify 'record' was set correctly which we can't do as ImportRecord is an internal class
}
@Test(groups={"broken"})
public void addCellTest(){
String columnLocalName = "author";
String text = "Author1, The";
int commonStartingRowIndex = 0;
project.rows.add(new Row(0));
SUT.addCellWrapper(project, columnGroup, record, columnLocalName, text, commonStartingRowIndex);
Assert.fail();
//TODO need to verify 'record' was set correctly which we can't do as ImportRecord is an internal class
}
//----------------helpers-------------
public void loadSampleXml(){
loadXml( XmlImporterTests.getSample() );
}
public void loadXml(String xml){
try {
inputStream = new ByteArrayInputStream( xml.getBytes( "UTF-8" ) );
} catch (UnsupportedEncodingException e1) {
Assert.fail();
}
}
public void ParserSkip(){
try {
parser.next(); //move parser forward once e.g. skip the START_DOCUMENT parser event
} catch (XMLStreamException e1) {
Assert.fail();
}
}
public void createParser(){
try {
parser = XMLInputFactory.newInstance().createXMLStreamReader(inputStream);
} catch (XMLStreamException e1) {
Assert.fail();
} catch (FactoryConfigurationError e1) {
Assert.fail();
}
}
}

432
main/tests/server/src/com/metaweb/gridworks/tests/importers/XmlImporterTests.java
View File

@ -1,207 +1,225 @@
package com.metaweb.gridworks.tests.importers; package com.metaweb.gridworks.tests.importers;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.Properties; import java.util.Properties;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterMethod; import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.metaweb.gridworks.importers.XmlImporter; import com.metaweb.gridworks.importers.XmlImporter;
import com.metaweb.gridworks.model.Cell; import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Column; import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
public class XmlImporterTests {
final static Logger logger = LoggerFactory.getLogger("XmlImporterTests");
public class XmlImporterTests {
final static Logger logger = LoggerFactory.getLogger("XmlImporterTests"); //dependencies
Project project = null;
//dependencies Properties options = null;
Project project = null; ByteArrayInputStream inputStream = null;
Properties options = null;
ByteArrayInputStream inputStream = null; //System Under Test
XmlImporter SUT = null;
//System Under Test
XmlImporter SUT = null;
@BeforeMethod
public void SetUp(){
@BeforeMethod SUT = new XmlImporter();
public void SetUp(){ project = new Project();
SUT = new XmlImporter(); options = mock(Properties.class);
project = new Project(); }
options = mock(Properties.class);
} @AfterMethod
public void TearDown() throws IOException{
@AfterMethod SUT = null;
public void TearDown(){ project = null;
SUT = null; options = null;
project = null; inputStream.close();
options = null; inputStream = null;
} }
@Test @Test
public void canParseSample(){ public void canParseSample(){
RunTest(getSample());
RunTest(getSample());
TestTools.AssertGridCreated(project, 4, 6);
AssertGridCreate(project, 4, 6); TestTools.PrintProject(project);
PrintProject(project);
Row row = project.rows.get(0);
Row row = project.rows.get(0); Assert.assertNotNull(row);
Assert.assertNotNull(row); Assert.assertNotNull(row.getCell(1));
Assert.assertNotNull(row.cells); Assert.assertEquals(row.getCell(1).value, "Author 1, The");
Assert.assertNotNull(row.cells.get(2)); }
Assert.assertEquals(row.cells.get(2).value, "Author 1, The");
@Test
public void canParseSampleWithDuplicateNestedElements(){
} RunTest(getSampleWithDuplicateNestedElements());
@Test TestTools.PrintProject(project);
public void testCanParseLineBreak(){ TestTools.AssertGridCreated(project, 4, 12);
RunTest(getSampleWithLineBreak()); Row row = project.rows.get(0);
Assert.assertNotNull(row);
AssertGridCreate(project, 4, 6); Assert.assertEquals(row.cells.size(), 4);
PrintProject(project); Assert.assertNotNull(row.getCell(2));
Assert.assertEquals(row.getCell(1).value, "Author 1, The");
Row row = project.rows.get(3); Assert.assertEquals(project.rows.get(1).getCell(1).value, "Author 1, Another");
Assert.assertNotNull(row); }
Assert.assertNotNull(row.cells);
Assert.assertNotNull(row.cells.get(2)); @Test
Assert.assertEquals(row.cells.get(2).value, "With line\n break"); public void testCanParseLineBreak(){
}
RunTest(getSampleWithLineBreak());
@Test(groups={"broken"})
public void testElementsWithVaryingStructure(){ TestTools.AssertGridCreated(project, 4, 6);
TestTools.PrintProject(project);
RunTest(getSampleWithVaryingStructure()); Row row = project.rows.get(3);
Assert.assertNotNull(row);
AssertGridCreate(project, 5, 6); Assert.assertEquals(row.cells.size(), 4);
PrintProject(project); Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "With line\n break");
Row row0 = project.rows.get(0); }
Assert.assertNotNull(row0);
Assert.assertNotNull(row0.cells); @Test(groups={"broken"})
Assert.assertEquals(row0.cells.size(),6); public void testElementsWithVaryingStructure(){
RunTest(getSampleWithVaryingStructure());
Row row5 = project.rows.get(5);
Assert.assertNotNull(row5); TestTools.AssertGridCreated(project, 5, 6);
Assert.assertNotNull(row5.cells); TestTools.PrintProject(project);
Assert.assertEquals(row5.cells.size(),6);
Row row0 = project.rows.get(0);
Assert.assertNotNull(row0);
} Assert.assertEquals(row0.cells.size(),6);
//------------helper methods--------------- Row row5 = project.rows.get(5);
Assert.assertNotNull(row5);
protected String getTypicalElement(int id){ Assert.assertEquals(row5.cells.size(),6);
return "<book id=\"" + id + "\">" + }
"<author>Author " + id + ", The</author>" +
"<title>Book title " + id + "</title>" + @Test
"<publish_date>2010-05-26</publish_date>" + public void testElementWithNestedTree(){
"</book>"; RunTest(getSampleWithTreeStructure());
} TestTools.AssertGridCreated(project, 5, 6);
TestTools.PrintProject(project);
protected String getSample(){ Assert.assertEquals(project.columnModel.columnGroups.size(),1);
StringBuilder sb = new StringBuilder(); Assert.assertEquals(project.columnModel.columnGroups.get(0).keyColumnIndex, 2);
sb.append("<?xml version=\"1.0\"?><library>"); Assert.assertEquals(project.columnModel.columnGroups.get(0).startColumnIndex, 2);
for(int i = 1; i < 7; i++){ Assert.assertNull(project.columnModel.columnGroups.get(0).parentGroup);
sb.append(getTypicalElement(i)); Assert.assertEquals(project.columnModel.columnGroups.get(0).subgroups.size(),0);
} Assert.assertEquals(project.columnModel.columnGroups.get(0).columnSpan,2);
sb.append("</library>"); }
return sb.toString();
} //------------helper methods---------------
protected String getSampleWithLineBreak(){ public static String getTypicalElement(int id){
StringBuilder sb = new StringBuilder(); return "<book id=\"" + id + "\">" +
sb.append("<?xml version=\"1.0\"?><library>"); "<author>Author " + id + ", The</author>" +
for(int i = 1; i < 4; i++){ "<title>Book title " + id + "</title>" +
sb.append(getTypicalElement(i)); "<publish_date>2010-05-26</publish_date>" +
} "</book>";
sb.append("<book id=\"4\">" + }
"<author>With line\n break</author>" +
"<title>Book title 4</title>" + public static String getElementWithDuplicateSubElement(int id){
"<publish_date>2010-05-26</publish_date>" + return "<book id=\"" + id + "\">" +
"</book>"); "<author>Author " + id + ", The</author>" +
sb.append(getTypicalElement(5)); "<author>Author " + id + ", Another</author>" +
sb.append(getTypicalElement(6)); "<title>Book title " + id + "</title>" +
sb.append("</library>"); "<publish_date>2010-05-26</publish_date>" +
return sb.toString(); "</book>";
} }
protected String getSampleWithVaryingStructure(){ public static String getSample(){
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append("<?xml version=\"1.0\"?><library>"); sb.append("<?xml version=\"1.0\"?><library>");
for(int i = 1; i < 6; i++){ for(int i = 1; i < 7; i++){
sb.append(getTypicalElement(i)); sb.append(getTypicalElement(i));
} }
sb.append("<book id=\"6\">" + sb.append("</library>");
"<author>With line\n break</author>" + return sb.toString();
"<title>Book title 6</title>" + }
"<genre>New element not seen in other records</genre>" +
"<publish_date>2010-05-26</publish_date>" + public static String getSampleWithDuplicateNestedElements(){
"</book>"); StringBuilder sb = new StringBuilder();
sb.append("</library>"); sb.append("<?xml version=\"1.0\"?><library>");
return sb.toString(); for(int i = 1; i < 7; i++){
} sb.append(getElementWithDuplicateSubElement(i));
}
private void RunTest(String testString){ sb.append("</library>");
try { return sb.toString();
inputStream = new ByteArrayInputStream( testString.getBytes( "UTF-8" ) );
} catch (UnsupportedEncodingException e1) { }
Assert.fail();
} public static String getSampleWithLineBreak(){
StringBuilder sb = new StringBuilder();
try { sb.append("<?xml version=\"1.0\"?><library>");
SUT.read(inputStream, project, options); for(int i = 1; i < 4; i++){
} catch (Exception e) { sb.append(getTypicalElement(i));
Assert.fail(); }
} sb.append("<book id=\"4\">" +
"<author>With line\n break</author>" +
try { "<title>Book title 4</title>" +
inputStream.close(); "<publish_date>2010-05-26</publish_date>" +
} catch (IOException e) { "</book>");
Assert.fail(); sb.append(getTypicalElement(5));
} sb.append(getTypicalElement(6));
} sb.append("</library>");
return sb.toString();
private void AssertGridCreate(Project project, int numCols, int numRows){ }
Assert.assertNotNull(project);
Assert.assertNotNull(project.columnModel); public static String getSampleWithVaryingStructure(){
Assert.assertNotNull(project.columnModel.columns); StringBuilder sb = new StringBuilder();
Assert.assertEquals(project.columnModel.columns.size(), numCols); sb.append("<?xml version=\"1.0\"?><library>");
Assert.assertNotNull(project.rows); for(int i = 1; i < 6; i++){
Assert.assertEquals(project.rows.size(), numRows); sb.append(getTypicalElement(i));
} }
sb.append("<book id=\"6\">" +
private void PrintProject(Project project){ "<author>Author 6, The</author>" +
//some quick and dirty debugging "<title>Book title 6</title>" +
StringBuilder sb = new StringBuilder(); "<genre>New element not seen in other records</genre>" +
for(Column c : project.columnModel.columns){ "<publish_date>2010-05-26</publish_date>" +
sb.append(c.getName()); "</book>");
sb.append("; "); sb.append("</library>");
} return sb.toString();
logger.info(sb.toString()); }
for(Row r : project.rows){
sb = new StringBuilder(); public static String getSampleWithTreeStructure(){
for(Cell c : r.cells){ StringBuilder sb = new StringBuilder();
if(c != null){ sb.append("<?xml version=\"1.0\"?><library>");
sb.append(c.value); for(int i = 1; i < 7; i++){
sb.append("; "); sb.append("<book id=\"" + i + "\">" +
}else{ "<author><author-name>Author " + i + ", The</author-name>" +
sb.append("null; "); "<author-dob>1950-0" + i + "-15</author-dob></author>" +
} "<title>Book title " + i + "</title>" +
} "<publish_date>2010-05-26</publish_date>" +
logger.info(sb.toString()); "</book>");
} }
} sb.append("</library>");
} return sb.toString();
}
private void RunTest(String testString){
try {
inputStream = new ByteArrayInputStream( testString.getBytes( "UTF-8" ) );
} catch (UnsupportedEncodingException e1) {
Assert.fail();
}
try {
SUT.read(inputStream, project, options);
} catch (Exception e) {
Assert.fail();
}
}
}

5
main/tests/server/src/log4j.properties
View File

@ -1,4 +1,7 @@
log4j.rootLogger=ERROR, console log4j.rootLogger=ERROR, console
log4j.logger.org.apache.http.headers=WARN
log4j.logger.org.apache.http.impl=WARN
log4j.logger.org.apache.http.client=WARN
log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=com.metaweb.util.logging.IndentingLayout log4j.appender.console.layout=com.metaweb.gridworks.logging.IndentingLayout

1
main/webapp/WEB-INF/web.xml
View File

@ -8,7 +8,6 @@
<servlet> <servlet>
<servlet-name>gridworks</servlet-name> <servlet-name>gridworks</servlet-name>
<servlet-class>com.metaweb.gridworks.GridworksServlet</servlet-class> <servlet-class>com.metaweb.gridworks.GridworksServlet</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet> </servlet>
<servlet-mapping> <servlet-mapping>

7
server/IDEs/eclipse/Gridworks.launch
View File

@ -1,12 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication"> <launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS"> <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
<listEntry value="/gridworks/src/server/java/com/metaweb/gridworks/Gridworks.java"/> <listEntry value="/gridworks server/src/com/metaweb/gridworks/Gridworks.java"/>
</listAttribute> </listAttribute>
<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES"> <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
<listEntry value="1"/> <listEntry value="1"/>
</listAttribute> </listAttribute>
<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="com.metaweb.gridworks.Gridworks"/> <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="com.metaweb.gridworks.Gridworks"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="gridworks"/> <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="gridworks server"/>
<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Dgridworks.autoreloading=true -Dgridworks.webapp=src/main/webapp"/> <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Dgridworks.autoreloading=true"/>
<stringAttribute key="org.eclipse.jdt.launching.WORKING_DIRECTORY" value="${workspace_loc:gridworks}"/>
</launchConfiguration> </launchConfiguration>

147
server/src/com/metaweb/gridworks/Gridworks.java
View File

@ -24,11 +24,14 @@ import org.apache.log4j.Level;
import org.mortbay.jetty.Connector; import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server; import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector; import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.servlet.ServletHolder;
import org.mortbay.jetty.webapp.WebAppContext; import org.mortbay.jetty.webapp.WebAppContext;
import org.mortbay.util.Scanner; import org.mortbay.util.Scanner;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.codeberry.jdatapath.DataPath;
import com.codeberry.jdatapath.JDataPathSystem;
import com.metaweb.util.threads.ThreadPoolExecutorAdapter; import com.metaweb.util.threads.ThreadPoolExecutorAdapter;
/** /**
@ -37,38 +40,15 @@ import com.metaweb.util.threads.ThreadPoolExecutorAdapter;
*/ */
public class Gridworks { public class Gridworks {
static private final String VERSION = "1.0";
static private final String DEFAULT_HOST = "127.0.0.1"; static private final String DEFAULT_HOST = "127.0.0.1";
static private final int DEFAULT_PORT = 3333; static private final int DEFAULT_PORT = 3333;
static private final int MAX_UPLOAD_SIZE = 1024 * 1024 * 1024;
static private File tempDir;
static private int port; static private int port;
static private String host; static private String host;
final static Logger logger = LoggerFactory.getLogger("gridworks"); final static Logger logger = LoggerFactory.getLogger("gridworks");
public static String getVersion() { public static void main(String[] args) throws Exception {
return VERSION;
}
public static File getTempFile(String name) {
return new File(tempDir, name);
}
public static File getTempDir() {
return tempDir;
}
public static int getMaxUploadSize() {
return Configurations.getInteger("gridworks.max_upload_size",MAX_UPLOAD_SIZE);
}
public static String getFullHost() {
return host + ":" + port;
}
public static void main(String[] args) throws Exception {
// tell jetty to use SLF4J for logging instead of its own stuff // tell jetty to use SLF4J for logging instead of its own stuff
System.setProperty("VERBOSE","false"); System.setProperty("VERBOSE","false");
@ -89,9 +69,6 @@ public class Gridworks {
// set the log verbosity level // set the log verbosity level
org.apache.log4j.Logger.getRootLogger().setLevel(Level.toLevel(Configurations.get("gridworks.verbosity","info"))); org.apache.log4j.Logger.getRootLogger().setLevel(Level.toLevel(Configurations.get("gridworks.verbosity","info")));
tempDir = new File(Configurations.get("gridworks.temp","temp"));
if (!tempDir.exists()) tempDir.mkdirs();
port = Configurations.getInteger("gridworks.port",DEFAULT_PORT); port = Configurations.getInteger("gridworks.port",DEFAULT_PORT);
host = Configurations.get("gridworks.host",DEFAULT_HOST); host = Configurations.get("gridworks.host",DEFAULT_HOST);
@ -117,7 +94,8 @@ public class Gridworks {
// hook up the signal handlers // hook up the signal handlers
Runtime.getRuntime().addShutdownHook( Runtime.getRuntime().addShutdownHook(
new Thread(new ShutdownSignalHandler(server))); new Thread(new ShutdownSignalHandler(server))
);
server.join(); server.join();
} }
@ -178,7 +156,15 @@ class GridworksServer extends Server {
scanForUpdates(contextRoot, context); scanForUpdates(contextRoot, context);
} }
// start the server
this.start(); this.start();
// inject configuration parameters in the servlets
// NOTE: this is done *after* starting the server because jetty might override the init
// parameters if we set them in the webapp context upon reading the web.xml file
ServletHolder servlet = context.getServletHandler().getServlet("gridworks");
servlet.setInitParameter("gridworks.data", getDataDir());
servlet.doStart();
} }
@Override @Override
@ -194,7 +180,7 @@ class GridworksServer extends Server {
} }
} }
private void scanForUpdates(final File contextRoot, final WebAppContext context) { static private void scanForUpdates(final File contextRoot, final WebAppContext context) {
List<File> scanList = new ArrayList<File>(); List<File> scanList = new ArrayList<File>();
scanList.add(new File(contextRoot, "WEB-INF/web.xml")); scanList.add(new File(contextRoot, "WEB-INF/web.xml"));
@ -225,7 +211,7 @@ class GridworksServer extends Server {
scanner.start(); scanner.start();
} }
private void findFiles(final String extension, File baseDir, final Collection<File> found) { static private void findFiles(final String extension, File baseDir, final Collection<File> found) {
baseDir.listFiles(new FileFilter() { baseDir.listFiles(new FileFilter() {
public boolean accept(File pathname) { public boolean accept(File pathname) {
if (pathname.isDirectory()) { if (pathname.isDirectory()) {
@ -238,6 +224,103 @@ class GridworksServer extends Server {
}); });
} }
static private String getDataDir() {
String data_dir = Configurations.get("gridworks.data_dir");
if (data_dir != null) {
return data_dir;
}
String os = System.getProperty("os.name").toLowerCase();
if (os.contains("windows")) {
try {
// NOTE(SM): finding the "local data app" in windows from java is actually a PITA
// see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
// so we're using a library that uses JNI to ask directly the win32 APIs,
// it's not elegant but it's the safest bet.
DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
File data = new File(fixWindowsUnicodePath(localDataPath.getPath()));
data.mkdirs();
return data.getAbsolutePath();
} catch (Error e) {
/*
* The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll
* we include is compiled for 32-bit. In this case, we just have to dig up
* environment variables and try our best to find a user-specific path.
*/
logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables");
File parentDir = null;
String appData = System.getenv("APPDATA");
if (appData != null && appData.length() > 0) {
// e.g., C:\Users\[userid]\AppData\Roaming
parentDir = new File(appData);
} else {
String userProfile = System.getenv("USERPROFILE");
if (userProfile != null && userProfile.length() > 0) {
// e.g., C:\Users\[userid]
parentDir = new File(userProfile);
}
}
if (parentDir == null) {
parentDir = new File(".");
}
File data = new File(parentDir, "Gridworks");
data.mkdirs();
return data.getAbsolutePath();
}
} else if (os.contains("mac os x")) {
// on macosx, use "~/Library/Application Support"
String home = System.getProperty("user.home");
String data_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks";
File data = new File(data_home);
data.mkdirs();
return data.getAbsolutePath();
} else { // most likely a UNIX flavor
// start with the XDG environment
// see http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
String data_home = System.getenv("XDG_DATA_HOME");
if (data_home == null) { // if not found, default back to ~/.local/share
String home = System.getProperty("user.home");
if (home == null) home = ".";
data_home = home + "/.local/share";
}
File data = new File(data_home + "/gridworks");
data.mkdirs();
return data.getAbsolutePath();
}
}
/**
* For Windows file paths that contain user IDs with non ASCII characters,
* those characters might get replaced with ?. We need to use the environment
* APPDATA value to substitute back the original user ID.
*/
static private String fixWindowsUnicodePath(String path) {
int q = path.indexOf('?');
if (q < 0) {
return path;
}
int pathSep = path.indexOf(File.separatorChar, q);
String goodPath = System.getenv("APPDATA");
if (goodPath == null || goodPath.length() == 0) {
goodPath = System.getenv("USERPROFILE");
if (!goodPath.endsWith(File.separator)) {
goodPath = goodPath + File.separator;
}
}
int goodPathSep = goodPath.indexOf(File.separatorChar, q);
return path.substring(0, q) + goodPath.substring(q, goodPathSep) + path.substring(pathSep);
}
} }
/* -------------- Gridworks Client ----------------- */ /* -------------- Gridworks Client ----------------- */

3
server/src/log4j.properties
View File

@ -1,9 +1,6 @@
log4j.rootLogger=INFO, console log4j.rootLogger=INFO, console
log4j.logger.org.mortbay.log=WARN log4j.logger.org.mortbay.log=WARN
log4j.logger.org.mortbay.jetty=ERROR log4j.logger.org.mortbay.jetty=ERROR
log4j.logger.org.apache.http.headers=WARN
log4j.logger.org.apache.http.impl=WARN
log4j.logger.org.apache.http.client=WARN
log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=com.metaweb.util.logging.IndentingLayout log4j.appender.console.layout=com.metaweb.util.logging.IndentingLayout