diff --git a/.classpath b/.classpath index 80c2952c7..16370e936 100644 --- a/.classpath +++ b/.classpath @@ -16,5 +16,6 @@ + diff --git a/gridworks.bat b/gridworks.bat index 988e89587..e7f9acdcd 100644 --- a/gridworks.bat +++ b/gridworks.bat @@ -8,7 +8,7 @@ rem rem JAVA_OPTIONS rem Extra options to pass to the JVM rem - + if "%OS%"=="Windows_NT" @setlocal if "%OS%"=="WINNT" @setlocal @@ -22,26 +22,26 @@ echo where [options] include: echo. echo /h print this message and exit echo. -echo /p the port that Gridworks will listen to -echo default: 3333 -echo. -echo /i the host interface gridworks should bind to -echo default: 127.0.0.1 -echo. -echo /w path to the webapp -echo default src\main\webapp -echo. -echo /d enable JVM debugging (on port 8000) +echo /p the port that Gridworks will listen to +echo default: 3333 +echo. +echo /i the host interface gridworks should bind to +echo default: 127.0.0.1 +echo. +echo /w path to the webapp +echo default src\main\webapp +echo. +echo /d enable JVM debugging (on port 8000) echo. echo /x enable JMX monitoring (for jconsole and friends) echo. -echo and is one of -echo. -echo build ..................... Build Gridworks -echo run ....................... Run Gridworks -echo. -echo clean ..................... Clean compiled classes -echo distclean ................. Remove all generated files +echo and is one of +echo. +echo build ..................... Build Gridworks +echo run ....................... Run Gridworks +echo. +echo clean ..................... Clean compiled classes +echo distclean ................. Remove all generated files echo. goto end @@ -73,16 +73,16 @@ goto endArgumentParsing :arg-p set GRIDWORKS_PORT=%2 goto shift2loop - -:arg-i -set GRIDWORKS_HOST=%2 -goto shift2loop - -:arg-w -set GRIDWORKS_WEBAPP=%2 -goto shift2loop - -:arg-d + +:arg-i +set GRIDWORKS_HOST=%2 +goto shift2loop + +:arg-w +set GRIDWORKS_WEBAPP=%2 +goto shift2loop + +:arg-d set OPTS=%OPTS% -Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n goto shift2loop @@ -108,43 +108,43 @@ if not "%GRIDWORKS_PORT%" == "" goto gotPort set GRIDWORKS_PORT=3333 :gotPort set OPTS=%OPTS% -Dgridworks.port=%GRIDWORKS_PORT% - + if not "%GRIDWORKS_HOST%" == "" goto gotHost set GRIDWORKS_HOST=127.0.0.1 :gotHOST set OPTS=%OPTS% -Dgridworks.host=%GRIDWORKS_HOST% - + if not "%GRIDWORKS_WEBAPP%" == "" goto gotHost set GRIDWORKS_WEBAPP=src\main\webapp :gotHOST set OPTS=%OPTS% -Dgridworks.webapp=%GRIDWORKS_WEBAPP% - + if not "%GRIDWORKS_BUILD_DIR%" == "" goto gotBuildDir set GRIDWORKS_BUILD_DIR=build :gotBuildDir - + if not "%GRIDWORKS_LIB_DIR%" == "" goto gotLibDir set GRIDWORKS_LIB_DIR=lib :gotLibDir rem ----- Respond to the action ---------------------------------------------------------- - + set ACTION=%1 if ""%ACTION%"" == ""build"" goto doAnt if ""%ACTION%"" == ""clean"" goto doAnt if ""%ACTION%"" == ""distclean"" goto doAnt if ""%ACTION%"" == ""run"" goto doRun - -goto usage + +goto usage :doRun -set CLASSPATH="%GRIDWORKS_BUILD_DIR%\classes;%GRIDWORKS_LIB_DIR%\*" -"%JAVA_HOME%\bin\java.exe" -cp %CLASSPATH% %OPTS% -Djava.library.path=lib/native/windows com.metaweb.gridworks.Gridworks -goto end - -:doAnt -ant -f build.xml -Dbuild.dir="%GRIDWORKS_BUILD_DI%" -Ddist.dir="%GRIDWORKS_DIST_DIR%" -Dversion="%VERSION%" %ACTION% -goto end +set CLASSPATH="%GRIDWORKS_BUILD_DIR%\classes;%GRIDWORKS_LIB_DIR%\*" +"%JAVA_HOME%\bin\java.exe" -cp %CLASSPATH% %OPTS% -Djava.library.path=lib/native/windows com.metaweb.gridworks.Gridworks +goto end + +:doAnt +ant -f build.xml -Dbuild.dir="%GRIDWORKS_BUILD_DIR%" -Ddist.dir="%GRIDWORKS_DIST_DIR%" -Dversion="%VERSION%" %ACTION% +goto end :end diff --git a/lib/apache-tools-tar.jar b/lib/apache-tools-tar.jar new file mode 100644 index 000000000..fecb83825 Binary files /dev/null and b/lib/apache-tools-tar.jar differ diff --git a/src/main/java/com/metaweb/gridworks/GridworksServlet.java b/src/main/java/com/metaweb/gridworks/GridworksServlet.java index 8ea2b1e86..d51428f52 100644 --- a/src/main/java/com/metaweb/gridworks/GridworksServlet.java +++ b/src/main/java/com/metaweb/gridworks/GridworksServlet.java @@ -18,6 +18,8 @@ import com.metaweb.gridworks.commands.edit.AnnotateRowsCommand; import com.metaweb.gridworks.commands.edit.ApplyOperationsCommand; import com.metaweb.gridworks.commands.edit.CreateProjectCommand; import com.metaweb.gridworks.commands.edit.DeleteProjectCommand; +import com.metaweb.gridworks.commands.edit.ExportProjectCommand; +import com.metaweb.gridworks.commands.edit.ImportProjectCommand; import com.metaweb.gridworks.commands.edit.TextTransformCommand; import com.metaweb.gridworks.commands.edit.EditOneCellCommand; import com.metaweb.gridworks.commands.edit.MassEditCommand; @@ -60,6 +62,8 @@ public class GridworksServlet extends HttpServlet { static { _commands.put("create-project-from-upload", new CreateProjectCommand()); + _commands.put("import-project", new ImportProjectCommand()); + _commands.put("export-project", new ExportProjectCommand()); _commands.put("export-rows", new ExportRowsCommand()); _commands.put("get-project-metadata", new GetProjectMetadataCommand()); diff --git a/src/main/java/com/metaweb/gridworks/ProjectManager.java b/src/main/java/com/metaweb/gridworks/ProjectManager.java index 75ba90f8f..7c158b385 100644 --- a/src/main/java/com/metaweb/gridworks/ProjectManager.java +++ b/src/main/java/com/metaweb/gridworks/ProjectManager.java @@ -117,6 +117,38 @@ public class ProjectManager { } } + public void importProject(long projectID) { + synchronized (this) { + ProjectMetadata metadata = ProjectMetadata.load(getProjectDir(projectID)); + + _projectsMetadata.put(projectID, metadata); + } + } + + public void ensureProjectSaved(long id) { + synchronized (this) { + File projectDir = getProjectDir(id); + + ProjectMetadata metadata = _projectsMetadata.get(id); + if (metadata != null) { + try { + metadata.save(projectDir); + } catch (Exception e) { + e.printStackTrace(); + } + } + + Project project = _projects.get(id); + if (project != null && metadata.getModified().after(project.lastSave)) { + try { + project.save(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + public ProjectMetadata getProjectMetadata(long id) { return _projectsMetadata.get(id); } diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java index 6125672fe..1332724a7 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java @@ -38,7 +38,7 @@ public class CreateProjectCommand extends Command { throws ServletException, IOException { try { - Properties options = parseUrlParameters(request); + Properties options = ParsingUtilities.parseUrlParameters(request); Project project = new Project(); internalImport(request, project, options); @@ -60,27 +60,6 @@ public class CreateProjectCommand extends Command { } } - protected Properties parseUrlParameters(HttpServletRequest request) { - Properties options = new Properties(); - - String query = request.getQueryString(); - if (query != null) { - if (query.startsWith("?")) { - query = query.substring(1); - } - - String[] pairs = query.split("&"); - for (String pairString : pairs) { - int equal = pairString.indexOf('='); - String name = equal >= 0 ? pairString.substring(0, equal) : ""; - String value = equal >= 0 ? ParsingUtilities.decode(pairString.substring(equal + 1)) : ""; - - options.put(name, value); - } - } - return options; - } - protected void internalImport( HttpServletRequest request, Project project, diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/ExportProjectCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/ExportProjectCommand.java new file mode 100644 index 000000000..e96bdb622 --- /dev/null +++ b/src/main/java/com/metaweb/gridworks/commands/edit/ExportProjectCommand.java @@ -0,0 +1,94 @@ +package com.metaweb.gridworks.commands.edit; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.tools.tar.TarEntry; +import org.apache.tools.tar.TarOutputStream; + +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; + +public class ExportProjectCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + ProjectManager.singleton.ensureProjectSaved(project.id); + + response.setHeader("Content-Type", "application/x-tar"); + + OutputStream os = response.getOutputStream(); + try { + tarToOutputStream( + ProjectManager.singleton.getProjectDir(project.id), + os + ); + } finally { + os.close(); + } + } catch (Exception e) { + respondException(response, e); + } + } + + protected void tarToOutputStream(File dir, OutputStream os) throws IOException { + TarOutputStream tos = new TarOutputStream(os); + try { + tarDir("", dir, tos); + } finally { + tos.close(); + } + } + + protected void tarDir(String relative, File dir, TarOutputStream tos) throws IOException { + File[] files = dir.listFiles(); + for (File file : files) { + if (!file.isHidden()) { + String path = relative + file.getName(); + + if (file.isDirectory()) { + tarDir(path + File.separator, file, tos); + } else { + TarEntry entry = new TarEntry(path); + + entry.setMode(TarEntry.DEFAULT_FILE_MODE); + entry.setSize(file.length()); + entry.setModTime(file.lastModified()); + + tos.putNextEntry(entry); + + copyFile(file, tos); + + tos.closeEntry(); + } + } + } + } + + protected void copyFile(File file, OutputStream os) throws IOException { + final int buffersize = 4096; + + FileInputStream fis = new FileInputStream(file); + try { + byte[] buf = new byte[buffersize]; + int count; + + while((count = fis.read(buf, 0, buffersize)) != -1) { + os.write(buf, 0, count); + } + } finally { + fis.close(); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/ImportProjectCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/ImportProjectCommand.java new file mode 100644 index 000000000..1c5f429e9 --- /dev/null +++ b/src/main/java/com/metaweb/gridworks/commands/edit/ImportProjectCommand.java @@ -0,0 +1,157 @@ +package com.metaweb.gridworks.commands.edit; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.tools.tar.TarEntry; +import org.apache.tools.tar.TarInputStream; + +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.util.ParsingUtilities; +import com.oreilly.servlet.multipart.FilePart; +import com.oreilly.servlet.multipart.MultipartParser; +import com.oreilly.servlet.multipart.ParamPart; +import com.oreilly.servlet.multipart.Part; + +public class ImportProjectCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Properties options = ParsingUtilities.parseUrlParameters(request); + long projectID = Project.generateID(); + + internalImport(request, options, projectID); + + ProjectManager.singleton.importProject(projectID); + if (options.containsKey("project-name")) { + String projectName = options.getProperty("project-name"); + if (projectName != null && projectName.length() > 0) { + ProjectManager.singleton.getProjectMetadata(projectID).setName(projectName); + } + } + + redirect(response, "/project.html?project=" + projectID); + } catch (Exception e) { + e.printStackTrace(); + } + } + + protected void internalImport( + HttpServletRequest request, + Properties options, + long projectID + ) throws Exception { + MultipartParser parser = null; + try { + parser = new MultipartParser(request, 20 * 1024 * 1024); + } catch (Exception e) { + // silent + } + + if (parser != null) { + Part part = null; + String url = null; + + while ((part = parser.readNextPart()) != null) { + if (part.isFile()) { + FilePart filePart = (FilePart) part; + InputStream inputStream = filePart.getInputStream(); + try { + internalImportInputStream(projectID, inputStream); + } finally { + inputStream.close(); + } + } else if (part.isParam()) { + ParamPart paramPart = (ParamPart) part; + String paramName = paramPart.getName(); + if (paramName.equals("url")) { + url = paramPart.getStringValue(); + } else { + options.put(paramName, paramPart.getStringValue()); + } + } + } + + if (url != null && url.length() > 0) { + internalImportURL(request, options, projectID, url); + } + } + } + + protected void internalImportURL( + HttpServletRequest request, + Properties options, + long projectID, + String urlString + ) throws Exception { + URL url = new URL(urlString); + URLConnection connection = null; + + try { + connection = url.openConnection(); + connection.setConnectTimeout(5000); + connection.connect(); + } catch (Exception e) { + throw new Exception("Cannot connect to " + urlString, e); + } + + InputStream inputStream = null; + try { + inputStream = connection.getInputStream(); + } catch (Exception e) { + throw new Exception("Cannot retrieve content from " + url, e); + } + + try { + internalImportInputStream(projectID, inputStream); + } finally { + inputStream.close(); + } + } + + protected void internalImportInputStream(long projectID, InputStream inputStream) throws IOException { + File destDir = ProjectManager.singleton.getProjectDir(projectID); + destDir.mkdirs(); + + untar(destDir, inputStream); + } + + protected void untar(File destDir, InputStream inputStream) throws IOException { + TarInputStream tin = new TarInputStream(inputStream); + TarEntry tarEntry = null; + + while ((tarEntry = tin.getNextEntry()) != null) { + File destEntry = new File(destDir, tarEntry.getName()); + File parent = destEntry.getParentFile(); + + if (!parent.exists()) { + parent.mkdirs(); + } + + if (tarEntry.isDirectory()) { + destEntry.mkdirs(); + } else { + FileOutputStream fout = new FileOutputStream(destEntry); + try { + tin.copyEntryContents(fout); + } finally { + fout.close(); + } + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/Project.java b/src/main/java/com/metaweb/gridworks/model/Project.java index ec1812e96..2f4a264ba 100644 --- a/src/main/java/com/metaweb/gridworks/model/Project.java +++ b/src/main/java/com/metaweb/gridworks/model/Project.java @@ -38,8 +38,12 @@ public class Project { transient public ProcessManager processManager = new ProcessManager(); transient public Date lastSave = new Date(); + static public long generateID() { + return System.currentTimeMillis() + Math.round(Math.random() * 1000000000000L); + } + public Project() { - id = System.currentTimeMillis() + Math.round(Math.random() * 1000000000000L); + id = generateID(); history = new History(this); } diff --git a/src/main/java/com/metaweb/gridworks/util/ParsingUtilities.java b/src/main/java/com/metaweb/gridworks/util/ParsingUtilities.java index 990ae7e90..46eda1cc7 100644 --- a/src/main/java/com/metaweb/gridworks/util/ParsingUtilities.java +++ b/src/main/java/com/metaweb/gridworks/util/ParsingUtilities.java @@ -8,6 +8,9 @@ import java.io.UnsupportedEncodingException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.Properties; + +import javax.servlet.http.HttpServletRequest; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.net.URLCodec; @@ -19,6 +22,27 @@ import org.json.JSONTokener; public class ParsingUtilities { static public SimpleDateFormat s_sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + static public Properties parseUrlParameters(HttpServletRequest request) { + Properties options = new Properties(); + + String query = request.getQueryString(); + if (query != null) { + if (query.startsWith("?")) { + query = query.substring(1); + } + + String[] pairs = query.split("&"); + for (String pairString : pairs) { + int equal = pairString.indexOf('='); + String name = equal >= 0 ? pairString.substring(0, equal) : ""; + String value = equal >= 0 ? ParsingUtilities.decode(pairString.substring(equal + 1)) : ""; + + options.put(name, value); + } + } + return options; + } + static public String inputStreamToString(InputStream is) throws IOException { Reader reader = new InputStreamReader(is, "UTF-8"); try { diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html index a20aafd5b..b2b75e05c 100644 --- a/src/main/webapp/index.html +++ b/src/main/webapp/index.html @@ -1 +1 @@ - Gridworks
Gridworks
Gridworks
Data File:
Project Name:
Load up to: data rows (optional)
Skip: initial data rows (optional)
\ No newline at end of file + Gridworks
Gridworks
Gridworks

Upload Data File

Data File:
Project Name:
Load up to: data rows (optional)
Skip: initial data rows (optional)

Import Existing Project

Project TAR File:
Re-name Project: (optional)
\ No newline at end of file diff --git a/src/main/webapp/scripts/project/menu-bar.js b/src/main/webapp/scripts/project/menu-bar.js index d1a62d536..1d000b57f 100644 --- a/src/main/webapp/scripts/project/menu-bar.js +++ b/src/main/webapp/scripts/project/menu-bar.js @@ -12,7 +12,7 @@ MenuBar.prototype._initializeUI = function() { var self = this; - this._createTopLevelMenuItem("Data Set", [ + this._createTopLevelMenuItem("Project", [ { "label": "Export Filtered Rows", "submenu": [ @@ -25,6 +25,10 @@ MenuBar.prototype._initializeUI = function() { "click": function() { self._doExportRows("tripleloader", "txt"); } } ] + }, + { + "label": "Export Project", + "click": function() { self._exportProject(); } } ]); this._createTopLevelMenuItem("Schemas", [ @@ -161,6 +165,27 @@ MenuBar.prototype._doExportRows = function(format, ext) { document.body.removeChild(form); }; +MenuBar.prototype._exportProject = function() { + var name = theProject.metadata.name.replace(/\W/g, ' ').replace(/\s+/g, '-'); + var form = document.createElement("form"); + $(form) + .css("display", "none") + .attr("method", "post") + .attr("action", "/command/export-project/" + name + ".gridworks.tar") + .attr("target", "gridworks-export"); + $('') + .attr("name", "project") + .attr("value", theProject.id) + .appendTo(form); + + document.body.appendChild(form); + + window.open("about:blank", "gridworks-export"); + form.submit(); + + document.body.removeChild(form); +}; + MenuBar.prototype._doAutoSchemaAlignment = function() { //SchemaAlignment.autoAlign(); }; diff --git a/src/main/webapp/styles/index.css b/src/main/webapp/styles/index.css index a3fb98c3f..7c8abf744 100644 --- a/src/main/webapp/styles/index.css +++ b/src/main/webapp/styles/index.css @@ -53,6 +53,11 @@ padding-bottom: 0px; } +#create-project-panel h1 { + font-size: 120%; + margin: 1em 0; +} + #projects { white-space: pre; }