From 92ecc0c0f5a00fe19784bbcec637a6217ffe3678 Mon Sep 17 00:00:00 2001 From: Stefano Mazzocchi Date: Tue, 4 May 2010 23:24:48 +0000 Subject: [PATCH] detab + dedos for java files (no functional changes) git-svn-id: http://google-refine.googlecode.com/svn/trunk@594 7d457c2a-affb-35e4-300a-418c747d4874 --- .../metaweb/gridworks/GridworksServlet.java | 466 ++++---- .../com/metaweb/gridworks/Jsonizable.java | 32 +- .../com/metaweb/gridworks/ProjectManager.java | 1002 ++++++++--------- .../metaweb/gridworks/ProjectMetadata.java | 422 +++---- .../browsing/ConjunctiveFilteredRows.java | 192 ++-- .../gridworks/browsing/DecoratedValue.java | 66 +- .../metaweb/gridworks/browsing/Engine.java | 218 ++-- .../gridworks/browsing/FilteredRows.java | 38 +- .../gridworks/browsing/RowVisitor.java | 44 +- .../facets/ExpressionNominalRowGrouper.java | 176 +-- .../facets/ExpressionNumericRowBinner.java | 250 ++-- .../gridworks/browsing/facets/Facet.java | 38 +- .../gridworks/browsing/facets/ListFacet.java | 406 +++---- .../browsing/facets/NominalFacetChoice.java | 64 +- .../browsing/facets/NumericBinIndex.java | 428 +++---- .../gridworks/browsing/facets/RangeFacet.java | 418 +++---- .../browsing/facets/TextSearchFacet.java | 210 ++-- .../filters/ExpressionEqualRowFilter.java | 202 ++-- .../ExpressionNumberComparisonRowFilter.java | 188 ++-- .../ExpressionStringComparisonRowFilter.java | 102 +- .../gridworks/browsing/filters/RowFilter.java | 24 +- .../metaweb/gridworks/commands/Command.java | 446 ++++---- .../commands/EngineDependentCommand.java | 112 +- .../commands/edit/AddColumnCommand.java | 68 +- .../commands/edit/AnnotateOneRowCommand.java | 240 ++-- .../commands/edit/AnnotateRowsCommand.java | 68 +- .../commands/edit/ApplyOperationsCommand.java | 120 +- .../commands/edit/CreateProjectCommand.java | 972 ++++++++-------- .../commands/edit/DenormalizeCommand.java | 64 +- .../commands/edit/EditOneCellCommand.java | 250 ++-- .../commands/edit/ExportProjectCommand.java | 208 ++-- .../commands/edit/ExtendDataCommand.java | 64 +- .../commands/edit/ImportProjectCommand.java | 344 +++--- .../edit/JoinMultiValueCellsCommand.java | 72 +- .../commands/edit/MassEditCommand.java | 58 +- .../commands/edit/RemoveColumnCommand.java | 68 +- .../commands/edit/RemoveRowsCommand.java | 40 +- .../commands/edit/RenameColumnCommand.java | 70 +- .../commands/edit/SaveProtographCommand.java | 80 +- .../commands/edit/SplitColumnCommand.java | 108 +- .../edit/SplitMultiValueCellsCommand.java | 74 +- .../commands/edit/TextTransformCommand.java | 76 +- .../commands/edit/UndoRedoCommand.java | 80 +- .../commands/info/ComputeFacetsCommand.java | 58 +- .../commands/info/ExportRowsCommand.java | 116 +- .../info/GetAllProjectMetadataCommand.java | 94 +- .../info/GetExpressionHistoryCommand.java | 122 +- .../commands/info/GetHistoryCommand.java | 54 +- .../commands/info/GetModelsCommand.java | 88 +- .../commands/info/GetOperationsCommand.java | 100 +- .../commands/info/GetProcessesCommand.java | 54 +- .../info/GetProjectMetadataCommand.java | 56 +- .../commands/info/GetRowsCommand.java | 258 ++--- .../recon/ReconDiscardJudgmentsCommand.java | 42 +- .../recon/ReconJudgeOneCellCommand.java | 410 +++---- .../recon/ReconJudgeSimilarCellsCommand.java | 100 +- .../recon/ReconMarkNewTopicsCommand.java | 48 +- .../ReconMatchBestCandidatesCommand.java | 44 +- .../recon/ReconMatchSpecificTopicCommand.java | 60 +- .../commands/recon/ReconcileCommand.java | 56 +- .../commands/util/CancelProcessesCommand.java | 58 +- .../GetExpressionLanguageInfoCommand.java | 116 +- .../util/GuessTypesOfColumnCommand.java | 438 +++---- .../commands/util/LogExpressionCommand.java | 62 +- .../util/PreviewExpressionCommand.java | 388 +++---- .../util/PreviewExtendDataCommand.java | 316 +++--- .../util/PreviewProtographCommand.java | 136 +-- .../com/metaweb/gridworks/expr/Evaluable.java | 32 +- .../com/metaweb/gridworks/expr/HasFields.java | 26 +- .../metaweb/gridworks/expr/HasFieldsList.java | 20 +- .../gridworks/expr/JythonEvaluable.java | 6 +- .../gridworks/expr/functions/Cross.java | 106 +- .../metaweb/gridworks/expr/functions/Get.java | 210 ++-- .../gridworks/expr/functions/Length.java | 86 +- .../gridworks/expr/functions/Slice.java | 172 +-- .../gridworks/expr/functions/ToNumber.java | 78 +- .../gridworks/expr/functions/ToString.java | 88 +- .../gridworks/expr/functions/arrays/Join.java | 124 +- .../expr/functions/arrays/Reverse.java | 104 +- .../gridworks/expr/functions/arrays/Sort.java | 100 +- .../expr/functions/booleans/And.java | 60 +- .../expr/functions/booleans/Not.java | 70 +- .../gridworks/expr/functions/booleans/Or.java | 60 +- .../gridworks/expr/functions/math/Ceil.java | 60 +- .../gridworks/expr/functions/math/Floor.java | 62 +- .../gridworks/expr/functions/math/Ln.java | 60 +- .../gridworks/expr/functions/math/Log.java | 60 +- .../gridworks/expr/functions/math/Max.java | 68 +- .../gridworks/expr/functions/math/Min.java | 68 +- .../gridworks/expr/functions/math/Mod.java | 70 +- .../gridworks/expr/functions/math/Round.java | 60 +- .../gridworks/expr/functions/math/Sum.java | 118 +- .../expr/functions/strings/EndsWith.java | 68 +- .../expr/functions/strings/IndexOf.java | 68 +- .../expr/functions/strings/LastIndexOf.java | 70 +- .../expr/functions/strings/Replace.java | 88 +- .../expr/functions/strings/Split.java | 84 +- .../functions/strings/SplitByLengths.java | 96 +- .../expr/functions/strings/StartsWith.java | 66 +- .../expr/functions/strings/ToLowercase.java | 62 +- .../expr/functions/strings/ToTitlecase.java | 70 +- .../expr/functions/strings/ToUppercase.java | 62 +- .../com/metaweb/gridworks/gel/Control.java | 34 +- .../com/metaweb/gridworks/gel/Function.java | 26 +- .../com/metaweb/gridworks/gel/Parser.java | 568 +++++----- .../com/metaweb/gridworks/gel/Scanner.java | 608 +++++----- .../gridworks/gel/ast/ControlCallExpr.java | 74 +- .../gridworks/gel/ast/FieldAccessorExpr.java | 82 +- .../gridworks/gel/ast/FunctionCallExpr.java | 96 +- .../gridworks/gel/ast/LiteralExpr.java | 52 +- .../gridworks/gel/ast/OperatorCallExpr.java | 178 +-- .../gridworks/gel/ast/VariableExpr.java | 56 +- .../gridworks/gel/controls/IsBlank.java | 30 +- .../gridworks/gel/controls/IsError.java | 30 +- .../gridworks/gel/controls/IsNonBlank.java | 30 +- .../gridworks/gel/controls/IsNotNull.java | 26 +- .../gridworks/gel/controls/IsNull.java | 26 +- .../com/metaweb/gridworks/history/Change.java | 42 +- .../metaweb/gridworks/history/History.java | 538 ++++----- .../gridworks/history/HistoryEntry.java | 418 +++---- .../gridworks/history/HistoryProcess.java | 146 +-- .../gridworks/importers/ExcelImporter.java | 502 ++++----- .../metaweb/gridworks/importers/Importer.java | 28 +- .../importers/ImporterUtilities.java | 220 ++-- .../gridworks/importers/MarcImporter.java | 158 +-- .../gridworks/importers/TsvCsvImporter.java | 206 ++-- .../gridworks/importers/XmlImporter.java | 128 +-- .../gridworks/model/AbstractOperation.java | 66 +- .../com/metaweb/gridworks/model/Cell.java | 302 ++--- .../com/metaweb/gridworks/model/Column.java | 246 ++-- .../metaweb/gridworks/model/ColumnGroup.java | 152 +-- .../metaweb/gridworks/model/ColumnModel.java | 444 ++++---- .../com/metaweb/gridworks/model/Project.java | 756 ++++++------- .../com/metaweb/gridworks/model/Recon.java | 660 +++++------ .../gridworks/model/ReconCandidate.java | 264 ++--- .../java/com/metaweb/gridworks/model/Row.java | 404 +++---- .../gridworks/model/changes/CellChange.java | 164 +-- .../model/changes/ColumnSplitChange.java | 136 +-- .../model/changes/DataExtensionChange.java | 294 ++--- .../model/changes/MassCellChange.java | 254 ++--- .../gridworks/model/changes/MassChange.java | 164 +-- .../gridworks/model/changes/ReconChange.java | 346 +++--- .../model/recon/DataExtensionReconConfig.java | 126 +-- .../model/recon/GuidBasedReconConfig.java | 350 +++--- .../model/recon/HeuristicReconConfig.java | 20 +- .../model/recon/IdBasedReconConfig.java | 360 +++--- .../model/recon/KeyBasedReconConfig.java | 388 +++---- .../gridworks/model/recon/ReconJob.java | 10 +- .../operations/ColumnAdditionOperation.java | 348 +++--- .../operations/ColumnRemovalOperation.java | 116 +- .../operations/ColumnRenameOperation.java | 124 +- .../operations/ColumnSplitOperation.java | 562 ++++----- .../operations/DenormalizeOperation.java | 150 +-- .../EngineDependentMassCellOperation.java | 118 +- .../operations/EngineDependentOperation.java | 76 +- .../operations/ExtendDataOperation.java | 526 ++++----- .../operations/MassEditOperation.java | 460 ++++---- .../MultiValuedCellJoinOperation.java | 258 ++--- .../MultiValuedCellSplitOperation.java | 292 ++--- .../metaweb/gridworks/operations/OnError.java | 6 +- .../ReconDiscardJudgmentsOperation.java | 226 ++-- .../ReconJudgeSimilarCellsOperation.java | 470 ++++---- .../ReconMarkNewTopicsOperation.java | 280 ++--- .../ReconMatchBestCandidatesOperation.java | 242 ++-- .../ReconMatchSpecificTopicOperation.java | 304 ++--- .../gridworks/operations/ReconOperation.java | 570 +++++----- .../operations/RowFlagOperation.java | 182 +-- .../operations/RowRemovalOperation.java | 162 +-- .../operations/RowStarOperation.java | 182 +-- .../operations/SaveProtographOperation.java | 234 ++-- .../operations/TextTransformOperation.java | 352 +++--- .../gridworks/process/LongRunningProcess.java | 142 +-- .../metaweb/gridworks/process/Process.java | 32 +- .../gridworks/process/ProcessManager.java | 210 ++-- .../process/QuickHistoryEntryProcess.java | 132 +-- .../gridworks/protograph/AnonymousNode.java | 90 +- .../gridworks/protograph/CellKeyNode.java | 58 +- .../gridworks/protograph/CellNode.java | 22 +- .../gridworks/protograph/CellTopicNode.java | 116 +- .../gridworks/protograph/CellValueNode.java | 68 +- .../protograph/FreebaseProperty.java | 18 +- .../gridworks/protograph/FreebaseTopic.java | 56 +- .../protograph/FreebaseTopicNode.java | 92 +- .../gridworks/protograph/FreebaseType.java | 72 +- .../metaweb/gridworks/protograph/Link.java | 72 +- .../metaweb/gridworks/protograph/Node.java | 12 +- .../gridworks/protograph/NodeWithLinks.java | 18 +- .../gridworks/protograph/Protograph.java | 300 ++--- .../gridworks/protograph/ValueNode.java | 58 +- .../MqlreadLikeTransposedNodeFactory.java | 632 +++++------ .../protograph/transpose/TransposedNode.java | 8 +- .../transpose/TransposedNodeFactory.java | 70 +- .../protograph/transpose/Transposer.java | 340 +++--- .../TripleLoaderTransposedNodeFactory.java | 778 ++++++------- .../util/FreebaseDataExtensionJob.java | 76 +- .../metaweb/gridworks/util/IndentWriter.java | 154 +-- .../com/metaweb/gridworks/util/JSObject.java | 266 ++--- .../gridworks/util/ParsingUtilities.java | 240 ++-- 198 files changed, 17612 insertions(+), 17612 deletions(-) diff --git a/src/main/java/com/metaweb/gridworks/GridworksServlet.java b/src/main/java/com/metaweb/gridworks/GridworksServlet.java index feb387922..a66391871 100644 --- a/src/main/java/com/metaweb/gridworks/GridworksServlet.java +++ b/src/main/java/com/metaweb/gridworks/GridworksServlet.java @@ -1,233 +1,233 @@ -package com.metaweb.gridworks; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.commands.auth.AuthorizeCommand; -import com.metaweb.gridworks.commands.auth.CheckAuthorizationCommand; -import com.metaweb.gridworks.commands.auth.DeAuthorizeCommand; -import com.metaweb.gridworks.commands.auth.GetUserBadgesCommand; -import com.metaweb.gridworks.commands.edit.AddColumnCommand; -import com.metaweb.gridworks.commands.edit.AnnotateOneRowCommand; -import com.metaweb.gridworks.commands.edit.AnnotateRowsCommand; -import com.metaweb.gridworks.commands.edit.ApplyOperationsCommand; -import com.metaweb.gridworks.commands.edit.CreateProjectCommand; -import com.metaweb.gridworks.commands.edit.DeleteProjectCommand; -import com.metaweb.gridworks.commands.edit.DenormalizeCommand; -import com.metaweb.gridworks.commands.edit.EditOneCellCommand; -import com.metaweb.gridworks.commands.edit.ExportProjectCommand; -import com.metaweb.gridworks.commands.edit.ExtendDataCommand; -import com.metaweb.gridworks.commands.edit.ImportProjectCommand; -import com.metaweb.gridworks.commands.edit.JoinMultiValueCellsCommand; -import com.metaweb.gridworks.commands.edit.MQLReadCommand; -import com.metaweb.gridworks.commands.edit.MQLWriteCommand; -import com.metaweb.gridworks.commands.edit.MassEditCommand; -import com.metaweb.gridworks.commands.edit.RemoveColumnCommand; -import com.metaweb.gridworks.commands.edit.RemoveRowsCommand; -import com.metaweb.gridworks.commands.edit.RenameColumnCommand; -import com.metaweb.gridworks.commands.edit.SaveProtographCommand; -import com.metaweb.gridworks.commands.edit.SplitColumnCommand; -import com.metaweb.gridworks.commands.edit.SplitMultiValueCellsCommand; -import com.metaweb.gridworks.commands.edit.TextTransformCommand; -import com.metaweb.gridworks.commands.edit.UndoRedoCommand; -import com.metaweb.gridworks.commands.edit.UploadDataCommand; -import com.metaweb.gridworks.commands.info.ComputeClustersCommand; -import com.metaweb.gridworks.commands.info.ComputeFacetsCommand; -import com.metaweb.gridworks.commands.info.ExportRowsCommand; -import com.metaweb.gridworks.commands.info.GetAllProjectMetadataCommand; -import com.metaweb.gridworks.commands.info.GetColumnsInfoCommand; -import com.metaweb.gridworks.commands.info.GetExpressionHistoryCommand; -import com.metaweb.gridworks.commands.info.GetHistoryCommand; -import com.metaweb.gridworks.commands.info.GetModelsCommand; -import com.metaweb.gridworks.commands.info.GetOperationsCommand; -import com.metaweb.gridworks.commands.info.GetProcessesCommand; -import com.metaweb.gridworks.commands.info.GetProjectMetadataCommand; -import com.metaweb.gridworks.commands.info.GetRowsCommand; -import com.metaweb.gridworks.commands.info.GetScatterplotCommand; -import com.metaweb.gridworks.commands.recon.ReconDiscardJudgmentsCommand; -import com.metaweb.gridworks.commands.recon.ReconJudgeOneCellCommand; -import com.metaweb.gridworks.commands.recon.ReconJudgeSimilarCellsCommand; -import com.metaweb.gridworks.commands.recon.ReconMarkNewTopicsCommand; -import com.metaweb.gridworks.commands.recon.ReconMatchBestCandidatesCommand; -import com.metaweb.gridworks.commands.recon.ReconMatchSpecificTopicCommand; -import com.metaweb.gridworks.commands.recon.ReconcileCommand; -import com.metaweb.gridworks.commands.util.CancelProcessesCommand; -import com.metaweb.gridworks.commands.util.GetExpressionLanguageInfoCommand; -import com.metaweb.gridworks.commands.util.GuessTypesOfColumnCommand; -import com.metaweb.gridworks.commands.util.LogExpressionCommand; -import com.metaweb.gridworks.commands.util.PreviewExpressionCommand; -import com.metaweb.gridworks.commands.util.PreviewExtendDataCommand; -import com.metaweb.gridworks.commands.util.PreviewProtographCommand; - -public class GridworksServlet extends HttpServlet { - - private static final long serialVersionUID = 2386057901503517403L; - - static final protected Map _commands = new HashMap(); - - // timer for periodically saving projects - static protected Timer _timer; - - final Logger logger = LoggerFactory.getLogger("servlet"); - - static { - _commands.put("create-project-from-upload", new CreateProjectCommand()); - _commands.put("import-project", new ImportProjectCommand()); - _commands.put("export-project", new ExportProjectCommand()); - _commands.put("export-rows", new ExportRowsCommand()); - - _commands.put("get-project-metadata", new GetProjectMetadataCommand()); - _commands.put("get-all-project-metadata", new GetAllProjectMetadataCommand()); - - _commands.put("delete-project", new DeleteProjectCommand()); - - _commands.put("get-models", new GetModelsCommand()); - _commands.put("get-rows", new GetRowsCommand()); - _commands.put("get-processes", new GetProcessesCommand()); - _commands.put("get-history", new GetHistoryCommand()); - _commands.put("get-operations", new GetOperationsCommand()); - _commands.put("get-columns-info", new GetColumnsInfoCommand()); - _commands.put("get-scatterplot", new GetScatterplotCommand()); - - _commands.put("undo-redo", new UndoRedoCommand()); - _commands.put("apply-operations", new ApplyOperationsCommand()); - _commands.put("cancel-processes", new CancelProcessesCommand()); - - _commands.put("compute-facets", new ComputeFacetsCommand()); - _commands.put("compute-clusters", new ComputeClustersCommand()); - - _commands.put("edit-one-cell", new EditOneCellCommand()); - _commands.put("text-transform", new TextTransformCommand()); - _commands.put("mass-edit", new MassEditCommand()); - _commands.put("join-multi-value-cells", new JoinMultiValueCellsCommand()); - _commands.put("split-multi-value-cells", new SplitMultiValueCellsCommand()); - - _commands.put("add-column", new AddColumnCommand()); - _commands.put("remove-column", new RemoveColumnCommand()); - _commands.put("rename-column", new RenameColumnCommand()); - _commands.put("split-column", new SplitColumnCommand()); - _commands.put("extend-data", new ExtendDataCommand()); - - _commands.put("denormalize", new DenormalizeCommand()); - - _commands.put("reconcile", new ReconcileCommand()); - _commands.put("recon-match-best-candidates", new ReconMatchBestCandidatesCommand()); - _commands.put("recon-mark-new-topics", new ReconMarkNewTopicsCommand()); - _commands.put("recon-discard-judgments", new ReconDiscardJudgmentsCommand()); - _commands.put("recon-match-specific-topic-to-cells", new ReconMatchSpecificTopicCommand()); - _commands.put("recon-judge-one-cell", new ReconJudgeOneCellCommand()); - _commands.put("recon-judge-similar-cells", new ReconJudgeSimilarCellsCommand()); - - _commands.put("annotate-one-row", new AnnotateOneRowCommand()); - _commands.put("annotate-rows", new AnnotateRowsCommand()); - _commands.put("remove-rows", new RemoveRowsCommand()); - - _commands.put("save-protograph", new SaveProtographCommand()); - - _commands.put("get-expression-language-info", new GetExpressionLanguageInfoCommand()); - _commands.put("get-expression-history", new GetExpressionHistoryCommand()); - _commands.put("log-expression", new LogExpressionCommand()); - - _commands.put("preview-expression", new PreviewExpressionCommand()); - _commands.put("preview-extend-data", new PreviewExtendDataCommand()); - _commands.put("preview-protograph", new PreviewProtographCommand()); - - _commands.put("guess-types-of-column", new GuessTypesOfColumnCommand()); - - _commands.put("check-authorization", new CheckAuthorizationCommand()); - _commands.put("authorize", new AuthorizeCommand()); - _commands.put("deauthorize", new DeAuthorizeCommand()); - _commands.put("user-badges", new GetUserBadgesCommand()); - - _commands.put("upload-data", new UploadDataCommand()); - _commands.put("mqlread", new MQLReadCommand()); - _commands.put("mqlwrite", new MQLWriteCommand()); - } - - @Override - public void init() throws ServletException { - super.init(); - logger.trace("> initialize"); - - ProjectManager.initialize(); - - if (_timer == null) { - _timer = new Timer(); - } - - long period = 1000 * 60 * 5; // 5 minutes - _timer.scheduleAtFixedRate(new TimerTask() { - @Override - public void run() { - ProjectManager.singleton.save(false); // quick, potentially incomplete save - } - }, period, period); - - logger.trace("< initialize"); - } - - @Override - public void destroy() { - logger.trace("> destroy"); - - // cancel automatic periodic saving and force a complete save. - if (_timer != null) { - _timer.cancel(); - _timer = null; - } - if (ProjectManager.singleton != null) { - ProjectManager.singleton.save(true); // complete save - ProjectManager.singleton = null; - } - - super.destroy(); - - logger.trace("< destroy"); - } - - protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - String commandName = getCommandName(request); - Command command = _commands.get(commandName); - if (command != null) { - logger.trace("> GET {}", commandName); - command.doGet(request, response); - logger.trace("< GET {}", commandName); - } else { - response.sendError(404); - } - } - - protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - String commandName = getCommandName(request); - Command command = _commands.get(commandName); - if (command != null) { - logger.trace("> POST {}", commandName); - command.doPost(request, response); - logger.trace("< POST {}", commandName); - } else { - response.sendError(404); - } - } - - protected String getCommandName(HttpServletRequest request) { - // Remove extraneous path segments that might be there for other purposes, - // e.g., for /export-rows/filename.ext, export-rows is the command while - // filename.ext is only for the browser to prompt a convenient filename. - String commandName = request.getPathInfo().substring(1); - int slash = commandName.indexOf('/'); - return slash > 0 ? commandName.substring(0, slash) : commandName; - } -} - +package com.metaweb.gridworks; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.commands.auth.AuthorizeCommand; +import com.metaweb.gridworks.commands.auth.CheckAuthorizationCommand; +import com.metaweb.gridworks.commands.auth.DeAuthorizeCommand; +import com.metaweb.gridworks.commands.auth.GetUserBadgesCommand; +import com.metaweb.gridworks.commands.edit.AddColumnCommand; +import com.metaweb.gridworks.commands.edit.AnnotateOneRowCommand; +import com.metaweb.gridworks.commands.edit.AnnotateRowsCommand; +import com.metaweb.gridworks.commands.edit.ApplyOperationsCommand; +import com.metaweb.gridworks.commands.edit.CreateProjectCommand; +import com.metaweb.gridworks.commands.edit.DeleteProjectCommand; +import com.metaweb.gridworks.commands.edit.DenormalizeCommand; +import com.metaweb.gridworks.commands.edit.EditOneCellCommand; +import com.metaweb.gridworks.commands.edit.ExportProjectCommand; +import com.metaweb.gridworks.commands.edit.ExtendDataCommand; +import com.metaweb.gridworks.commands.edit.ImportProjectCommand; +import com.metaweb.gridworks.commands.edit.JoinMultiValueCellsCommand; +import com.metaweb.gridworks.commands.edit.MQLReadCommand; +import com.metaweb.gridworks.commands.edit.MQLWriteCommand; +import com.metaweb.gridworks.commands.edit.MassEditCommand; +import com.metaweb.gridworks.commands.edit.RemoveColumnCommand; +import com.metaweb.gridworks.commands.edit.RemoveRowsCommand; +import com.metaweb.gridworks.commands.edit.RenameColumnCommand; +import com.metaweb.gridworks.commands.edit.SaveProtographCommand; +import com.metaweb.gridworks.commands.edit.SplitColumnCommand; +import com.metaweb.gridworks.commands.edit.SplitMultiValueCellsCommand; +import com.metaweb.gridworks.commands.edit.TextTransformCommand; +import com.metaweb.gridworks.commands.edit.UndoRedoCommand; +import com.metaweb.gridworks.commands.edit.UploadDataCommand; +import com.metaweb.gridworks.commands.info.ComputeClustersCommand; +import com.metaweb.gridworks.commands.info.ComputeFacetsCommand; +import com.metaweb.gridworks.commands.info.ExportRowsCommand; +import com.metaweb.gridworks.commands.info.GetAllProjectMetadataCommand; +import com.metaweb.gridworks.commands.info.GetColumnsInfoCommand; +import com.metaweb.gridworks.commands.info.GetExpressionHistoryCommand; +import com.metaweb.gridworks.commands.info.GetHistoryCommand; +import com.metaweb.gridworks.commands.info.GetModelsCommand; +import com.metaweb.gridworks.commands.info.GetOperationsCommand; +import com.metaweb.gridworks.commands.info.GetProcessesCommand; +import com.metaweb.gridworks.commands.info.GetProjectMetadataCommand; +import com.metaweb.gridworks.commands.info.GetRowsCommand; +import com.metaweb.gridworks.commands.info.GetScatterplotCommand; +import com.metaweb.gridworks.commands.recon.ReconDiscardJudgmentsCommand; +import com.metaweb.gridworks.commands.recon.ReconJudgeOneCellCommand; +import com.metaweb.gridworks.commands.recon.ReconJudgeSimilarCellsCommand; +import com.metaweb.gridworks.commands.recon.ReconMarkNewTopicsCommand; +import com.metaweb.gridworks.commands.recon.ReconMatchBestCandidatesCommand; +import com.metaweb.gridworks.commands.recon.ReconMatchSpecificTopicCommand; +import com.metaweb.gridworks.commands.recon.ReconcileCommand; +import com.metaweb.gridworks.commands.util.CancelProcessesCommand; +import com.metaweb.gridworks.commands.util.GetExpressionLanguageInfoCommand; +import com.metaweb.gridworks.commands.util.GuessTypesOfColumnCommand; +import com.metaweb.gridworks.commands.util.LogExpressionCommand; +import com.metaweb.gridworks.commands.util.PreviewExpressionCommand; +import com.metaweb.gridworks.commands.util.PreviewExtendDataCommand; +import com.metaweb.gridworks.commands.util.PreviewProtographCommand; + +public class GridworksServlet extends HttpServlet { + + private static final long serialVersionUID = 2386057901503517403L; + + static final protected Map _commands = new HashMap(); + + // timer for periodically saving projects + static protected Timer _timer; + + final Logger logger = LoggerFactory.getLogger("servlet"); + + static { + _commands.put("create-project-from-upload", new CreateProjectCommand()); + _commands.put("import-project", new ImportProjectCommand()); + _commands.put("export-project", new ExportProjectCommand()); + _commands.put("export-rows", new ExportRowsCommand()); + + _commands.put("get-project-metadata", new GetProjectMetadataCommand()); + _commands.put("get-all-project-metadata", new GetAllProjectMetadataCommand()); + + _commands.put("delete-project", new DeleteProjectCommand()); + + _commands.put("get-models", new GetModelsCommand()); + _commands.put("get-rows", new GetRowsCommand()); + _commands.put("get-processes", new GetProcessesCommand()); + _commands.put("get-history", new GetHistoryCommand()); + _commands.put("get-operations", new GetOperationsCommand()); + _commands.put("get-columns-info", new GetColumnsInfoCommand()); + _commands.put("get-scatterplot", new GetScatterplotCommand()); + + _commands.put("undo-redo", new UndoRedoCommand()); + _commands.put("apply-operations", new ApplyOperationsCommand()); + _commands.put("cancel-processes", new CancelProcessesCommand()); + + _commands.put("compute-facets", new ComputeFacetsCommand()); + _commands.put("compute-clusters", new ComputeClustersCommand()); + + _commands.put("edit-one-cell", new EditOneCellCommand()); + _commands.put("text-transform", new TextTransformCommand()); + _commands.put("mass-edit", new MassEditCommand()); + _commands.put("join-multi-value-cells", new JoinMultiValueCellsCommand()); + _commands.put("split-multi-value-cells", new SplitMultiValueCellsCommand()); + + _commands.put("add-column", new AddColumnCommand()); + _commands.put("remove-column", new RemoveColumnCommand()); + _commands.put("rename-column", new RenameColumnCommand()); + _commands.put("split-column", new SplitColumnCommand()); + _commands.put("extend-data", new ExtendDataCommand()); + + _commands.put("denormalize", new DenormalizeCommand()); + + _commands.put("reconcile", new ReconcileCommand()); + _commands.put("recon-match-best-candidates", new ReconMatchBestCandidatesCommand()); + _commands.put("recon-mark-new-topics", new ReconMarkNewTopicsCommand()); + _commands.put("recon-discard-judgments", new ReconDiscardJudgmentsCommand()); + _commands.put("recon-match-specific-topic-to-cells", new ReconMatchSpecificTopicCommand()); + _commands.put("recon-judge-one-cell", new ReconJudgeOneCellCommand()); + _commands.put("recon-judge-similar-cells", new ReconJudgeSimilarCellsCommand()); + + _commands.put("annotate-one-row", new AnnotateOneRowCommand()); + _commands.put("annotate-rows", new AnnotateRowsCommand()); + _commands.put("remove-rows", new RemoveRowsCommand()); + + _commands.put("save-protograph", new SaveProtographCommand()); + + _commands.put("get-expression-language-info", new GetExpressionLanguageInfoCommand()); + _commands.put("get-expression-history", new GetExpressionHistoryCommand()); + _commands.put("log-expression", new LogExpressionCommand()); + + _commands.put("preview-expression", new PreviewExpressionCommand()); + _commands.put("preview-extend-data", new PreviewExtendDataCommand()); + _commands.put("preview-protograph", new PreviewProtographCommand()); + + _commands.put("guess-types-of-column", new GuessTypesOfColumnCommand()); + + _commands.put("check-authorization", new CheckAuthorizationCommand()); + _commands.put("authorize", new AuthorizeCommand()); + _commands.put("deauthorize", new DeAuthorizeCommand()); + _commands.put("user-badges", new GetUserBadgesCommand()); + + _commands.put("upload-data", new UploadDataCommand()); + _commands.put("mqlread", new MQLReadCommand()); + _commands.put("mqlwrite", new MQLWriteCommand()); + } + + @Override + public void init() throws ServletException { + super.init(); + logger.trace("> initialize"); + + ProjectManager.initialize(); + + if (_timer == null) { + _timer = new Timer(); + } + + long period = 1000 * 60 * 5; // 5 minutes + _timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + ProjectManager.singleton.save(false); // quick, potentially incomplete save + } + }, period, period); + + logger.trace("< initialize"); + } + + @Override + public void destroy() { + logger.trace("> destroy"); + + // cancel automatic periodic saving and force a complete save. + if (_timer != null) { + _timer.cancel(); + _timer = null; + } + if (ProjectManager.singleton != null) { + ProjectManager.singleton.save(true); // complete save + ProjectManager.singleton = null; + } + + super.destroy(); + + logger.trace("< destroy"); + } + + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + String commandName = getCommandName(request); + Command command = _commands.get(commandName); + if (command != null) { + logger.trace("> GET {}", commandName); + command.doGet(request, response); + logger.trace("< GET {}", commandName); + } else { + response.sendError(404); + } + } + + protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + String commandName = getCommandName(request); + Command command = _commands.get(commandName); + if (command != null) { + logger.trace("> POST {}", commandName); + command.doPost(request, response); + logger.trace("< POST {}", commandName); + } else { + response.sendError(404); + } + } + + protected String getCommandName(HttpServletRequest request) { + // Remove extraneous path segments that might be there for other purposes, + // e.g., for /export-rows/filename.ext, export-rows is the command while + // filename.ext is only for the browser to prompt a convenient filename. + String commandName = request.getPathInfo().substring(1); + int slash = commandName.indexOf('/'); + return slash > 0 ? commandName.substring(0, slash) : commandName; + } +} + diff --git a/src/main/java/com/metaweb/gridworks/Jsonizable.java b/src/main/java/com/metaweb/gridworks/Jsonizable.java index b11931f9a..295220537 100644 --- a/src/main/java/com/metaweb/gridworks/Jsonizable.java +++ b/src/main/java/com/metaweb/gridworks/Jsonizable.java @@ -1,16 +1,16 @@ -package com.metaweb.gridworks; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -/** - * Interface for streaming out JSON, either into HTTP responses or - * serialization files. - * - * @author dfhuynh - */ -public interface Jsonizable { - public void write(JSONWriter writer, Properties options) throws JSONException; -} +package com.metaweb.gridworks; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +/** + * Interface for streaming out JSON, either into HTTP responses or + * serialization files. + * + * @author dfhuynh + */ +public interface Jsonizable { + public void write(JSONWriter writer, Properties options) throws JSONException; +} diff --git a/src/main/java/com/metaweb/gridworks/ProjectManager.java b/src/main/java/com/metaweb/gridworks/ProjectManager.java index a64c63c82..78fed1b9e 100644 --- a/src/main/java/com/metaweb/gridworks/ProjectManager.java +++ b/src/main/java/com/metaweb/gridworks/ProjectManager.java @@ -1,501 +1,501 @@ -package com.metaweb.gridworks; - -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONTokener; -import org.json.JSONWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.codeberry.jdatapath.DataPath; -import com.codeberry.jdatapath.JDataPathSystem; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.util.JSONUtilities; - -public class ProjectManager { - - // last n expressions used across all projects - static protected final int s_expressionHistoryMax = 100; - - protected File _workspaceDir; - protected Map _projectsMetadata; - protected List _expressions; - - final static Logger logger = LoggerFactory.getLogger("project_manager"); - - /** - * While each project's metadata is loaded completely at start-up, each project's raw data - * is loaded only when the project is accessed by the user. This is because project - * metadata is tiny compared to raw project data. This hash map from project ID to project - * is more like a last accessed-last out cache. - */ - transient protected Map _projects; - - /** - * What caches the joins between projects. - */ - transient protected InterProjectModel _interProjectModel = new InterProjectModel(); - - static public ProjectManager singleton; - - static public synchronized void initialize() { - if (singleton == null) { - File dir = getProjectLocation(); - logger.info("Using workspace directory: {}", dir.getAbsolutePath()); - - singleton = new ProjectManager(dir); - } - } - - static protected File getProjectLocation() { - String data_dir = Configurations.get("gridworks.data_dir"); - if (data_dir != null) { - return new File(data_dir); - } - - String os = Configurations.get("os.name").toLowerCase(); - if (os.contains("windows")) { - try { - // NOTE(SM): finding the "local data app" in windows from java is actually a PITA - // see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java - // so we're using a library that uses JNI to ask directly the win32 APIs, - // it's not elegant but it's the safest bet. - - DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks"); - File data = new File(localDataPath.getPath()); - data.mkdirs(); - return data; - } catch (Error e) { - /* - * The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll - * we include is compiled for 32-bit. In this case, we just have to dig up - * environment variables and try our best to find a user-specific path. - */ - - logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables"); - - File parentDir = null; - { - String appData = System.getenv("APPDATA"); - if (appData != null && appData.length() > 0) { - // e.g., C:\Users\[userid]\AppData\Roaming - parentDir = new File(appData); - } else { - String userProfile = System.getenv("USERPROFILE"); - if (userProfile != null && userProfile.length() > 0) { - // e.g., C:\Users\[userid] - parentDir = new File(userProfile); - } - } - } - if (parentDir == null) { - parentDir = new File("."); - } - - File data = new File(parentDir, "Gridworks"); - data.mkdirs(); - - return data; - } - } else if (os.contains("mac os x")) { - // on macosx, use "~/Library/Application Support" - String home = System.getProperty("user.home"); - String data_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks"; - File data = new File(data_home); - data.mkdirs(); - return data; - } else { // most likely a UNIX flavor - // start with the XDG environment - // see http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html - String data_home = System.getenv("XDG_DATA_HOME"); - if (data_home == null) { // if not found, default back to ~/.local/share - String home = System.getProperty("user.home"); - if (home == null) home = "."; - data_home = home + "/.local/share"; - } - File data = new File(data_home + "/gridworks"); - data.mkdirs(); - return data; - } - } - - private ProjectManager(File dir) { - _workspaceDir = dir; - _workspaceDir.mkdirs(); - - _projectsMetadata = new HashMap(); - _expressions = new LinkedList(); - _projects = new HashMap(); - - load(); - } - - public InterProjectModel getInterProjectModel() { - return _interProjectModel; - } - - public File getWorkspaceDir() { - return _workspaceDir; - } - - static public File getProjectDir(File workspaceDir, long projectID) { - File dir = new File(workspaceDir, projectID + ".project"); - if (!dir.exists()) { - dir.mkdir(); - } - return dir; - } - - public File getProjectDir(long projectID) { - return getProjectDir(_workspaceDir, projectID); - } - - public void registerProject(Project project, ProjectMetadata projectMetadata) { - synchronized (this) { - _projects.put(project.id, project); - _projectsMetadata.put(project.id, projectMetadata); - } - } - - /** - * Import an external project that has been received as a .tar file, expanded, and - * copied into our workspace directory. - * - * @param projectID - */ - public boolean importProject(long projectID) { - synchronized (this) { - ProjectMetadata metadata = ProjectMetadata.load(getProjectDir(projectID)); - if (metadata != null) { - _projectsMetadata.put(projectID, metadata); - return true; - } else { - return false; - } - } - } - - /** - * Make sure that a project's metadata and data are saved to file. For example, - * this method is called before the project is exported to a .tar file. - * - * @param id - */ - public void ensureProjectSaved(long id) { - synchronized (this) { - File projectDir = getProjectDir(id); - - ProjectMetadata metadata = _projectsMetadata.get(id); - if (metadata != null) { - try { - metadata.save(projectDir); - } catch (Exception e) { - e.printStackTrace(); - } - } - - Project project = _projects.get(id); - if (project != null && metadata.getModified().after(project.lastSave)) { - try { - project.save(); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - } - - public ProjectMetadata getProjectMetadata(long id) { - return _projectsMetadata.get(id); - } - - public ProjectMetadata getProjectMetadata(String name) { - for (ProjectMetadata pm : _projectsMetadata.values()) { - if (pm.getName().equals(name)) { - return pm; - } - } - return null; - } - - public long getProjectID(String name) { - for (Entry entry : _projectsMetadata.entrySet()) { - if (entry.getValue().getName().equals(name)) { - return entry.getKey(); - } - } - return -1; - } - - public Map getAllProjectMetadata() { - return _projectsMetadata; - } - - public Project getProject(long id) { - synchronized (this) { - if (_projects.containsKey(id)) { - return _projects.get(id); - } else { - Project project = Project.load(getProjectDir(id), id); - - _projects.put(id, project); - - return project; - } - } - } - - public void addLatestExpression(String s) { - synchronized (this) { - _expressions.remove(s); - _expressions.add(0, s); - while (_expressions.size() > s_expressionHistoryMax) { - _expressions.remove(_expressions.size() - 1); - } - } - } - - public List getExpressions() { - return _expressions; - } - - public void save(boolean allModified) { - saveProjects(allModified); - saveWorkspace(); - } - - /** - * Save the workspace's data out to file in a safe way: save to a temporary file first - * and rename it to the real file. - */ - protected void saveWorkspace() { - synchronized (this) { - File tempFile = new File(_workspaceDir, "workspace.temp.json"); - try { - saveToFile(tempFile); - } catch (Exception e) { - e.printStackTrace(); - - logger.warn("Failed to save workspace"); - return; - } - - File file = new File(_workspaceDir, "workspace.json"); - File oldFile = new File(_workspaceDir, "workspace.old.json"); - - if (file.exists()) { - file.renameTo(oldFile); - } - - tempFile.renameTo(file); - if (oldFile.exists()) { - oldFile.delete(); - } - - logger.info("Saved workspace"); - } - } - - protected void saveToFile(File file) throws IOException, JSONException { - FileWriter writer = new FileWriter(file); - try { - JSONWriter jsonWriter = new JSONWriter(writer); - jsonWriter.object(); - jsonWriter.key("projectIDs"); - jsonWriter.array(); - for (Long id : _projectsMetadata.keySet()) { - ProjectMetadata metadata = _projectsMetadata.get(id); - if (metadata != null) { - jsonWriter.value(id); - - try { - metadata.save(getProjectDir(id)); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - jsonWriter.endArray(); - writer.write('\n'); - - jsonWriter.key("expressions"); JSONUtilities.writeStringList(jsonWriter, _expressions); - jsonWriter.endObject(); - } finally { - writer.close(); - } - } - - /** - * A utility class to prioritize projects for saving, depending on how long ago - * they have been changed but have not been saved. - */ - static protected class SaveRecord { - final Project project; - final long overdue; - - SaveRecord(Project project, long overdue) { - this.project = project; - this.overdue = overdue; - } - } - - static protected final int s_projectFlushDelay = 1000 * 60 * 60; // 1 hour - static protected final int s_quickSaveTimeout = 1000 * 30; // 30 secs - - protected void saveProjects(boolean allModified) { - List records = new ArrayList(); - Date now = new Date(); - - synchronized (this) { - for (long id : _projectsMetadata.keySet()) { - ProjectMetadata metadata = _projectsMetadata.get(id); - Project project = _projects.get(id); - - if (project != null) { - boolean hasUnsavedChanges = - metadata.getModified().getTime() > project.lastSave.getTime(); - - if (hasUnsavedChanges) { - long msecsOverdue = now.getTime() - project.lastSave.getTime(); - - records.add(new SaveRecord(project, msecsOverdue)); - - } else if (now.getTime() - project.lastSave.getTime() > s_projectFlushDelay) { - /* - * It's been a while since the project was last saved and it hasn't been - * modified. We can safely remove it from the cache to save some memory. - */ - _projects.remove(id); - } - } - } - } - - if (records.size() > 0) { - Collections.sort(records, new Comparator() { - public int compare(SaveRecord o1, SaveRecord o2) { - if (o1.overdue < o2.overdue) { - return 1; - } else if (o1.overdue > o2.overdue) { - return -1; - } else { - return 0; - } - } - }); - - logger.info(allModified ? - "Saving all modified projects ..." : - "Saving some modified projects ..." - ); - - for (int i = 0; - i < records.size() && - (allModified || (new Date().getTime() - now.getTime() < s_quickSaveTimeout)); - i++) { - - try { - records.get(i).project.save(); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - } - - public void deleteProject(Project project) { - deleteProject(project.id); - } - - public void deleteProject(long projectID) { - synchronized (this) { - if (_projectsMetadata.containsKey(projectID)) { - _projectsMetadata.remove(projectID); - } - if (_projects.containsKey(projectID)) { - _projects.remove(projectID); - } - - File dir = getProjectDir(projectID); - if (dir.exists()) { - deleteDir(dir); - } - } - - saveWorkspace(); - } - - static protected void deleteDir(File dir) { - for (File file : dir.listFiles()) { - if (file.isDirectory()) { - deleteDir(file); - } else { - file.delete(); - } - } - dir.delete(); - } - - protected void load() { - if (loadFromFile(new File(_workspaceDir, "workspace.json"))) return; - if (loadFromFile(new File(_workspaceDir, "workspace.temp.json"))) return; - if (loadFromFile(new File(_workspaceDir, "workspace.old.json"))) return; - } - - protected boolean loadFromFile(File file) { - logger.info("Loading workspace: {}", file.getAbsolutePath()); - - _projectsMetadata.clear(); - _expressions.clear(); - - if (file.exists() || file.canRead()) { - FileReader reader = null; - try { - reader = new FileReader(file); - JSONTokener tokener = new JSONTokener(reader); - JSONObject obj = (JSONObject) tokener.nextValue(); - - JSONArray a = obj.getJSONArray("projectIDs"); - int count = a.length(); - for (int i = 0; i < count; i++) { - long id = a.getLong(i); - - File projectDir = getProjectDir(id); - ProjectMetadata metadata = ProjectMetadata.load(projectDir); - - _projectsMetadata.put(id, metadata); - } - - JSONUtilities.getStringList(obj, "expressions", _expressions); - return true; - } catch (JSONException e) { - logger.warn("Error reading file", e); - } catch (IOException e) { - logger.warn("Error reading file", e); - } finally { - try { - reader.close(); - } catch (IOException e) { - logger.warn("Exception closing file",e); - } - } - } - - return false; - } -} +package com.metaweb.gridworks; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.json.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.codeberry.jdatapath.DataPath; +import com.codeberry.jdatapath.JDataPathSystem; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.util.JSONUtilities; + +public class ProjectManager { + + // last n expressions used across all projects + static protected final int s_expressionHistoryMax = 100; + + protected File _workspaceDir; + protected Map _projectsMetadata; + protected List _expressions; + + final static Logger logger = LoggerFactory.getLogger("project_manager"); + + /** + * While each project's metadata is loaded completely at start-up, each project's raw data + * is loaded only when the project is accessed by the user. This is because project + * metadata is tiny compared to raw project data. This hash map from project ID to project + * is more like a last accessed-last out cache. + */ + transient protected Map _projects; + + /** + * What caches the joins between projects. + */ + transient protected InterProjectModel _interProjectModel = new InterProjectModel(); + + static public ProjectManager singleton; + + static public synchronized void initialize() { + if (singleton == null) { + File dir = getProjectLocation(); + logger.info("Using workspace directory: {}", dir.getAbsolutePath()); + + singleton = new ProjectManager(dir); + } + } + + static protected File getProjectLocation() { + String data_dir = Configurations.get("gridworks.data_dir"); + if (data_dir != null) { + return new File(data_dir); + } + + String os = Configurations.get("os.name").toLowerCase(); + if (os.contains("windows")) { + try { + // NOTE(SM): finding the "local data app" in windows from java is actually a PITA + // see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java + // so we're using a library that uses JNI to ask directly the win32 APIs, + // it's not elegant but it's the safest bet. + + DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks"); + File data = new File(localDataPath.getPath()); + data.mkdirs(); + return data; + } catch (Error e) { + /* + * The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll + * we include is compiled for 32-bit. In this case, we just have to dig up + * environment variables and try our best to find a user-specific path. + */ + + logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables"); + + File parentDir = null; + { + String appData = System.getenv("APPDATA"); + if (appData != null && appData.length() > 0) { + // e.g., C:\Users\[userid]\AppData\Roaming + parentDir = new File(appData); + } else { + String userProfile = System.getenv("USERPROFILE"); + if (userProfile != null && userProfile.length() > 0) { + // e.g., C:\Users\[userid] + parentDir = new File(userProfile); + } + } + } + if (parentDir == null) { + parentDir = new File("."); + } + + File data = new File(parentDir, "Gridworks"); + data.mkdirs(); + + return data; + } + } else if (os.contains("mac os x")) { + // on macosx, use "~/Library/Application Support" + String home = System.getProperty("user.home"); + String data_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks"; + File data = new File(data_home); + data.mkdirs(); + return data; + } else { // most likely a UNIX flavor + // start with the XDG environment + // see http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html + String data_home = System.getenv("XDG_DATA_HOME"); + if (data_home == null) { // if not found, default back to ~/.local/share + String home = System.getProperty("user.home"); + if (home == null) home = "."; + data_home = home + "/.local/share"; + } + File data = new File(data_home + "/gridworks"); + data.mkdirs(); + return data; + } + } + + private ProjectManager(File dir) { + _workspaceDir = dir; + _workspaceDir.mkdirs(); + + _projectsMetadata = new HashMap(); + _expressions = new LinkedList(); + _projects = new HashMap(); + + load(); + } + + public InterProjectModel getInterProjectModel() { + return _interProjectModel; + } + + public File getWorkspaceDir() { + return _workspaceDir; + } + + static public File getProjectDir(File workspaceDir, long projectID) { + File dir = new File(workspaceDir, projectID + ".project"); + if (!dir.exists()) { + dir.mkdir(); + } + return dir; + } + + public File getProjectDir(long projectID) { + return getProjectDir(_workspaceDir, projectID); + } + + public void registerProject(Project project, ProjectMetadata projectMetadata) { + synchronized (this) { + _projects.put(project.id, project); + _projectsMetadata.put(project.id, projectMetadata); + } + } + + /** + * Import an external project that has been received as a .tar file, expanded, and + * copied into our workspace directory. + * + * @param projectID + */ + public boolean importProject(long projectID) { + synchronized (this) { + ProjectMetadata metadata = ProjectMetadata.load(getProjectDir(projectID)); + if (metadata != null) { + _projectsMetadata.put(projectID, metadata); + return true; + } else { + return false; + } + } + } + + /** + * Make sure that a project's metadata and data are saved to file. For example, + * this method is called before the project is exported to a .tar file. + * + * @param id + */ + public void ensureProjectSaved(long id) { + synchronized (this) { + File projectDir = getProjectDir(id); + + ProjectMetadata metadata = _projectsMetadata.get(id); + if (metadata != null) { + try { + metadata.save(projectDir); + } catch (Exception e) { + e.printStackTrace(); + } + } + + Project project = _projects.get(id); + if (project != null && metadata.getModified().after(project.lastSave)) { + try { + project.save(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + + public ProjectMetadata getProjectMetadata(long id) { + return _projectsMetadata.get(id); + } + + public ProjectMetadata getProjectMetadata(String name) { + for (ProjectMetadata pm : _projectsMetadata.values()) { + if (pm.getName().equals(name)) { + return pm; + } + } + return null; + } + + public long getProjectID(String name) { + for (Entry entry : _projectsMetadata.entrySet()) { + if (entry.getValue().getName().equals(name)) { + return entry.getKey(); + } + } + return -1; + } + + public Map getAllProjectMetadata() { + return _projectsMetadata; + } + + public Project getProject(long id) { + synchronized (this) { + if (_projects.containsKey(id)) { + return _projects.get(id); + } else { + Project project = Project.load(getProjectDir(id), id); + + _projects.put(id, project); + + return project; + } + } + } + + public void addLatestExpression(String s) { + synchronized (this) { + _expressions.remove(s); + _expressions.add(0, s); + while (_expressions.size() > s_expressionHistoryMax) { + _expressions.remove(_expressions.size() - 1); + } + } + } + + public List getExpressions() { + return _expressions; + } + + public void save(boolean allModified) { + saveProjects(allModified); + saveWorkspace(); + } + + /** + * Save the workspace's data out to file in a safe way: save to a temporary file first + * and rename it to the real file. + */ + protected void saveWorkspace() { + synchronized (this) { + File tempFile = new File(_workspaceDir, "workspace.temp.json"); + try { + saveToFile(tempFile); + } catch (Exception e) { + e.printStackTrace(); + + logger.warn("Failed to save workspace"); + return; + } + + File file = new File(_workspaceDir, "workspace.json"); + File oldFile = new File(_workspaceDir, "workspace.old.json"); + + if (file.exists()) { + file.renameTo(oldFile); + } + + tempFile.renameTo(file); + if (oldFile.exists()) { + oldFile.delete(); + } + + logger.info("Saved workspace"); + } + } + + protected void saveToFile(File file) throws IOException, JSONException { + FileWriter writer = new FileWriter(file); + try { + JSONWriter jsonWriter = new JSONWriter(writer); + jsonWriter.object(); + jsonWriter.key("projectIDs"); + jsonWriter.array(); + for (Long id : _projectsMetadata.keySet()) { + ProjectMetadata metadata = _projectsMetadata.get(id); + if (metadata != null) { + jsonWriter.value(id); + + try { + metadata.save(getProjectDir(id)); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + jsonWriter.endArray(); + writer.write('\n'); + + jsonWriter.key("expressions"); JSONUtilities.writeStringList(jsonWriter, _expressions); + jsonWriter.endObject(); + } finally { + writer.close(); + } + } + + /** + * A utility class to prioritize projects for saving, depending on how long ago + * they have been changed but have not been saved. + */ + static protected class SaveRecord { + final Project project; + final long overdue; + + SaveRecord(Project project, long overdue) { + this.project = project; + this.overdue = overdue; + } + } + + static protected final int s_projectFlushDelay = 1000 * 60 * 60; // 1 hour + static protected final int s_quickSaveTimeout = 1000 * 30; // 30 secs + + protected void saveProjects(boolean allModified) { + List records = new ArrayList(); + Date now = new Date(); + + synchronized (this) { + for (long id : _projectsMetadata.keySet()) { + ProjectMetadata metadata = _projectsMetadata.get(id); + Project project = _projects.get(id); + + if (project != null) { + boolean hasUnsavedChanges = + metadata.getModified().getTime() > project.lastSave.getTime(); + + if (hasUnsavedChanges) { + long msecsOverdue = now.getTime() - project.lastSave.getTime(); + + records.add(new SaveRecord(project, msecsOverdue)); + + } else if (now.getTime() - project.lastSave.getTime() > s_projectFlushDelay) { + /* + * It's been a while since the project was last saved and it hasn't been + * modified. We can safely remove it from the cache to save some memory. + */ + _projects.remove(id); + } + } + } + } + + if (records.size() > 0) { + Collections.sort(records, new Comparator() { + public int compare(SaveRecord o1, SaveRecord o2) { + if (o1.overdue < o2.overdue) { + return 1; + } else if (o1.overdue > o2.overdue) { + return -1; + } else { + return 0; + } + } + }); + + logger.info(allModified ? + "Saving all modified projects ..." : + "Saving some modified projects ..." + ); + + for (int i = 0; + i < records.size() && + (allModified || (new Date().getTime() - now.getTime() < s_quickSaveTimeout)); + i++) { + + try { + records.get(i).project.save(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + + public void deleteProject(Project project) { + deleteProject(project.id); + } + + public void deleteProject(long projectID) { + synchronized (this) { + if (_projectsMetadata.containsKey(projectID)) { + _projectsMetadata.remove(projectID); + } + if (_projects.containsKey(projectID)) { + _projects.remove(projectID); + } + + File dir = getProjectDir(projectID); + if (dir.exists()) { + deleteDir(dir); + } + } + + saveWorkspace(); + } + + static protected void deleteDir(File dir) { + for (File file : dir.listFiles()) { + if (file.isDirectory()) { + deleteDir(file); + } else { + file.delete(); + } + } + dir.delete(); + } + + protected void load() { + if (loadFromFile(new File(_workspaceDir, "workspace.json"))) return; + if (loadFromFile(new File(_workspaceDir, "workspace.temp.json"))) return; + if (loadFromFile(new File(_workspaceDir, "workspace.old.json"))) return; + } + + protected boolean loadFromFile(File file) { + logger.info("Loading workspace: {}", file.getAbsolutePath()); + + _projectsMetadata.clear(); + _expressions.clear(); + + if (file.exists() || file.canRead()) { + FileReader reader = null; + try { + reader = new FileReader(file); + JSONTokener tokener = new JSONTokener(reader); + JSONObject obj = (JSONObject) tokener.nextValue(); + + JSONArray a = obj.getJSONArray("projectIDs"); + int count = a.length(); + for (int i = 0; i < count; i++) { + long id = a.getLong(i); + + File projectDir = getProjectDir(id); + ProjectMetadata metadata = ProjectMetadata.load(projectDir); + + _projectsMetadata.put(id, metadata); + } + + JSONUtilities.getStringList(obj, "expressions", _expressions); + return true; + } catch (JSONException e) { + logger.warn("Error reading file", e); + } catch (IOException e) { + logger.warn("Error reading file", e); + } finally { + try { + reader.close(); + } catch (IOException e) { + logger.warn("Exception closing file",e); + } + } + } + + return false; + } +} diff --git a/src/main/java/com/metaweb/gridworks/ProjectMetadata.java b/src/main/java/com/metaweb/gridworks/ProjectMetadata.java index fb162dc57..2008d0690 100644 --- a/src/main/java/com/metaweb/gridworks/ProjectMetadata.java +++ b/src/main/java/com/metaweb/gridworks/ProjectMetadata.java @@ -1,211 +1,211 @@ -package com.metaweb.gridworks; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.util.Date; -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONTokener; -import org.json.JSONWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.metaweb.gridworks.util.JSONUtilities; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class ProjectMetadata implements Jsonizable { - private static final int s_expressionHistoryMax = 20; // last n expressions used in this project - - private final Date _created; - private Date _modified; - private String _name; - private String _password; - - private String _encoding; - private int _encodingConfidence; - private List _expressions = new LinkedList(); - - final Logger logger = LoggerFactory.getLogger("project_metadata"); - - protected ProjectMetadata(Date date) { - _created = date; - } - - public ProjectMetadata() { - _created = new Date(); - _modified = _created; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("name"); writer.value(_name); - writer.key("created"); writer.value(ParsingUtilities.dateToString(_created)); - writer.key("modified"); writer.value(ParsingUtilities.dateToString(_modified)); - - if ("save".equals(options.getProperty("mode"))) { - writer.key("password"); writer.value(_password); - - writer.key("encoding"); writer.value(_encoding); - writer.key("encodingConfidence"); writer.value(_encodingConfidence); - writer.key("expressions"); JSONUtilities.writeStringList(writer, _expressions); - } - writer.endObject(); - } - - public void save(File projectDir) throws Exception { - File tempFile = new File(projectDir, "metadata.temp.json"); - try { - saveToFile(tempFile); - } catch (Exception e) { - e.printStackTrace(); - - logger.warn("Failed to save project metadata"); - return; - } - - File file = new File(projectDir, "metadata.json"); - File oldFile = new File(projectDir, "metadata.old.json"); - - if (file.exists()) { - file.renameTo(oldFile); - } - - tempFile.renameTo(file); - if (oldFile.exists()) { - oldFile.delete(); - } - } - - protected void saveToFile(File metadataFile) throws Exception { - Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile)); - try { - Properties options = new Properties(); - options.setProperty("mode", "save"); - - JSONWriter jsonWriter = new JSONWriter(writer); - - write(jsonWriter, options); - } finally { - writer.close(); - } - } - - static public ProjectMetadata load(File projectDir) { - try { - return loadFromFile(new File(projectDir, "metadata.json")); - } catch (Exception e) { - } - - try { - return loadFromFile(new File(projectDir, "metadata.temp.json")); - } catch (Exception e) { - } - - try { - return loadFromFile(new File(projectDir, "metadata.old.json")); - } catch (Exception e) { - } - - return null; - } - - static protected ProjectMetadata loadFromFile(File metadataFile) throws Exception { - FileReader reader = new FileReader(metadataFile); - try { - JSONTokener tokener = new JSONTokener(reader); - JSONObject obj = (JSONObject) tokener.nextValue(); - - return loadFromJSON(obj); - } finally { - reader.close(); - } - } - - static protected ProjectMetadata loadFromJSON(JSONObject obj) { - ProjectMetadata pm = new ProjectMetadata(JSONUtilities.getDate(obj, "modified", new Date())); - - pm._modified = JSONUtilities.getDate(obj, "modified", new Date()); - pm._name = JSONUtilities.getString(obj, "name", ""); - pm._password = JSONUtilities.getString(obj, "password", ""); - - pm._encoding = JSONUtilities.getString(obj, "encoding", ""); - pm._encodingConfidence = JSONUtilities.getInt(obj, "encodingConfidence", 0); - - JSONUtilities.getStringList(obj, "expressions", pm._expressions); - - return pm; - } - - public Date getCreated() { - return _created; - } - - public void setName(String name) { - this._name = name; - } - - public String getName() { - return _name; - } - - public void setEncoding(String encoding) { - this._encoding = encoding; - } - - public String getEncoding() { - return _encoding; - } - - public void setEncodingConfidence(int confidence) { - this._encodingConfidence = confidence; - } - - public void setEncodingConfidence(String confidence) { - if (confidence != null) { - this.setEncodingConfidence(Integer.parseInt(confidence)); - } - } - - public int getEncodingConfidence() { - return _encodingConfidence; - } - - public void setPassword(String password) { - this._password = password; - } - - public String getPassword() { - return _password; - } - - public Date getModified() { - return _modified; - } - - public void updateModified() { - _modified = new Date(); - } - - public void addLatestExpression(String s) { - _expressions.remove(s); - _expressions.add(0, s); - while (_expressions.size() > s_expressionHistoryMax) { - _expressions.remove(_expressions.size() - 1); - } - - ProjectManager.singleton.addLatestExpression(s); - } - - public List getExpressions() { - return _expressions; - } -} +package com.metaweb.gridworks; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.json.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.metaweb.gridworks.util.JSONUtilities; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class ProjectMetadata implements Jsonizable { + private static final int s_expressionHistoryMax = 20; // last n expressions used in this project + + private final Date _created; + private Date _modified; + private String _name; + private String _password; + + private String _encoding; + private int _encodingConfidence; + private List _expressions = new LinkedList(); + + final Logger logger = LoggerFactory.getLogger("project_metadata"); + + protected ProjectMetadata(Date date) { + _created = date; + } + + public ProjectMetadata() { + _created = new Date(); + _modified = _created; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("name"); writer.value(_name); + writer.key("created"); writer.value(ParsingUtilities.dateToString(_created)); + writer.key("modified"); writer.value(ParsingUtilities.dateToString(_modified)); + + if ("save".equals(options.getProperty("mode"))) { + writer.key("password"); writer.value(_password); + + writer.key("encoding"); writer.value(_encoding); + writer.key("encodingConfidence"); writer.value(_encodingConfidence); + writer.key("expressions"); JSONUtilities.writeStringList(writer, _expressions); + } + writer.endObject(); + } + + public void save(File projectDir) throws Exception { + File tempFile = new File(projectDir, "metadata.temp.json"); + try { + saveToFile(tempFile); + } catch (Exception e) { + e.printStackTrace(); + + logger.warn("Failed to save project metadata"); + return; + } + + File file = new File(projectDir, "metadata.json"); + File oldFile = new File(projectDir, "metadata.old.json"); + + if (file.exists()) { + file.renameTo(oldFile); + } + + tempFile.renameTo(file); + if (oldFile.exists()) { + oldFile.delete(); + } + } + + protected void saveToFile(File metadataFile) throws Exception { + Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile)); + try { + Properties options = new Properties(); + options.setProperty("mode", "save"); + + JSONWriter jsonWriter = new JSONWriter(writer); + + write(jsonWriter, options); + } finally { + writer.close(); + } + } + + static public ProjectMetadata load(File projectDir) { + try { + return loadFromFile(new File(projectDir, "metadata.json")); + } catch (Exception e) { + } + + try { + return loadFromFile(new File(projectDir, "metadata.temp.json")); + } catch (Exception e) { + } + + try { + return loadFromFile(new File(projectDir, "metadata.old.json")); + } catch (Exception e) { + } + + return null; + } + + static protected ProjectMetadata loadFromFile(File metadataFile) throws Exception { + FileReader reader = new FileReader(metadataFile); + try { + JSONTokener tokener = new JSONTokener(reader); + JSONObject obj = (JSONObject) tokener.nextValue(); + + return loadFromJSON(obj); + } finally { + reader.close(); + } + } + + static protected ProjectMetadata loadFromJSON(JSONObject obj) { + ProjectMetadata pm = new ProjectMetadata(JSONUtilities.getDate(obj, "modified", new Date())); + + pm._modified = JSONUtilities.getDate(obj, "modified", new Date()); + pm._name = JSONUtilities.getString(obj, "name", ""); + pm._password = JSONUtilities.getString(obj, "password", ""); + + pm._encoding = JSONUtilities.getString(obj, "encoding", ""); + pm._encodingConfidence = JSONUtilities.getInt(obj, "encodingConfidence", 0); + + JSONUtilities.getStringList(obj, "expressions", pm._expressions); + + return pm; + } + + public Date getCreated() { + return _created; + } + + public void setName(String name) { + this._name = name; + } + + public String getName() { + return _name; + } + + public void setEncoding(String encoding) { + this._encoding = encoding; + } + + public String getEncoding() { + return _encoding; + } + + public void setEncodingConfidence(int confidence) { + this._encodingConfidence = confidence; + } + + public void setEncodingConfidence(String confidence) { + if (confidence != null) { + this.setEncodingConfidence(Integer.parseInt(confidence)); + } + } + + public int getEncodingConfidence() { + return _encodingConfidence; + } + + public void setPassword(String password) { + this._password = password; + } + + public String getPassword() { + return _password; + } + + public Date getModified() { + return _modified; + } + + public void updateModified() { + _modified = new Date(); + } + + public void addLatestExpression(String s) { + _expressions.remove(s); + _expressions.add(0, s); + while (_expressions.size() > s_expressionHistoryMax) { + _expressions.remove(_expressions.size() - 1); + } + + ProjectManager.singleton.addLatestExpression(s); + } + + public List getExpressions() { + return _expressions; + } +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/ConjunctiveFilteredRows.java b/src/main/java/com/metaweb/gridworks/browsing/ConjunctiveFilteredRows.java index 149c8a816..b66d7540b 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/ConjunctiveFilteredRows.java +++ b/src/main/java/com/metaweb/gridworks/browsing/ConjunctiveFilteredRows.java @@ -1,96 +1,96 @@ -package com.metaweb.gridworks.browsing; - -import java.util.LinkedList; -import java.util.List; - -import com.metaweb.gridworks.browsing.filters.RowFilter; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - -/** - * Encapsulate logic for visiting rows that match all give row filters. Also visit - * context rows and dependent rows if configured so. - */ -public class ConjunctiveFilteredRows implements FilteredRows { - final protected List _rowFilters = new LinkedList(); - final protected boolean _includeContextual; - final protected boolean _includeDependent; - - public ConjunctiveFilteredRows(boolean includeContextual, boolean includeDependent) { - _includeContextual = includeContextual; - _includeDependent = includeDependent; - } - - public void add(RowFilter rowFilter) { - _rowFilters.add(rowFilter); - } - - public void accept(Project project, RowVisitor visitor) { - int lastVisitedRowRowIndex = -1; - int lastRecordRowAcceptedRowIndex = -1; - - int c = project.rows.size(); - for (int rowIndex = 0; rowIndex < c; rowIndex++) { - Row row = project.rows.get(rowIndex); - - if (matchRow(project, rowIndex, row)) { - if (row.recordIndex >= 0) { - lastRecordRowAcceptedRowIndex = rowIndex; // this is a record row itself - } - - visitRow(project, visitor, rowIndex, row, lastVisitedRowRowIndex); - - lastVisitedRowRowIndex = rowIndex; - } else if ( - // this row doesn't match by itself but ... - // we want to include dependent rows - - _includeDependent && - // and this row is a dependent row since it's not a record row - row.recordIndex < 0 && - row.contextRows != null && - row.contextRows.size() > 0 && - - row.contextRows.get(0) == lastRecordRowAcceptedRowIndex - ) { - // this row depends on the last previously matched record row, - // so we visit it as well as a dependent row - - visitor.visit(project, rowIndex, row, false, true); - lastVisitedRowRowIndex = rowIndex; - } - } - } - - protected void visitRow(Project project, RowVisitor visitor, int rowIndex, Row row, int lastVisitedRow) { - if (_includeContextual && // we need to include any context row and - row.contextRows != null && // this row itself isn't a context row and - lastVisitedRow < rowIndex - 1 // there is definitely some rows before this row - // that we haven't visited yet - ) { - for (int contextRowIndex : row.contextRows) { - if (contextRowIndex > lastVisitedRow) { - visitor.visit( - project, - contextRowIndex, - project.rows.get(contextRowIndex), - true, // is visited as a context row - false // is not visited as a dependent row - ); - lastVisitedRow = contextRowIndex; - } - } - } - - visitor.visit(project, rowIndex, row, false, false); - } - - protected boolean matchRow(Project project, int rowIndex, Row row) { - for (RowFilter rowFilter : _rowFilters) { - if (!rowFilter.filterRow(project, rowIndex, row)) { - return false; - } - } - return true; - } -} +package com.metaweb.gridworks.browsing; + +import java.util.LinkedList; +import java.util.List; + +import com.metaweb.gridworks.browsing.filters.RowFilter; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +/** + * Encapsulate logic for visiting rows that match all give row filters. Also visit + * context rows and dependent rows if configured so. + */ +public class ConjunctiveFilteredRows implements FilteredRows { + final protected List _rowFilters = new LinkedList(); + final protected boolean _includeContextual; + final protected boolean _includeDependent; + + public ConjunctiveFilteredRows(boolean includeContextual, boolean includeDependent) { + _includeContextual = includeContextual; + _includeDependent = includeDependent; + } + + public void add(RowFilter rowFilter) { + _rowFilters.add(rowFilter); + } + + public void accept(Project project, RowVisitor visitor) { + int lastVisitedRowRowIndex = -1; + int lastRecordRowAcceptedRowIndex = -1; + + int c = project.rows.size(); + for (int rowIndex = 0; rowIndex < c; rowIndex++) { + Row row = project.rows.get(rowIndex); + + if (matchRow(project, rowIndex, row)) { + if (row.recordIndex >= 0) { + lastRecordRowAcceptedRowIndex = rowIndex; // this is a record row itself + } + + visitRow(project, visitor, rowIndex, row, lastVisitedRowRowIndex); + + lastVisitedRowRowIndex = rowIndex; + } else if ( + // this row doesn't match by itself but ... + // we want to include dependent rows + + _includeDependent && + // and this row is a dependent row since it's not a record row + row.recordIndex < 0 && + row.contextRows != null && + row.contextRows.size() > 0 && + + row.contextRows.get(0) == lastRecordRowAcceptedRowIndex + ) { + // this row depends on the last previously matched record row, + // so we visit it as well as a dependent row + + visitor.visit(project, rowIndex, row, false, true); + lastVisitedRowRowIndex = rowIndex; + } + } + } + + protected void visitRow(Project project, RowVisitor visitor, int rowIndex, Row row, int lastVisitedRow) { + if (_includeContextual && // we need to include any context row and + row.contextRows != null && // this row itself isn't a context row and + lastVisitedRow < rowIndex - 1 // there is definitely some rows before this row + // that we haven't visited yet + ) { + for (int contextRowIndex : row.contextRows) { + if (contextRowIndex > lastVisitedRow) { + visitor.visit( + project, + contextRowIndex, + project.rows.get(contextRowIndex), + true, // is visited as a context row + false // is not visited as a dependent row + ); + lastVisitedRow = contextRowIndex; + } + } + } + + visitor.visit(project, rowIndex, row, false, false); + } + + protected boolean matchRow(Project project, int rowIndex, Row row) { + for (RowFilter rowFilter : _rowFilters) { + if (!rowFilter.filterRow(project, rowIndex, row)) { + return false; + } + } + return true; + } +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/DecoratedValue.java b/src/main/java/com/metaweb/gridworks/browsing/DecoratedValue.java index 3c9672fab..b291eb6d0 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/DecoratedValue.java +++ b/src/main/java/com/metaweb/gridworks/browsing/DecoratedValue.java @@ -1,33 +1,33 @@ -package com.metaweb.gridworks.browsing; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; - -/** - * Store a value and its text label, in case the value is not a string itself. - * For instance, if a value is a date, then its label can be one particular - * rendering of that date. - * - * Facet choices that are presented to the user as text are stored as decorated values. - */ -public class DecoratedValue implements Jsonizable { - final public Object value; - final public String label; - - public DecoratedValue(Object value, String label) { - this.value = value; - this.label = label; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - writer.object(); - writer.key("v"); writer.value(value); - writer.key("l"); writer.value(label); - writer.endObject(); - } -} +package com.metaweb.gridworks.browsing; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; + +/** + * Store a value and its text label, in case the value is not a string itself. + * For instance, if a value is a date, then its label can be one particular + * rendering of that date. + * + * Facet choices that are presented to the user as text are stored as decorated values. + */ +public class DecoratedValue implements Jsonizable { + final public Object value; + final public String label; + + public DecoratedValue(Object value, String label) { + this.value = value; + this.label = label; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + writer.object(); + writer.key("v"); writer.value(value); + writer.key("l"); writer.value(label); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/Engine.java b/src/main/java/com/metaweb/gridworks/browsing/Engine.java index 5a6f249e4..6ce7dbbcd 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/Engine.java +++ b/src/main/java/com/metaweb/gridworks/browsing/Engine.java @@ -1,109 +1,109 @@ -package com.metaweb.gridworks.browsing; - -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.browsing.facets.Facet; -import com.metaweb.gridworks.browsing.facets.ListFacet; -import com.metaweb.gridworks.browsing.facets.RangeFacet; -import com.metaweb.gridworks.browsing.facets.ScatterplotFacet; -import com.metaweb.gridworks.browsing.facets.TextSearchFacet; -import com.metaweb.gridworks.browsing.filters.RowFilter; -import com.metaweb.gridworks.model.Project; - -/** - * Faceted browsing engine. - */ -public class Engine implements Jsonizable { - protected Project _project; - protected List _facets = new LinkedList(); - protected boolean _includeDependent; - - public final static String INCLUDE_DEPENDENT = "includeDependent"; - - public Engine(Project project) { - _project = project; - } - - public FilteredRows getAllFilteredRows(boolean includeContextual) { - return getFilteredRows(null, includeContextual); - } - - public FilteredRows getFilteredRows(Facet except, boolean includeContextual) { - ConjunctiveFilteredRows cfr = new ConjunctiveFilteredRows(includeContextual, _includeDependent); - for (Facet facet : _facets) { - if (facet != except) { - RowFilter rowFilter = facet.getRowFilter(); - if (rowFilter != null) { - cfr.add(rowFilter); - } - } - } - return cfr; - } - - public void initializeFromJSON(JSONObject o) throws Exception { - if (o == null) { - return; - } - - if (o.has("facets") && !o.isNull("facets")) { - JSONArray a = o.getJSONArray("facets"); - int length = a.length(); - - for (int i = 0; i < length; i++) { - JSONObject fo = a.getJSONObject(i); - String type = fo.has("type") ? fo.getString("type") : "list"; - - Facet facet = null; - if ("list".equals(type)) { - facet = new ListFacet(); - } else if ("range".equals(type)) { - facet = new RangeFacet(); - } else if ("scatterplot".equals(type)) { - facet = new ScatterplotFacet(); - } else if ("text".equals(type)) { - facet = new TextSearchFacet(); - } - - if (facet != null) { - facet.initializeFromJSON(_project, fo); - _facets.add(facet); - } - } - } - - if (o.has(INCLUDE_DEPENDENT) && !o.isNull(INCLUDE_DEPENDENT)) { - _includeDependent = o.getBoolean(INCLUDE_DEPENDENT); - } - } - - public void computeFacets() throws JSONException { - for (Facet facet : _facets) { - FilteredRows filteredRows = getFilteredRows(facet, false); - - facet.computeChoices(_project, filteredRows); - } - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("facets"); - writer.array(); - for (Facet facet : _facets) { - facet.write(writer, options); - } - writer.endArray(); - writer.key(INCLUDE_DEPENDENT); writer.value(_includeDependent); - writer.endObject(); - } -} +package com.metaweb.gridworks.browsing; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.browsing.facets.Facet; +import com.metaweb.gridworks.browsing.facets.ListFacet; +import com.metaweb.gridworks.browsing.facets.RangeFacet; +import com.metaweb.gridworks.browsing.facets.ScatterplotFacet; +import com.metaweb.gridworks.browsing.facets.TextSearchFacet; +import com.metaweb.gridworks.browsing.filters.RowFilter; +import com.metaweb.gridworks.model.Project; + +/** + * Faceted browsing engine. + */ +public class Engine implements Jsonizable { + protected Project _project; + protected List _facets = new LinkedList(); + protected boolean _includeDependent; + + public final static String INCLUDE_DEPENDENT = "includeDependent"; + + public Engine(Project project) { + _project = project; + } + + public FilteredRows getAllFilteredRows(boolean includeContextual) { + return getFilteredRows(null, includeContextual); + } + + public FilteredRows getFilteredRows(Facet except, boolean includeContextual) { + ConjunctiveFilteredRows cfr = new ConjunctiveFilteredRows(includeContextual, _includeDependent); + for (Facet facet : _facets) { + if (facet != except) { + RowFilter rowFilter = facet.getRowFilter(); + if (rowFilter != null) { + cfr.add(rowFilter); + } + } + } + return cfr; + } + + public void initializeFromJSON(JSONObject o) throws Exception { + if (o == null) { + return; + } + + if (o.has("facets") && !o.isNull("facets")) { + JSONArray a = o.getJSONArray("facets"); + int length = a.length(); + + for (int i = 0; i < length; i++) { + JSONObject fo = a.getJSONObject(i); + String type = fo.has("type") ? fo.getString("type") : "list"; + + Facet facet = null; + if ("list".equals(type)) { + facet = new ListFacet(); + } else if ("range".equals(type)) { + facet = new RangeFacet(); + } else if ("scatterplot".equals(type)) { + facet = new ScatterplotFacet(); + } else if ("text".equals(type)) { + facet = new TextSearchFacet(); + } + + if (facet != null) { + facet.initializeFromJSON(_project, fo); + _facets.add(facet); + } + } + } + + if (o.has(INCLUDE_DEPENDENT) && !o.isNull(INCLUDE_DEPENDENT)) { + _includeDependent = o.getBoolean(INCLUDE_DEPENDENT); + } + } + + public void computeFacets() throws JSONException { + for (Facet facet : _facets) { + FilteredRows filteredRows = getFilteredRows(facet, false); + + facet.computeChoices(_project, filteredRows); + } + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("facets"); + writer.array(); + for (Facet facet : _facets) { + facet.write(writer, options); + } + writer.endArray(); + writer.key(INCLUDE_DEPENDENT); writer.value(_includeDependent); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/FilteredRows.java b/src/main/java/com/metaweb/gridworks/browsing/FilteredRows.java index 489411ed6..ff9d16dfa 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/FilteredRows.java +++ b/src/main/java/com/metaweb/gridworks/browsing/FilteredRows.java @@ -1,19 +1,19 @@ -package com.metaweb.gridworks.browsing; - -import com.metaweb.gridworks.model.Project; - -/** - * Interface for anything that can decide which rows match and which rows don't match - * based on some particular criteria. - */ -public interface FilteredRows { - /** - * Go through the rows of the given project, determine which match and which don't, - * and call visitor.visit() on those that match, and possibly their context and - * dependent rows. - * - * @param project - * @param visitor - */ - public void accept(Project project, RowVisitor visitor); -} +package com.metaweb.gridworks.browsing; + +import com.metaweb.gridworks.model.Project; + +/** + * Interface for anything that can decide which rows match and which rows don't match + * based on some particular criteria. + */ +public interface FilteredRows { + /** + * Go through the rows of the given project, determine which match and which don't, + * and call visitor.visit() on those that match, and possibly their context and + * dependent rows. + * + * @param project + * @param visitor + */ + public void accept(Project project, RowVisitor visitor); +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/RowVisitor.java b/src/main/java/com/metaweb/gridworks/browsing/RowVisitor.java index 36df82383..c4bcfcd43 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/RowVisitor.java +++ b/src/main/java/com/metaweb/gridworks/browsing/RowVisitor.java @@ -1,22 +1,22 @@ -package com.metaweb.gridworks.browsing; - -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - -/** - * Interface for visiting rows one by one. The rows visited are only those that match some - * particular criteria, such as facets' constraints, or those that are related to the matched - * rows. The related rows can be those that the matched rows depend on, or those that depend - * on the matched rows. - */ -public interface RowVisitor { - public boolean visit( - Project project, - int rowIndex, // zero-based row index - Row row, - boolean contextual, // true if this row is included because it's the context row - // of a matched row, that is, a matched row depends on it - boolean dependent // true if this row is included because it depends on a matched row, - // that is, it depends on a matched row - ); -} +package com.metaweb.gridworks.browsing; + +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +/** + * Interface for visiting rows one by one. The rows visited are only those that match some + * particular criteria, such as facets' constraints, or those that are related to the matched + * rows. The related rows can be those that the matched rows depend on, or those that depend + * on the matched rows. + */ +public interface RowVisitor { + public boolean visit( + Project project, + int rowIndex, // zero-based row index + Row row, + boolean contextual, // true if this row is included because it's the context row + // of a matched row, that is, a matched row depends on it + boolean dependent // true if this row is included because it depends on a matched row, + // that is, it depends on a matched row + ); +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNominalRowGrouper.java b/src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNominalRowGrouper.java index 52d99b29d..84c0fb965 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNominalRowGrouper.java +++ b/src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNominalRowGrouper.java @@ -1,88 +1,88 @@ -package com.metaweb.gridworks.browsing.facets; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import com.metaweb.gridworks.browsing.DecoratedValue; -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - -/** - * Visit matched rows and group them into facet choices based on the values computed - * from a given expression. - */ -public class ExpressionNominalRowGrouper implements RowVisitor { - /* - * Configuration - */ - final protected Evaluable _evaluable; - final protected String _columnName; - final protected int _cellIndex; - - /* - * Computed results - */ - final public Map choices = new HashMap(); - public int blankCount = 0; - public int errorCount = 0; - - public ExpressionNominalRowGrouper(Evaluable evaluable, String columnName, int cellIndex) { - _evaluable = evaluable; - _columnName = columnName; - _cellIndex = cellIndex; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex); - - Properties bindings = ExpressionUtils.createBindings(project); - ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); - - Object value = _evaluable.evaluate(bindings); - if (value != null) { - if (value.getClass().isArray()) { - Object[] a = (Object[]) value; - for (Object v : a) { - processValue(v); - } - return false; - } else if (value instanceof Collection) { - for (Object v : ExpressionUtils.toObjectCollection(value)) { - processValue(v); - } - return false; - } // else, fall through - } - - processValue(value); - return false; - } - - protected void processValue(Object value) { - if (ExpressionUtils.isError(value)) { - errorCount++; - } else if (ExpressionUtils.isNonBlankData(value)) { - String valueString = value.toString(); - String label = value.toString(); - - DecoratedValue dValue = new DecoratedValue(value, label); - - if (choices.containsKey(valueString)) { - choices.get(valueString).count++; - } else { - NominalFacetChoice choice = new NominalFacetChoice(dValue); - choice.count = 1; - - choices.put(valueString, choice); - } - } else { - blankCount++; - } - } -} +package com.metaweb.gridworks.browsing.facets; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import com.metaweb.gridworks.browsing.DecoratedValue; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +/** + * Visit matched rows and group them into facet choices based on the values computed + * from a given expression. + */ +public class ExpressionNominalRowGrouper implements RowVisitor { + /* + * Configuration + */ + final protected Evaluable _evaluable; + final protected String _columnName; + final protected int _cellIndex; + + /* + * Computed results + */ + final public Map choices = new HashMap(); + public int blankCount = 0; + public int errorCount = 0; + + public ExpressionNominalRowGrouper(Evaluable evaluable, String columnName, int cellIndex) { + _evaluable = evaluable; + _columnName = columnName; + _cellIndex = cellIndex; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex); + + Properties bindings = ExpressionUtils.createBindings(project); + ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); + + Object value = _evaluable.evaluate(bindings); + if (value != null) { + if (value.getClass().isArray()) { + Object[] a = (Object[]) value; + for (Object v : a) { + processValue(v); + } + return false; + } else if (value instanceof Collection) { + for (Object v : ExpressionUtils.toObjectCollection(value)) { + processValue(v); + } + return false; + } // else, fall through + } + + processValue(value); + return false; + } + + protected void processValue(Object value) { + if (ExpressionUtils.isError(value)) { + errorCount++; + } else if (ExpressionUtils.isNonBlankData(value)) { + String valueString = value.toString(); + String label = value.toString(); + + DecoratedValue dValue = new DecoratedValue(value, label); + + if (choices.containsKey(valueString)) { + choices.get(valueString).count++; + } else { + NominalFacetChoice choice = new NominalFacetChoice(dValue); + choice.count = 1; + + choices.put(valueString, choice); + } + } else { + blankCount++; + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNumericRowBinner.java b/src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNumericRowBinner.java index bf51bba0d..723f976a3 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNumericRowBinner.java +++ b/src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNumericRowBinner.java @@ -1,125 +1,125 @@ -package com.metaweb.gridworks.browsing.facets; - -import java.util.Collection; -import java.util.Properties; - -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - -/** - * Visit matched rows and slot them into bins based on the numbers computed - * from a given expression. - */ -public class ExpressionNumericRowBinner implements RowVisitor { - /* - * Configuration - */ - final protected Evaluable _evaluable; - final protected String _columnName; - final protected int _cellIndex; - final protected NumericBinIndex _index; // base bins - - /* - * Computed results - */ - final public int[] bins; - public int numericCount; - public int nonNumericCount; - public int blankCount; - public int errorCount; - - /* - * Scratchpad variables - */ - private boolean rowHasError; - private boolean rowHasBlank; - private boolean rowHasNumeric; - private boolean rowHasNonNumeric; - - public ExpressionNumericRowBinner(Evaluable evaluable, String columnName, int cellIndex, NumericBinIndex index) { - _evaluable = evaluable; - _columnName = columnName; - _cellIndex = cellIndex; - _index = index; - bins = new int[_index.getBins().length]; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - Cell cell = row.getCell(_cellIndex); - - Properties bindings = ExpressionUtils.createBindings(project); - ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); - - Object value = _evaluable.evaluate(bindings); - - rowHasError = false; - rowHasBlank = false; - rowHasNumeric = false; - rowHasNonNumeric = false; - - if (value != null) { - if (value.getClass().isArray()) { - Object[] a = (Object[]) value; - for (Object v : a) { - processValue(v); - } - updateCounts(); - return false; - } else if (value instanceof Collection) { - for (Object v : ExpressionUtils.toObjectCollection(value)) { - processValue(v); - } - updateCounts(); - return false; - } // else, fall through - } - - processValue(value); - updateCounts(); - - return false; - } - - protected void updateCounts() { - if (rowHasError) { - errorCount++; - } - if (rowHasBlank) { - blankCount++; - } - if (rowHasNumeric) { - numericCount++; - } - if (rowHasNonNumeric) { - nonNumericCount++; - } - } - - protected void processValue(Object value) { - if (ExpressionUtils.isError(value)) { - rowHasError = true; - } else if (ExpressionUtils.isNonBlankData(value)) { - if (value instanceof Number) { - double d = ((Number) value).doubleValue(); - if (!Double.isInfinite(d) && !Double.isNaN(d)) { - rowHasNumeric = true; - - int bin = (int) Math.floor((d - _index.getMin()) / _index.getStep()); - if (bin >= 0 && bin < bins.length) { // as a precaution - bins[bin]++; - } - } else { - rowHasError = true; - } - } else { - rowHasNonNumeric = true; - } - } else { - rowHasBlank = true; - } - } -} +package com.metaweb.gridworks.browsing.facets; + +import java.util.Collection; +import java.util.Properties; + +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +/** + * Visit matched rows and slot them into bins based on the numbers computed + * from a given expression. + */ +public class ExpressionNumericRowBinner implements RowVisitor { + /* + * Configuration + */ + final protected Evaluable _evaluable; + final protected String _columnName; + final protected int _cellIndex; + final protected NumericBinIndex _index; // base bins + + /* + * Computed results + */ + final public int[] bins; + public int numericCount; + public int nonNumericCount; + public int blankCount; + public int errorCount; + + /* + * Scratchpad variables + */ + private boolean rowHasError; + private boolean rowHasBlank; + private boolean rowHasNumeric; + private boolean rowHasNonNumeric; + + public ExpressionNumericRowBinner(Evaluable evaluable, String columnName, int cellIndex, NumericBinIndex index) { + _evaluable = evaluable; + _columnName = columnName; + _cellIndex = cellIndex; + _index = index; + bins = new int[_index.getBins().length]; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + Cell cell = row.getCell(_cellIndex); + + Properties bindings = ExpressionUtils.createBindings(project); + ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); + + Object value = _evaluable.evaluate(bindings); + + rowHasError = false; + rowHasBlank = false; + rowHasNumeric = false; + rowHasNonNumeric = false; + + if (value != null) { + if (value.getClass().isArray()) { + Object[] a = (Object[]) value; + for (Object v : a) { + processValue(v); + } + updateCounts(); + return false; + } else if (value instanceof Collection) { + for (Object v : ExpressionUtils.toObjectCollection(value)) { + processValue(v); + } + updateCounts(); + return false; + } // else, fall through + } + + processValue(value); + updateCounts(); + + return false; + } + + protected void updateCounts() { + if (rowHasError) { + errorCount++; + } + if (rowHasBlank) { + blankCount++; + } + if (rowHasNumeric) { + numericCount++; + } + if (rowHasNonNumeric) { + nonNumericCount++; + } + } + + protected void processValue(Object value) { + if (ExpressionUtils.isError(value)) { + rowHasError = true; + } else if (ExpressionUtils.isNonBlankData(value)) { + if (value instanceof Number) { + double d = ((Number) value).doubleValue(); + if (!Double.isInfinite(d) && !Double.isNaN(d)) { + rowHasNumeric = true; + + int bin = (int) Math.floor((d - _index.getMin()) / _index.getStep()); + if (bin >= 0 && bin < bins.length) { // as a precaution + bins[bin]++; + } + } else { + rowHasError = true; + } + } else { + rowHasNonNumeric = true; + } + } else { + rowHasBlank = true; + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/facets/Facet.java b/src/main/java/com/metaweb/gridworks/browsing/facets/Facet.java index b49f3e454..ae8fe8fd3 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/facets/Facet.java +++ b/src/main/java/com/metaweb/gridworks/browsing/facets/Facet.java @@ -1,19 +1,19 @@ -package com.metaweb.gridworks.browsing.facets; - -import org.json.JSONObject; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.filters.RowFilter; -import com.metaweb.gridworks.model.Project; - -/** - * Interface of facets. - */ -public interface Facet extends Jsonizable { - public RowFilter getRowFilter(); - - public void computeChoices(Project project, FilteredRows filteredRows); - - public void initializeFromJSON(Project project, JSONObject o) throws Exception; -} +package com.metaweb.gridworks.browsing.facets; + +import org.json.JSONObject; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.filters.RowFilter; +import com.metaweb.gridworks.model.Project; + +/** + * Interface of facets. + */ +public interface Facet extends Jsonizable { + public RowFilter getRowFilter(); + + public void computeChoices(Project project, FilteredRows filteredRows); + + public void initializeFromJSON(Project project, JSONObject o) throws Exception; +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/facets/ListFacet.java b/src/main/java/com/metaweb/gridworks/browsing/facets/ListFacet.java index af0703b54..31626cff9 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/facets/ListFacet.java +++ b/src/main/java/com/metaweb/gridworks/browsing/facets/ListFacet.java @@ -1,203 +1,203 @@ -package com.metaweb.gridworks.browsing.facets; - -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.DecoratedValue; -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.filters.ExpressionEqualRowFilter; -import com.metaweb.gridworks.browsing.filters.RowFilter; -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.MetaParser; -import com.metaweb.gridworks.expr.ParsingException; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.util.JSONUtilities; - -public class ListFacet implements Facet { - /* - * Configuration - */ - protected String _name; - protected String _expression; - protected String _columnName; - protected boolean _invert; - - // If true, then facet won't show the blank and error choices - protected boolean _omitBlank; - protected boolean _omitError; - - protected List _selection = new LinkedList(); - protected boolean _selectBlank; - protected boolean _selectError; - - /* - * Derived configuration - */ - protected int _cellIndex; - protected Evaluable _eval; - protected String _errorMessage; - - /* - * Computed results - */ - protected List _choices = new LinkedList(); - protected int _blankCount; - protected int _errorCount; - - public ListFacet() { - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("name"); writer.value(_name); - writer.key("expression"); writer.value(_expression); - writer.key("columnName"); writer.value(_columnName); - writer.key("invert"); writer.value(_invert); - - if (_errorMessage != null) { - writer.key("error"); writer.value(_errorMessage); - } else if (_choices.size() > 2000) { - writer.key("error"); writer.value("Too many choices"); - } else { - writer.key("choices"); writer.array(); - for (NominalFacetChoice choice : _choices) { - choice.write(writer, options); - } - writer.endArray(); - - if (!_omitBlank && (_selectBlank || _blankCount > 0)) { - writer.key("blankChoice"); - writer.object(); - writer.key("s"); writer.value(_selectBlank); - writer.key("c"); writer.value(_blankCount); - writer.endObject(); - } - if (!_omitError && (_selectError || _errorCount > 0)) { - writer.key("errorChoice"); - writer.object(); - writer.key("s"); writer.value(_selectError); - writer.key("c"); writer.value(_errorCount); - writer.endObject(); - } - } - - writer.endObject(); - } - - public void initializeFromJSON(Project project, JSONObject o) throws Exception { - _name = o.getString("name"); - _expression = o.getString("expression"); - _columnName = o.getString("columnName"); - _invert = o.has("invert") && o.getBoolean("invert"); - - if (_columnName.length() > 0) { - Column column = project.columnModel.getColumnByName(_columnName); - if (column != null) { - _cellIndex = column.getCellIndex(); - } else { - _errorMessage = "No column named " + _columnName; - } - } else { - _cellIndex = -1; - } - - try { - _eval = MetaParser.parse(_expression); - } catch (ParsingException e) { - _errorMessage = e.getMessage(); - } - - _selection.clear(); - - JSONArray a = o.getJSONArray("selection"); - int length = a.length(); - - for (int i = 0; i < length; i++) { - JSONObject oc = a.getJSONObject(i); - JSONObject ocv = oc.getJSONObject("v"); - - DecoratedValue decoratedValue = new DecoratedValue( - ocv.get("v"), ocv.getString("l")); - - NominalFacetChoice nominalFacetChoice = new NominalFacetChoice(decoratedValue); - nominalFacetChoice.selected = true; - - _selection.add(nominalFacetChoice); - } - - _omitBlank = JSONUtilities.getBoolean(o, "omitBlank", false); - _omitError = JSONUtilities.getBoolean(o, "omitError", false); - - _selectBlank = JSONUtilities.getBoolean(o, "selectBlank", false); - _selectError = JSONUtilities.getBoolean(o, "selectError", false); - } - - public RowFilter getRowFilter() { - return - _eval == null || - _errorMessage != null || - (_selection.size() == 0 && !_selectBlank && !_selectError) ? - null : - new ExpressionEqualRowFilter( - _eval, - _columnName, - _cellIndex, - createMatches(), - _selectBlank, - _selectError, - _invert); - } - - public void computeChoices(Project project, FilteredRows filteredRows) { - if (_eval != null && _errorMessage == null) { - ExpressionNominalRowGrouper grouper = - new ExpressionNominalRowGrouper(_eval, _columnName, _cellIndex); - - filteredRows.accept(project, grouper); - - _choices.clear(); - _choices.addAll(grouper.choices.values()); - - for (NominalFacetChoice choice : _selection) { - String valueString = choice.decoratedValue.value.toString(); - - if (grouper.choices.containsKey(valueString)) { - grouper.choices.get(valueString).selected = true; - } else { - /* - * A selected choice can have zero count if it is selected together - * with other choices, and some other facets' constraints eliminate - * all rows projected to this choice altogether. For example, if you - * select both "car" and "bicycle" in the "type of vehicle" facet, and - * then constrain the "wheels" facet to more than 2, then the "bicycle" - * choice now has zero count even if it's still selected. The grouper - * won't be able to detect the "bicycle" choice, so we need to inject - * that choice into the choice list ourselves. - */ - choice.count = 0; - _choices.add(choice); - } - } - - _blankCount = grouper.blankCount; - _errorCount = grouper.errorCount; - } - } - - protected Object[] createMatches() { - Object[] a = new Object[_selection.size()]; - for (int i = 0; i < a.length; i++) { - a[i] = _selection.get(i).decoratedValue.value; - } - return a; - } -} +package com.metaweb.gridworks.browsing.facets; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.DecoratedValue; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.filters.ExpressionEqualRowFilter; +import com.metaweb.gridworks.browsing.filters.RowFilter; +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.MetaParser; +import com.metaweb.gridworks.expr.ParsingException; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.util.JSONUtilities; + +public class ListFacet implements Facet { + /* + * Configuration + */ + protected String _name; + protected String _expression; + protected String _columnName; + protected boolean _invert; + + // If true, then facet won't show the blank and error choices + protected boolean _omitBlank; + protected boolean _omitError; + + protected List _selection = new LinkedList(); + protected boolean _selectBlank; + protected boolean _selectError; + + /* + * Derived configuration + */ + protected int _cellIndex; + protected Evaluable _eval; + protected String _errorMessage; + + /* + * Computed results + */ + protected List _choices = new LinkedList(); + protected int _blankCount; + protected int _errorCount; + + public ListFacet() { + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("name"); writer.value(_name); + writer.key("expression"); writer.value(_expression); + writer.key("columnName"); writer.value(_columnName); + writer.key("invert"); writer.value(_invert); + + if (_errorMessage != null) { + writer.key("error"); writer.value(_errorMessage); + } else if (_choices.size() > 2000) { + writer.key("error"); writer.value("Too many choices"); + } else { + writer.key("choices"); writer.array(); + for (NominalFacetChoice choice : _choices) { + choice.write(writer, options); + } + writer.endArray(); + + if (!_omitBlank && (_selectBlank || _blankCount > 0)) { + writer.key("blankChoice"); + writer.object(); + writer.key("s"); writer.value(_selectBlank); + writer.key("c"); writer.value(_blankCount); + writer.endObject(); + } + if (!_omitError && (_selectError || _errorCount > 0)) { + writer.key("errorChoice"); + writer.object(); + writer.key("s"); writer.value(_selectError); + writer.key("c"); writer.value(_errorCount); + writer.endObject(); + } + } + + writer.endObject(); + } + + public void initializeFromJSON(Project project, JSONObject o) throws Exception { + _name = o.getString("name"); + _expression = o.getString("expression"); + _columnName = o.getString("columnName"); + _invert = o.has("invert") && o.getBoolean("invert"); + + if (_columnName.length() > 0) { + Column column = project.columnModel.getColumnByName(_columnName); + if (column != null) { + _cellIndex = column.getCellIndex(); + } else { + _errorMessage = "No column named " + _columnName; + } + } else { + _cellIndex = -1; + } + + try { + _eval = MetaParser.parse(_expression); + } catch (ParsingException e) { + _errorMessage = e.getMessage(); + } + + _selection.clear(); + + JSONArray a = o.getJSONArray("selection"); + int length = a.length(); + + for (int i = 0; i < length; i++) { + JSONObject oc = a.getJSONObject(i); + JSONObject ocv = oc.getJSONObject("v"); + + DecoratedValue decoratedValue = new DecoratedValue( + ocv.get("v"), ocv.getString("l")); + + NominalFacetChoice nominalFacetChoice = new NominalFacetChoice(decoratedValue); + nominalFacetChoice.selected = true; + + _selection.add(nominalFacetChoice); + } + + _omitBlank = JSONUtilities.getBoolean(o, "omitBlank", false); + _omitError = JSONUtilities.getBoolean(o, "omitError", false); + + _selectBlank = JSONUtilities.getBoolean(o, "selectBlank", false); + _selectError = JSONUtilities.getBoolean(o, "selectError", false); + } + + public RowFilter getRowFilter() { + return + _eval == null || + _errorMessage != null || + (_selection.size() == 0 && !_selectBlank && !_selectError) ? + null : + new ExpressionEqualRowFilter( + _eval, + _columnName, + _cellIndex, + createMatches(), + _selectBlank, + _selectError, + _invert); + } + + public void computeChoices(Project project, FilteredRows filteredRows) { + if (_eval != null && _errorMessage == null) { + ExpressionNominalRowGrouper grouper = + new ExpressionNominalRowGrouper(_eval, _columnName, _cellIndex); + + filteredRows.accept(project, grouper); + + _choices.clear(); + _choices.addAll(grouper.choices.values()); + + for (NominalFacetChoice choice : _selection) { + String valueString = choice.decoratedValue.value.toString(); + + if (grouper.choices.containsKey(valueString)) { + grouper.choices.get(valueString).selected = true; + } else { + /* + * A selected choice can have zero count if it is selected together + * with other choices, and some other facets' constraints eliminate + * all rows projected to this choice altogether. For example, if you + * select both "car" and "bicycle" in the "type of vehicle" facet, and + * then constrain the "wheels" facet to more than 2, then the "bicycle" + * choice now has zero count even if it's still selected. The grouper + * won't be able to detect the "bicycle" choice, so we need to inject + * that choice into the choice list ourselves. + */ + choice.count = 0; + _choices.add(choice); + } + } + + _blankCount = grouper.blankCount; + _errorCount = grouper.errorCount; + } + } + + protected Object[] createMatches() { + Object[] a = new Object[_selection.size()]; + for (int i = 0; i < a.length; i++) { + a[i] = _selection.get(i).decoratedValue.value; + } + return a; + } +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/facets/NominalFacetChoice.java b/src/main/java/com/metaweb/gridworks/browsing/facets/NominalFacetChoice.java index 96990c9f8..10c9ecbc3 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/facets/NominalFacetChoice.java +++ b/src/main/java/com/metaweb/gridworks/browsing/facets/NominalFacetChoice.java @@ -1,32 +1,32 @@ -package com.metaweb.gridworks.browsing.facets; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.browsing.DecoratedValue; - -/** - * Store a facet choice that has a decorated value, a count of matched rows, - * and a flag of whether it has been selected. - */ -public class NominalFacetChoice implements Jsonizable { - final public DecoratedValue decoratedValue; - public int count; - public boolean selected; - - public NominalFacetChoice(DecoratedValue decoratedValue) { - this.decoratedValue = decoratedValue; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - writer.object(); - writer.key("v"); decoratedValue.write(writer, options); - writer.key("c"); writer.value(count); - writer.key("s"); writer.value(selected); - writer.endObject(); - } -} +package com.metaweb.gridworks.browsing.facets; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.browsing.DecoratedValue; + +/** + * Store a facet choice that has a decorated value, a count of matched rows, + * and a flag of whether it has been selected. + */ +public class NominalFacetChoice implements Jsonizable { + final public DecoratedValue decoratedValue; + public int count; + public boolean selected; + + public NominalFacetChoice(DecoratedValue decoratedValue) { + this.decoratedValue = decoratedValue; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + writer.object(); + writer.key("v"); decoratedValue.write(writer, options); + writer.key("c"); writer.value(count); + writer.key("s"); writer.value(selected); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/facets/NumericBinIndex.java b/src/main/java/com/metaweb/gridworks/browsing/facets/NumericBinIndex.java index f6984dc31..e0ccd5736 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/facets/NumericBinIndex.java +++ b/src/main/java/com/metaweb/gridworks/browsing/facets/NumericBinIndex.java @@ -1,214 +1,214 @@ -package com.metaweb.gridworks.browsing.facets; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Properties; - -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - -/** - * A utility class for computing the base bins that form the base histograms of - * numeric range facets. It evaluates an expression on all the rows of a project to - * get numeric values, determines how many bins to distribute those values in, and - * bins the rows accordingly. - * - * This class processes all rows rather than just the filtered rows because it - * needs to compute the base bins of a numeric range facet, which remain unchanged - * as the user interacts with the facet. - */ -public class NumericBinIndex { - - private int _totalValueCount; - private int _numbericValueCount; - private double _min; - private double _max; - private double _step; - private int[] _bins; - - private int _numericRowCount; - private int _nonNumericRowCount; - private int _blankRowCount; - private int _errorRowCount; - - public NumericBinIndex(Project project, String columnName, int cellIndex, Evaluable eval) { - Properties bindings = ExpressionUtils.createBindings(project); - - _min = Double.POSITIVE_INFINITY; - _max = Double.NEGATIVE_INFINITY; - - List allValues = new ArrayList(); - for (int i = 0; i < project.rows.size(); i++) { - Row row = project.rows.get(i); - Cell cell = row.getCell(cellIndex); - - ExpressionUtils.bind(bindings, row, i, columnName, cell); - - Object value = eval.evaluate(bindings); - - boolean rowHasError = false; - boolean rowHasNonNumeric = false; - boolean rowHasNumeric = false; - boolean rowHasBlank = false; - - if (ExpressionUtils.isError(value)) { - rowHasError = true; - } else if (ExpressionUtils.isNonBlankData(value)) { - if (value.getClass().isArray()) { - Object[] a = (Object[]) value; - for (Object v : a) { - _totalValueCount++; - - if (ExpressionUtils.isError(v)) { - rowHasError = true; - } else if (ExpressionUtils.isNonBlankData(v)) { - if (v instanceof Number) { - rowHasNumeric = true; - processValue(((Number) v).doubleValue(), allValues); - } else { - rowHasNonNumeric = true; - } - } else { - rowHasBlank = true; - } - } - } else if (value instanceof Collection) { - for (Object v : ExpressionUtils.toObjectCollection(value)) { - _totalValueCount++; - - if (ExpressionUtils.isError(v)) { - rowHasError = true; - } else if (ExpressionUtils.isNonBlankData(v)) { - if (v instanceof Number) { - rowHasNumeric = true; - processValue(((Number) v).doubleValue(), allValues); - } else { - rowHasNonNumeric = true; - } - } else { - rowHasBlank = true; - } - } - } else { - _totalValueCount++; - - if (value instanceof Number) { - rowHasNumeric = true; - processValue(((Number) value).doubleValue(), allValues); - } else { - rowHasNonNumeric = true; - } - } - } else { - rowHasBlank = true; - } - - if (rowHasError) { - _errorRowCount++; - } - if (rowHasBlank) { - _blankRowCount++; - } - if (rowHasNumeric) { - _numericRowCount++; - } - if (rowHasNonNumeric) { - _nonNumericRowCount++; - } - } - - _numbericValueCount = allValues.size(); - - if (_min >= _max) { - _step = 1; - _min = Math.min(_min, _max); - _max = _step; - _bins = new int[1]; - - return; - } - - double diff = _max - _min; - - _step = 1; - if (diff > 10) { - while (_step * 100 < diff) { - _step *= 10; - } - } else { - while (_step * 100 > diff) { - _step /= 10; - } - } - - double originalMax = _max; - _min = (Math.floor(_min / _step) * _step); - _max = (Math.ceil(_max / _step) * _step); - - double binCount = (_max - _min) / _step; - if (binCount > 100) { - _step *= 2; - binCount = (binCount + 1) / 2; - } - - if (_max <= originalMax) { - _max += _step; - binCount++; - } - - _bins = new int[(int) Math.round(binCount)]; - for (double d : allValues) { - int bin = Math.max((int) Math.floor((d - _min) / _step),0); - _bins[bin]++; - } - } - - public boolean isNumeric() { - return _numbericValueCount > _totalValueCount / 2; - } - - public double getMin() { - return _min; - } - - public double getMax() { - return _max; - } - - public double getStep() { - return _step; - } - - public int[] getBins() { - return _bins; - } - - public int getNumericRowCount() { - return _numericRowCount; - } - - public int getNonNumericRowCount() { - return _nonNumericRowCount; - } - - public int getBlankRowCount() { - return _blankRowCount; - } - - public int getErrorRowCount() { - return _errorRowCount; - } - - protected void processValue(double v, List allValues) { - if (!Double.isInfinite(v) && !Double.isNaN(v)) { - _min = Math.min(_min, v); - _max = Math.max(_max, v); - allValues.add(v); - } - } - -} +package com.metaweb.gridworks.browsing.facets; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Properties; + +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +/** + * A utility class for computing the base bins that form the base histograms of + * numeric range facets. It evaluates an expression on all the rows of a project to + * get numeric values, determines how many bins to distribute those values in, and + * bins the rows accordingly. + * + * This class processes all rows rather than just the filtered rows because it + * needs to compute the base bins of a numeric range facet, which remain unchanged + * as the user interacts with the facet. + */ +public class NumericBinIndex { + + private int _totalValueCount; + private int _numbericValueCount; + private double _min; + private double _max; + private double _step; + private int[] _bins; + + private int _numericRowCount; + private int _nonNumericRowCount; + private int _blankRowCount; + private int _errorRowCount; + + public NumericBinIndex(Project project, String columnName, int cellIndex, Evaluable eval) { + Properties bindings = ExpressionUtils.createBindings(project); + + _min = Double.POSITIVE_INFINITY; + _max = Double.NEGATIVE_INFINITY; + + List allValues = new ArrayList(); + for (int i = 0; i < project.rows.size(); i++) { + Row row = project.rows.get(i); + Cell cell = row.getCell(cellIndex); + + ExpressionUtils.bind(bindings, row, i, columnName, cell); + + Object value = eval.evaluate(bindings); + + boolean rowHasError = false; + boolean rowHasNonNumeric = false; + boolean rowHasNumeric = false; + boolean rowHasBlank = false; + + if (ExpressionUtils.isError(value)) { + rowHasError = true; + } else if (ExpressionUtils.isNonBlankData(value)) { + if (value.getClass().isArray()) { + Object[] a = (Object[]) value; + for (Object v : a) { + _totalValueCount++; + + if (ExpressionUtils.isError(v)) { + rowHasError = true; + } else if (ExpressionUtils.isNonBlankData(v)) { + if (v instanceof Number) { + rowHasNumeric = true; + processValue(((Number) v).doubleValue(), allValues); + } else { + rowHasNonNumeric = true; + } + } else { + rowHasBlank = true; + } + } + } else if (value instanceof Collection) { + for (Object v : ExpressionUtils.toObjectCollection(value)) { + _totalValueCount++; + + if (ExpressionUtils.isError(v)) { + rowHasError = true; + } else if (ExpressionUtils.isNonBlankData(v)) { + if (v instanceof Number) { + rowHasNumeric = true; + processValue(((Number) v).doubleValue(), allValues); + } else { + rowHasNonNumeric = true; + } + } else { + rowHasBlank = true; + } + } + } else { + _totalValueCount++; + + if (value instanceof Number) { + rowHasNumeric = true; + processValue(((Number) value).doubleValue(), allValues); + } else { + rowHasNonNumeric = true; + } + } + } else { + rowHasBlank = true; + } + + if (rowHasError) { + _errorRowCount++; + } + if (rowHasBlank) { + _blankRowCount++; + } + if (rowHasNumeric) { + _numericRowCount++; + } + if (rowHasNonNumeric) { + _nonNumericRowCount++; + } + } + + _numbericValueCount = allValues.size(); + + if (_min >= _max) { + _step = 1; + _min = Math.min(_min, _max); + _max = _step; + _bins = new int[1]; + + return; + } + + double diff = _max - _min; + + _step = 1; + if (diff > 10) { + while (_step * 100 < diff) { + _step *= 10; + } + } else { + while (_step * 100 > diff) { + _step /= 10; + } + } + + double originalMax = _max; + _min = (Math.floor(_min / _step) * _step); + _max = (Math.ceil(_max / _step) * _step); + + double binCount = (_max - _min) / _step; + if (binCount > 100) { + _step *= 2; + binCount = (binCount + 1) / 2; + } + + if (_max <= originalMax) { + _max += _step; + binCount++; + } + + _bins = new int[(int) Math.round(binCount)]; + for (double d : allValues) { + int bin = Math.max((int) Math.floor((d - _min) / _step),0); + _bins[bin]++; + } + } + + public boolean isNumeric() { + return _numbericValueCount > _totalValueCount / 2; + } + + public double getMin() { + return _min; + } + + public double getMax() { + return _max; + } + + public double getStep() { + return _step; + } + + public int[] getBins() { + return _bins; + } + + public int getNumericRowCount() { + return _numericRowCount; + } + + public int getNonNumericRowCount() { + return _nonNumericRowCount; + } + + public int getBlankRowCount() { + return _blankRowCount; + } + + public int getErrorRowCount() { + return _errorRowCount; + } + + protected void processValue(double v, List allValues) { + if (!Double.isInfinite(v) && !Double.isNaN(v)) { + _min = Math.min(_min, v); + _max = Math.max(_max, v); + allValues.add(v); + } + } + +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/facets/RangeFacet.java b/src/main/java/com/metaweb/gridworks/browsing/facets/RangeFacet.java index b22936ac5..325858dfc 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/facets/RangeFacet.java +++ b/src/main/java/com/metaweb/gridworks/browsing/facets/RangeFacet.java @@ -1,209 +1,209 @@ -package com.metaweb.gridworks.browsing.facets; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.filters.ExpressionNumberComparisonRowFilter; -import com.metaweb.gridworks.browsing.filters.RowFilter; -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.MetaParser; -import com.metaweb.gridworks.expr.ParsingException; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.util.JSONUtilities; - -public class RangeFacet implements Facet { - /* - * Configuration, from the client side - */ - protected String _name; // name of facet - protected String _expression; // expression to compute numeric value(s) per row - protected String _columnName; // column to base expression on, if any - - protected double _from; // the numeric selection - protected double _to; - - protected boolean _selectNumeric; // whether the numeric selection applies, default true - protected boolean _selectNonNumeric; - protected boolean _selectBlank; - protected boolean _selectError; - - /* - * Derived configuration data - */ - protected int _cellIndex; - protected Evaluable _eval; - protected String _errorMessage; - protected boolean _selected; // false if we're certain that all rows will match - // and there isn't any filtering to do - - /* - * Computed data, to return to the client side - */ - protected double _min; - protected double _max; - protected double _step; - protected int[] _baseBins; - protected int[] _bins; - - protected int _baseNumericCount; - protected int _baseNonNumericCount; - protected int _baseBlankCount; - protected int _baseErrorCount; - - protected int _numericCount; - protected int _nonNumericCount; - protected int _blankCount; - protected int _errorCount; - - public RangeFacet() { - } - - private static final String MIN = "min"; - private static final String MAX = "max"; - private static final String TO = "to"; - private static final String FROM = "from"; - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("name"); writer.value(_name); - writer.key("expression"); writer.value(_expression); - writer.key("columnName"); writer.value(_columnName); - - if (_errorMessage != null) { - writer.key("error"); writer.value(_errorMessage); - } else { - if (!Double.isInfinite(_min) && !Double.isInfinite(_max)) { - writer.key(MIN); writer.value(_min); - writer.key(MAX); writer.value(_max); - writer.key("step"); writer.value(_step); - - writer.key("bins"); writer.array(); - for (int b : _bins) { - writer.value(b); - } - writer.endArray(); - - writer.key("baseBins"); writer.array(); - for (int b : _baseBins) { - writer.value(b); - } - writer.endArray(); - - writer.key(FROM); writer.value(_from); - writer.key(TO); writer.value(_to); - } - - writer.key("baseNumericCount"); writer.value(_baseNumericCount); - writer.key("baseNonNumericCount"); writer.value(_baseNonNumericCount); - writer.key("baseBlankCount"); writer.value(_baseBlankCount); - writer.key("baseErrorCount"); writer.value(_baseErrorCount); - - writer.key("numericCount"); writer.value(_numericCount); - writer.key("nonNumericCount"); writer.value(_nonNumericCount); - writer.key("blankCount"); writer.value(_blankCount); - writer.key("errorCount"); writer.value(_errorCount); - } - writer.endObject(); - } - - public void initializeFromJSON(Project project, JSONObject o) throws Exception { - _name = o.getString("name"); - _expression = o.getString("expression"); - _columnName = o.getString("columnName"); - - if (_columnName.length() > 0) { - Column column = project.columnModel.getColumnByName(_columnName); - if (column != null) { - _cellIndex = column.getCellIndex(); - } else { - _errorMessage = "No column named " + _columnName; - } - } else { - _cellIndex = -1; - } - - try { - _eval = MetaParser.parse(_expression); - } catch (ParsingException e) { - _errorMessage = e.getMessage(); - } - - if (o.has(FROM) || o.has(TO)) { - _from = o.has(FROM) ? o.getDouble(FROM) : _min; - _to = o.has(TO) ? o.getDouble(TO) : _max; - _selected = true; - } - - _selectNumeric = JSONUtilities.getBoolean(o, "selectNumeric", true); - _selectNonNumeric = JSONUtilities.getBoolean(o, "selectNonNumeric", true); - _selectBlank = JSONUtilities.getBoolean(o, "selectBlank", true); - _selectError = JSONUtilities.getBoolean(o, "selectError", true); - - if (!_selectNumeric || !_selectNonNumeric || !_selectBlank || !_selectError) { - _selected = true; - } - } - - public RowFilter getRowFilter() { - if (_eval != null && _errorMessage == null && _selected) { - return new ExpressionNumberComparisonRowFilter( - _eval, _columnName, _cellIndex, _selectNumeric, _selectNonNumeric, _selectBlank, _selectError) { - - protected boolean checkValue(double d) { - return d >= _from && d < _to; - }; - }; - } else { - return null; - } - } - - public void computeChoices(Project project, FilteredRows filteredRows) { - if (_eval != null && _errorMessage == null) { - Column column = project.columnModel.getColumnByCellIndex(_cellIndex); - - String key = "numeric-bin:" + _expression; - NumericBinIndex index = (NumericBinIndex) column.getPrecompute(key); - if (index == null) { - index = new NumericBinIndex(project, _columnName, _cellIndex, _eval); - column.setPrecompute(key, index); - } - - _min = index.getMin(); - _max = index.getMax(); - _step = index.getStep(); - _baseBins = index.getBins(); - - _baseNumericCount = index.getNumericRowCount(); - _baseNonNumericCount = index.getNonNumericRowCount(); - _baseBlankCount = index.getBlankRowCount(); - _baseErrorCount = index.getErrorRowCount(); - - if (_selected) { - _from = Math.max(_from, _min); - _to = Math.min(_to, _max); - } else { - _from = _min; - _to = _max; - } - - ExpressionNumericRowBinner binner = - new ExpressionNumericRowBinner(_eval, _columnName, _cellIndex, index); - - filteredRows.accept(project, binner); - - _bins = binner.bins; - _numericCount = binner.numericCount; - _nonNumericCount = binner.nonNumericCount; - _blankCount = binner.blankCount; - _errorCount = binner.errorCount; - } - } -} +package com.metaweb.gridworks.browsing.facets; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.filters.ExpressionNumberComparisonRowFilter; +import com.metaweb.gridworks.browsing.filters.RowFilter; +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.MetaParser; +import com.metaweb.gridworks.expr.ParsingException; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.util.JSONUtilities; + +public class RangeFacet implements Facet { + /* + * Configuration, from the client side + */ + protected String _name; // name of facet + protected String _expression; // expression to compute numeric value(s) per row + protected String _columnName; // column to base expression on, if any + + protected double _from; // the numeric selection + protected double _to; + + protected boolean _selectNumeric; // whether the numeric selection applies, default true + protected boolean _selectNonNumeric; + protected boolean _selectBlank; + protected boolean _selectError; + + /* + * Derived configuration data + */ + protected int _cellIndex; + protected Evaluable _eval; + protected String _errorMessage; + protected boolean _selected; // false if we're certain that all rows will match + // and there isn't any filtering to do + + /* + * Computed data, to return to the client side + */ + protected double _min; + protected double _max; + protected double _step; + protected int[] _baseBins; + protected int[] _bins; + + protected int _baseNumericCount; + protected int _baseNonNumericCount; + protected int _baseBlankCount; + protected int _baseErrorCount; + + protected int _numericCount; + protected int _nonNumericCount; + protected int _blankCount; + protected int _errorCount; + + public RangeFacet() { + } + + private static final String MIN = "min"; + private static final String MAX = "max"; + private static final String TO = "to"; + private static final String FROM = "from"; + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("name"); writer.value(_name); + writer.key("expression"); writer.value(_expression); + writer.key("columnName"); writer.value(_columnName); + + if (_errorMessage != null) { + writer.key("error"); writer.value(_errorMessage); + } else { + if (!Double.isInfinite(_min) && !Double.isInfinite(_max)) { + writer.key(MIN); writer.value(_min); + writer.key(MAX); writer.value(_max); + writer.key("step"); writer.value(_step); + + writer.key("bins"); writer.array(); + for (int b : _bins) { + writer.value(b); + } + writer.endArray(); + + writer.key("baseBins"); writer.array(); + for (int b : _baseBins) { + writer.value(b); + } + writer.endArray(); + + writer.key(FROM); writer.value(_from); + writer.key(TO); writer.value(_to); + } + + writer.key("baseNumericCount"); writer.value(_baseNumericCount); + writer.key("baseNonNumericCount"); writer.value(_baseNonNumericCount); + writer.key("baseBlankCount"); writer.value(_baseBlankCount); + writer.key("baseErrorCount"); writer.value(_baseErrorCount); + + writer.key("numericCount"); writer.value(_numericCount); + writer.key("nonNumericCount"); writer.value(_nonNumericCount); + writer.key("blankCount"); writer.value(_blankCount); + writer.key("errorCount"); writer.value(_errorCount); + } + writer.endObject(); + } + + public void initializeFromJSON(Project project, JSONObject o) throws Exception { + _name = o.getString("name"); + _expression = o.getString("expression"); + _columnName = o.getString("columnName"); + + if (_columnName.length() > 0) { + Column column = project.columnModel.getColumnByName(_columnName); + if (column != null) { + _cellIndex = column.getCellIndex(); + } else { + _errorMessage = "No column named " + _columnName; + } + } else { + _cellIndex = -1; + } + + try { + _eval = MetaParser.parse(_expression); + } catch (ParsingException e) { + _errorMessage = e.getMessage(); + } + + if (o.has(FROM) || o.has(TO)) { + _from = o.has(FROM) ? o.getDouble(FROM) : _min; + _to = o.has(TO) ? o.getDouble(TO) : _max; + _selected = true; + } + + _selectNumeric = JSONUtilities.getBoolean(o, "selectNumeric", true); + _selectNonNumeric = JSONUtilities.getBoolean(o, "selectNonNumeric", true); + _selectBlank = JSONUtilities.getBoolean(o, "selectBlank", true); + _selectError = JSONUtilities.getBoolean(o, "selectError", true); + + if (!_selectNumeric || !_selectNonNumeric || !_selectBlank || !_selectError) { + _selected = true; + } + } + + public RowFilter getRowFilter() { + if (_eval != null && _errorMessage == null && _selected) { + return new ExpressionNumberComparisonRowFilter( + _eval, _columnName, _cellIndex, _selectNumeric, _selectNonNumeric, _selectBlank, _selectError) { + + protected boolean checkValue(double d) { + return d >= _from && d < _to; + }; + }; + } else { + return null; + } + } + + public void computeChoices(Project project, FilteredRows filteredRows) { + if (_eval != null && _errorMessage == null) { + Column column = project.columnModel.getColumnByCellIndex(_cellIndex); + + String key = "numeric-bin:" + _expression; + NumericBinIndex index = (NumericBinIndex) column.getPrecompute(key); + if (index == null) { + index = new NumericBinIndex(project, _columnName, _cellIndex, _eval); + column.setPrecompute(key, index); + } + + _min = index.getMin(); + _max = index.getMax(); + _step = index.getStep(); + _baseBins = index.getBins(); + + _baseNumericCount = index.getNumericRowCount(); + _baseNonNumericCount = index.getNonNumericRowCount(); + _baseBlankCount = index.getBlankRowCount(); + _baseErrorCount = index.getErrorRowCount(); + + if (_selected) { + _from = Math.max(_from, _min); + _to = Math.min(_to, _max); + } else { + _from = _min; + _to = _max; + } + + ExpressionNumericRowBinner binner = + new ExpressionNumericRowBinner(_eval, _columnName, _cellIndex, index); + + filteredRows.accept(project, binner); + + _bins = binner.bins; + _numericCount = binner.numericCount; + _nonNumericCount = binner.nonNumericCount; + _blankCount = binner.blankCount; + _errorCount = binner.errorCount; + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/facets/TextSearchFacet.java b/src/main/java/com/metaweb/gridworks/browsing/facets/TextSearchFacet.java index f6af99db7..4d1c6315f 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/facets/TextSearchFacet.java +++ b/src/main/java/com/metaweb/gridworks/browsing/facets/TextSearchFacet.java @@ -1,105 +1,105 @@ -package com.metaweb.gridworks.browsing.facets; - -import java.util.Properties; -import java.util.regex.Pattern; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.filters.ExpressionStringComparisonRowFilter; -import com.metaweb.gridworks.browsing.filters.RowFilter; -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.gel.ast.VariableExpr; -import com.metaweb.gridworks.model.Project; - -public class TextSearchFacet implements Facet { - /* - * Configuration - */ - protected String _name; - protected String _columnName; - protected String _query; - protected String _mode; - protected boolean _caseSensitive; - - /* - * Derived configuration - */ - protected int _cellIndex; - protected Pattern _pattern; - - public TextSearchFacet() { - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("name"); writer.value(_name); - writer.key("columnName"); writer.value(_columnName); - writer.key("query"); writer.value(_query); - writer.key("mode"); writer.value(_mode); - writer.key("caseSensitive"); writer.value(_caseSensitive); - writer.endObject(); - } - - public void initializeFromJSON(Project project, JSONObject o) throws Exception { - _name = o.getString("name"); - _columnName = o.getString("columnName"); - - _cellIndex = project.columnModel.getColumnByName(_columnName).getCellIndex(); - - if (!o.isNull("query")) { - _query = o.getString("query"); - } - - _mode = o.getString("mode"); - _caseSensitive = o.getBoolean("caseSensitive"); - if (_query != null) { - _query = _query.trim(); - if (_query.length() > 0) { - if (!_caseSensitive) { - _query = _query.toLowerCase(); - } - - if ("regex".equals(_mode)) { - try { - _pattern = Pattern.compile(_query); - } catch (java.util.regex.PatternSyntaxException e) { - //e.printStackTrace(); - } - } - } - } - } - - public RowFilter getRowFilter() { - if (_query == null || _query.length() == 0) { - return null; - } else if ("regex".equals(_mode) && _pattern == null) { - return null; - } - - Evaluable eval = new VariableExpr("value"); - - if ("regex".equals(_mode)) { - return new ExpressionStringComparisonRowFilter(eval, _columnName, _cellIndex) { - protected boolean checkValue(String s) { - return _pattern.matcher(s).find(); - }; - }; - } else { - return new ExpressionStringComparisonRowFilter(eval, _columnName, _cellIndex) { - protected boolean checkValue(String s) { - return (_caseSensitive ? s : s.toLowerCase()).contains(_query); - }; - }; - } - } - - public void computeChoices(Project project, FilteredRows filteredRows) { - // nothing to do - } -} +package com.metaweb.gridworks.browsing.facets; + +import java.util.Properties; +import java.util.regex.Pattern; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.filters.ExpressionStringComparisonRowFilter; +import com.metaweb.gridworks.browsing.filters.RowFilter; +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.gel.ast.VariableExpr; +import com.metaweb.gridworks.model.Project; + +public class TextSearchFacet implements Facet { + /* + * Configuration + */ + protected String _name; + protected String _columnName; + protected String _query; + protected String _mode; + protected boolean _caseSensitive; + + /* + * Derived configuration + */ + protected int _cellIndex; + protected Pattern _pattern; + + public TextSearchFacet() { + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("name"); writer.value(_name); + writer.key("columnName"); writer.value(_columnName); + writer.key("query"); writer.value(_query); + writer.key("mode"); writer.value(_mode); + writer.key("caseSensitive"); writer.value(_caseSensitive); + writer.endObject(); + } + + public void initializeFromJSON(Project project, JSONObject o) throws Exception { + _name = o.getString("name"); + _columnName = o.getString("columnName"); + + _cellIndex = project.columnModel.getColumnByName(_columnName).getCellIndex(); + + if (!o.isNull("query")) { + _query = o.getString("query"); + } + + _mode = o.getString("mode"); + _caseSensitive = o.getBoolean("caseSensitive"); + if (_query != null) { + _query = _query.trim(); + if (_query.length() > 0) { + if (!_caseSensitive) { + _query = _query.toLowerCase(); + } + + if ("regex".equals(_mode)) { + try { + _pattern = Pattern.compile(_query); + } catch (java.util.regex.PatternSyntaxException e) { + //e.printStackTrace(); + } + } + } + } + } + + public RowFilter getRowFilter() { + if (_query == null || _query.length() == 0) { + return null; + } else if ("regex".equals(_mode) && _pattern == null) { + return null; + } + + Evaluable eval = new VariableExpr("value"); + + if ("regex".equals(_mode)) { + return new ExpressionStringComparisonRowFilter(eval, _columnName, _cellIndex) { + protected boolean checkValue(String s) { + return _pattern.matcher(s).find(); + }; + }; + } else { + return new ExpressionStringComparisonRowFilter(eval, _columnName, _cellIndex) { + protected boolean checkValue(String s) { + return (_caseSensitive ? s : s.toLowerCase()).contains(_query); + }; + }; + } + } + + public void computeChoices(Project project, FilteredRows filteredRows) { + // nothing to do + } +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionEqualRowFilter.java b/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionEqualRowFilter.java index 8430197cc..6b05ad3fa 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionEqualRowFilter.java +++ b/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionEqualRowFilter.java @@ -1,101 +1,101 @@ -package com.metaweb.gridworks.browsing.filters; - -import java.util.Collection; -import java.util.Properties; - -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - -/** - * Judge if a row matches by evaluating a given expression on the row, based on a particular - * column, and checking the result. It's a match if the result is any one of a given list of - * values, or if the result is blank or error and we want blank or error values. - */ -public class ExpressionEqualRowFilter implements RowFilter { - final protected Evaluable _evaluable; // the expression to evaluate - - final protected String _columnName; - final protected int _cellIndex; // the expression is based on this column; - // -1 if based on no column in particular, - // for expression such as "row.starred". - - final protected Object[] _matches; - final protected boolean _selectBlank; - final protected boolean _selectError; - final protected boolean _invert; - - public ExpressionEqualRowFilter( - Evaluable evaluable, - String columnName, - int cellIndex, - Object[] matches, - boolean selectBlank, - boolean selectError, - boolean invert - ) { - _evaluable = evaluable; - _columnName = columnName; - _cellIndex = cellIndex; - _matches = matches; - _selectBlank = selectBlank; - _selectError = selectError; - _invert = invert; - } - - public boolean filterRow(Project project, int rowIndex, Row row) { - return _invert != internalFilterRow(project, rowIndex, row); - } - - public boolean internalFilterRow(Project project, int rowIndex, Row row) { - Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex); - - Properties bindings = ExpressionUtils.createBindings(project); - ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); - - Object value = _evaluable.evaluate(bindings); - if (value != null) { - if (value.getClass().isArray()) { - Object[] a = (Object[]) value; - for (Object v : a) { - if (testValue(v)) { - return true; - } - } - return false; - } else if (value instanceof Collection) { - for (Object v : ExpressionUtils.toObjectCollection(value)) { - if (testValue(v)) { - return true; - } - } - return false; - } // else, fall through - } - - return testValue(value); - } - - protected boolean testValue(Object v) { - if (ExpressionUtils.isError(v)) { - return _selectError; - } else if (ExpressionUtils.isNonBlankData(v)) { - for (Object match : _matches) { - if (testValue(v, match)) { - return true; - } - } - return false; - } else { - return _selectBlank; - } - } - - protected boolean testValue(Object v, Object match) { - return (v instanceof Number && match instanceof Number) ? - ((Number) match).doubleValue() == ((Number) v).doubleValue() : - match.equals(v); - } -} +package com.metaweb.gridworks.browsing.filters; + +import java.util.Collection; +import java.util.Properties; + +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +/** + * Judge if a row matches by evaluating a given expression on the row, based on a particular + * column, and checking the result. It's a match if the result is any one of a given list of + * values, or if the result is blank or error and we want blank or error values. + */ +public class ExpressionEqualRowFilter implements RowFilter { + final protected Evaluable _evaluable; // the expression to evaluate + + final protected String _columnName; + final protected int _cellIndex; // the expression is based on this column; + // -1 if based on no column in particular, + // for expression such as "row.starred". + + final protected Object[] _matches; + final protected boolean _selectBlank; + final protected boolean _selectError; + final protected boolean _invert; + + public ExpressionEqualRowFilter( + Evaluable evaluable, + String columnName, + int cellIndex, + Object[] matches, + boolean selectBlank, + boolean selectError, + boolean invert + ) { + _evaluable = evaluable; + _columnName = columnName; + _cellIndex = cellIndex; + _matches = matches; + _selectBlank = selectBlank; + _selectError = selectError; + _invert = invert; + } + + public boolean filterRow(Project project, int rowIndex, Row row) { + return _invert != internalFilterRow(project, rowIndex, row); + } + + public boolean internalFilterRow(Project project, int rowIndex, Row row) { + Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex); + + Properties bindings = ExpressionUtils.createBindings(project); + ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); + + Object value = _evaluable.evaluate(bindings); + if (value != null) { + if (value.getClass().isArray()) { + Object[] a = (Object[]) value; + for (Object v : a) { + if (testValue(v)) { + return true; + } + } + return false; + } else if (value instanceof Collection) { + for (Object v : ExpressionUtils.toObjectCollection(value)) { + if (testValue(v)) { + return true; + } + } + return false; + } // else, fall through + } + + return testValue(value); + } + + protected boolean testValue(Object v) { + if (ExpressionUtils.isError(v)) { + return _selectError; + } else if (ExpressionUtils.isNonBlankData(v)) { + for (Object match : _matches) { + if (testValue(v, match)) { + return true; + } + } + return false; + } else { + return _selectBlank; + } + } + + protected boolean testValue(Object v, Object match) { + return (v instanceof Number && match instanceof Number) ? + ((Number) match).doubleValue() == ((Number) v).doubleValue() : + match.equals(v); + } +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionNumberComparisonRowFilter.java b/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionNumberComparisonRowFilter.java index 5d5d25bbe..6fd3e950e 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionNumberComparisonRowFilter.java +++ b/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionNumberComparisonRowFilter.java @@ -1,94 +1,94 @@ -package com.metaweb.gridworks.browsing.filters; - -import java.util.Collection; -import java.util.Properties; - -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - -/** - * Judge if a row matches by evaluating a given expression on the row, based on a particular - * column, and checking the result. It's a match if the result satisfies some numeric comparisons, - * or if the result is non-numeric or blank or error and we want non-numeric or blank or error - * values. - */ -abstract public class ExpressionNumberComparisonRowFilter implements RowFilter { - final protected Evaluable _evaluable; - final protected String _columnName; - final protected int _cellIndex; - final protected boolean _selectNumeric; - final protected boolean _selectNonNumeric; - final protected boolean _selectBlank; - final protected boolean _selectError; - - public ExpressionNumberComparisonRowFilter( - Evaluable evaluable, - String columnName, - int cellIndex, - boolean selectNumeric, - boolean selectNonNumeric, - boolean selectBlank, - boolean selectError - ) { - _evaluable = evaluable; - _columnName = columnName; - _cellIndex = cellIndex; - _selectNumeric = selectNumeric; - _selectNonNumeric = selectNonNumeric; - _selectBlank = selectBlank; - _selectError = selectError; - } - - public boolean filterRow(Project project, int rowIndex, Row row) { - Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex); - - Properties bindings = ExpressionUtils.createBindings(project); - ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); - - Object value = _evaluable.evaluate(bindings); - if (value != null) { - if (value.getClass().isArray()) { - Object[] a = (Object[]) value; - for (Object v : a) { - if (checkValue(v)) { - return true; - } - } - return false; - } else if (value instanceof Collection) { - for (Object v : ExpressionUtils.toObjectCollection(value)) { - if (checkValue(v)) { - return true; - } - } - return false; - } // else, fall through - } - - return checkValue(value); - } - - protected boolean checkValue(Object v) { - if (ExpressionUtils.isError(v)) { - return _selectError; - } else if (ExpressionUtils.isNonBlankData(v)) { - if (v instanceof Number) { - double d = ((Number) v).doubleValue(); - if (Double.isInfinite(d) || Double.isNaN(d)) { - return _selectError; - } else { - return _selectNumeric && checkValue(d); - } - } else { - return _selectNonNumeric; - } - } else { - return _selectBlank; - } - } - - abstract protected boolean checkValue(double d); -} +package com.metaweb.gridworks.browsing.filters; + +import java.util.Collection; +import java.util.Properties; + +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +/** + * Judge if a row matches by evaluating a given expression on the row, based on a particular + * column, and checking the result. It's a match if the result satisfies some numeric comparisons, + * or if the result is non-numeric or blank or error and we want non-numeric or blank or error + * values. + */ +abstract public class ExpressionNumberComparisonRowFilter implements RowFilter { + final protected Evaluable _evaluable; + final protected String _columnName; + final protected int _cellIndex; + final protected boolean _selectNumeric; + final protected boolean _selectNonNumeric; + final protected boolean _selectBlank; + final protected boolean _selectError; + + public ExpressionNumberComparisonRowFilter( + Evaluable evaluable, + String columnName, + int cellIndex, + boolean selectNumeric, + boolean selectNonNumeric, + boolean selectBlank, + boolean selectError + ) { + _evaluable = evaluable; + _columnName = columnName; + _cellIndex = cellIndex; + _selectNumeric = selectNumeric; + _selectNonNumeric = selectNonNumeric; + _selectBlank = selectBlank; + _selectError = selectError; + } + + public boolean filterRow(Project project, int rowIndex, Row row) { + Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex); + + Properties bindings = ExpressionUtils.createBindings(project); + ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); + + Object value = _evaluable.evaluate(bindings); + if (value != null) { + if (value.getClass().isArray()) { + Object[] a = (Object[]) value; + for (Object v : a) { + if (checkValue(v)) { + return true; + } + } + return false; + } else if (value instanceof Collection) { + for (Object v : ExpressionUtils.toObjectCollection(value)) { + if (checkValue(v)) { + return true; + } + } + return false; + } // else, fall through + } + + return checkValue(value); + } + + protected boolean checkValue(Object v) { + if (ExpressionUtils.isError(v)) { + return _selectError; + } else if (ExpressionUtils.isNonBlankData(v)) { + if (v instanceof Number) { + double d = ((Number) v).doubleValue(); + if (Double.isInfinite(d) || Double.isNaN(d)) { + return _selectError; + } else { + return _selectNumeric && checkValue(d); + } + } else { + return _selectNonNumeric; + } + } else { + return _selectBlank; + } + } + + abstract protected boolean checkValue(double d); +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionStringComparisonRowFilter.java b/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionStringComparisonRowFilter.java index e912e37c2..b9f6142d4 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionStringComparisonRowFilter.java +++ b/src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionStringComparisonRowFilter.java @@ -1,51 +1,51 @@ -package com.metaweb.gridworks.browsing.filters; - -import java.util.Properties; - -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - -/** - * Judge if a row matches by evaluating a given expression on the row, based on a particular - * column, and checking the result. It's a match if the result satisfies some string comparisons. - */ -abstract public class ExpressionStringComparisonRowFilter implements RowFilter { - final protected Evaluable _evaluable; - final protected String _columnName; - final protected int _cellIndex; - - public ExpressionStringComparisonRowFilter(Evaluable evaluable, String columnName, int cellIndex) { - _evaluable = evaluable; - _columnName = columnName; - _cellIndex = cellIndex; - } - - public boolean filterRow(Project project, int rowIndex, Row row) { - Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex); - - Properties bindings = ExpressionUtils.createBindings(project); - ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); - - Object value = _evaluable.evaluate(bindings); - if (value != null) { - if (value.getClass().isArray()) { - Object[] a = (Object[]) value; - for (Object v : a) { - if (checkValue(v instanceof String ? ((String) v) : v.toString())) { - return true; - } - } - } else { - if (checkValue(value instanceof String ? ((String) value) : value.toString())) { - return true; - } - } - } - return false; - } - - abstract protected boolean checkValue(String s); -} +package com.metaweb.gridworks.browsing.filters; + +import java.util.Properties; + +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +/** + * Judge if a row matches by evaluating a given expression on the row, based on a particular + * column, and checking the result. It's a match if the result satisfies some string comparisons. + */ +abstract public class ExpressionStringComparisonRowFilter implements RowFilter { + final protected Evaluable _evaluable; + final protected String _columnName; + final protected int _cellIndex; + + public ExpressionStringComparisonRowFilter(Evaluable evaluable, String columnName, int cellIndex) { + _evaluable = evaluable; + _columnName = columnName; + _cellIndex = cellIndex; + } + + public boolean filterRow(Project project, int rowIndex, Row row) { + Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex); + + Properties bindings = ExpressionUtils.createBindings(project); + ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); + + Object value = _evaluable.evaluate(bindings); + if (value != null) { + if (value.getClass().isArray()) { + Object[] a = (Object[]) value; + for (Object v : a) { + if (checkValue(v instanceof String ? ((String) v) : v.toString())) { + return true; + } + } + } else { + if (checkValue(value instanceof String ? ((String) value) : value.toString())) { + return true; + } + } + } + return false; + } + + abstract protected boolean checkValue(String s); +} diff --git a/src/main/java/com/metaweb/gridworks/browsing/filters/RowFilter.java b/src/main/java/com/metaweb/gridworks/browsing/filters/RowFilter.java index 3b47de46e..0e6ff136d 100644 --- a/src/main/java/com/metaweb/gridworks/browsing/filters/RowFilter.java +++ b/src/main/java/com/metaweb/gridworks/browsing/filters/RowFilter.java @@ -1,12 +1,12 @@ -package com.metaweb.gridworks.browsing.filters; - -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - -/** - * Interface for judging if a particular row matches or doesn't match some - * particular criterion, such as a facet constraint. - */ -public interface RowFilter { - public boolean filterRow(Project project, int rowIndex, Row row); -} +package com.metaweb.gridworks.browsing.filters; + +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +/** + * Interface for judging if a particular row matches or doesn't match some + * particular criterion, such as a facet constraint. + */ +public interface RowFilter { + public boolean filterRow(Project project, int rowIndex, Row row); +} diff --git a/src/main/java/com/metaweb/gridworks/commands/Command.java b/src/main/java/com/metaweb/gridworks/commands/Command.java index 0bb44342a..f6fdc66f1 100644 --- a/src/main/java/com/metaweb/gridworks/commands/Command.java +++ b/src/main/java/com/metaweb/gridworks/commands/Command.java @@ -1,223 +1,223 @@ -package com.metaweb.gridworks.commands; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.io.Writer; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.commons.lang.NotImplementedException; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.ProjectManager; -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.process.Process; -import com.metaweb.gridworks.util.ParsingUtilities; - -/** - * The super class of all calls that the client side can invoke, most of which - * are AJAX calls. - */ -public abstract class Command { - - final static protected Logger logger = LoggerFactory.getLogger("command"); - - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - throw new NotImplementedException(); - }; - - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - throw new NotImplementedException(); - }; - - /** - * Utility function to get the browsing engine's configuration as a JSON object - * from the "engine" request parameter, most often in the POST body. - * - * @param request - * @return - * @throws Exception - */ - static protected JSONObject getEngineConfig(HttpServletRequest request) throws Exception { - String json = request.getParameter("engine"); - return (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json); - } - - /** - * Utility function to reconstruct the browsing engine from the "engine" request parameter, - * most often in the POST body. - * - * @param request - * @param project - * @return - * @throws Exception - */ - static protected Engine getEngine(HttpServletRequest request, Project project) throws Exception { - Engine engine = new Engine(project); - String json = request.getParameter("engine"); - if (json != null) { - JSONObject o = ParsingUtilities.evaluateJsonStringToObject(json); - engine.initializeFromJSON(o); - } - return engine; - } - - /** - * Utility method for retrieving the Project object having the ID specified - * in the "project" URL parameter. - * - * @param request - * @return - * @throws ServletException - */ - static protected Project getProject(HttpServletRequest request) throws ServletException { - try { - Project p = ProjectManager.singleton.getProject(Long.parseLong(request.getParameter("project"))); - if (p != null) { - return p; - } - } catch (Exception e) { - // ignore - } - throw new ServletException("Missing or bad project URL parameter"); - } - - static protected int getIntegerParameter(HttpServletRequest request, String name, int def) { - try { - return Integer.parseInt(request.getParameter(name)); - } catch (Exception e) { - // ignore - } - return def; - } - - static protected JSONObject getJsonParameter(HttpServletRequest request, String name) { - String value = request.getParameter(name); - if (value != null) { - try { - return ParsingUtilities.evaluateJsonStringToObject(value); - } catch (JSONException e) { - logger.warn("error getting json parameter",e); - } - } - return null; - } - - static protected void performProcessAndRespond( - HttpServletRequest request, - HttpServletResponse response, - Project project, - Process process - ) throws Exception { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - HistoryEntry historyEntry = project.processManager.queueProcess(process); - if (historyEntry != null) { - Writer w = response.getWriter(); - JSONWriter writer = new JSONWriter(w); - Properties options = new Properties(); - - writer.object(); - writer.key("code"); writer.value("ok"); - writer.key("historyEntry"); historyEntry.write(writer, options); - writer.endObject(); - - w.flush(); - w.close(); - } else { - respond(response, "{ \"code\" : \"pending\" }"); - } - } - - static protected void respond(HttpServletResponse response, String content) - throws IOException { - - response.setCharacterEncoding("UTF-8"); - response.setStatus(HttpServletResponse.SC_OK); - Writer w = response.getWriter(); - w.write(content); - w.flush(); - w.close(); - } - - static protected void respond(HttpServletResponse response, String status, String message) - throws IOException, JSONException { - - Writer w = response.getWriter(); - JSONWriter writer = new JSONWriter(w); - writer.object(); - writer.key("status"); writer.value(status); - writer.key("message"); writer.value(message); - writer.endObject(); - w.flush(); - w.close(); - } - - static protected void respondJSON(HttpServletResponse response, Jsonizable o) - throws IOException, JSONException { - - respondJSON(response, o, new Properties()); - } - - static protected void respondJSON( - HttpServletResponse response, Jsonizable o, Properties options) - throws IOException, JSONException { - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - Writer w = response.getWriter(); - JSONWriter writer = new JSONWriter(w); - - o.write(writer, options); - w.flush(); - w.close(); - } - - static protected void respondException(HttpServletResponse response, Exception e) - throws IOException { - - logger.warn("Exception caught", e); - - try { - JSONObject o = new JSONObject(); - o.put("code", "error"); - o.put("message", e.getMessage()); - - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - e.printStackTrace(pw); - pw.flush(); - sw.flush(); - - o.put("stack", sw.toString()); - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - respond(response, o.toString()); - } catch (JSONException e1) { - e.printStackTrace(response.getWriter()); - } - } - - static protected void redirect(HttpServletResponse response, String url) throws IOException { - response.sendRedirect(url); - } - -} +package com.metaweb.gridworks.commands; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.io.Writer; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.lang.NotImplementedException; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.process.Process; +import com.metaweb.gridworks.util.ParsingUtilities; + +/** + * The super class of all calls that the client side can invoke, most of which + * are AJAX calls. + */ +public abstract class Command { + + final static protected Logger logger = LoggerFactory.getLogger("command"); + + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + throw new NotImplementedException(); + }; + + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + throw new NotImplementedException(); + }; + + /** + * Utility function to get the browsing engine's configuration as a JSON object + * from the "engine" request parameter, most often in the POST body. + * + * @param request + * @return + * @throws Exception + */ + static protected JSONObject getEngineConfig(HttpServletRequest request) throws Exception { + String json = request.getParameter("engine"); + return (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json); + } + + /** + * Utility function to reconstruct the browsing engine from the "engine" request parameter, + * most often in the POST body. + * + * @param request + * @param project + * @return + * @throws Exception + */ + static protected Engine getEngine(HttpServletRequest request, Project project) throws Exception { + Engine engine = new Engine(project); + String json = request.getParameter("engine"); + if (json != null) { + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(json); + engine.initializeFromJSON(o); + } + return engine; + } + + /** + * Utility method for retrieving the Project object having the ID specified + * in the "project" URL parameter. + * + * @param request + * @return + * @throws ServletException + */ + static protected Project getProject(HttpServletRequest request) throws ServletException { + try { + Project p = ProjectManager.singleton.getProject(Long.parseLong(request.getParameter("project"))); + if (p != null) { + return p; + } + } catch (Exception e) { + // ignore + } + throw new ServletException("Missing or bad project URL parameter"); + } + + static protected int getIntegerParameter(HttpServletRequest request, String name, int def) { + try { + return Integer.parseInt(request.getParameter(name)); + } catch (Exception e) { + // ignore + } + return def; + } + + static protected JSONObject getJsonParameter(HttpServletRequest request, String name) { + String value = request.getParameter(name); + if (value != null) { + try { + return ParsingUtilities.evaluateJsonStringToObject(value); + } catch (JSONException e) { + logger.warn("error getting json parameter",e); + } + } + return null; + } + + static protected void performProcessAndRespond( + HttpServletRequest request, + HttpServletResponse response, + Project project, + Process process + ) throws Exception { + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + HistoryEntry historyEntry = project.processManager.queueProcess(process); + if (historyEntry != null) { + Writer w = response.getWriter(); + JSONWriter writer = new JSONWriter(w); + Properties options = new Properties(); + + writer.object(); + writer.key("code"); writer.value("ok"); + writer.key("historyEntry"); historyEntry.write(writer, options); + writer.endObject(); + + w.flush(); + w.close(); + } else { + respond(response, "{ \"code\" : \"pending\" }"); + } + } + + static protected void respond(HttpServletResponse response, String content) + throws IOException { + + response.setCharacterEncoding("UTF-8"); + response.setStatus(HttpServletResponse.SC_OK); + Writer w = response.getWriter(); + w.write(content); + w.flush(); + w.close(); + } + + static protected void respond(HttpServletResponse response, String status, String message) + throws IOException, JSONException { + + Writer w = response.getWriter(); + JSONWriter writer = new JSONWriter(w); + writer.object(); + writer.key("status"); writer.value(status); + writer.key("message"); writer.value(message); + writer.endObject(); + w.flush(); + w.close(); + } + + static protected void respondJSON(HttpServletResponse response, Jsonizable o) + throws IOException, JSONException { + + respondJSON(response, o, new Properties()); + } + + static protected void respondJSON( + HttpServletResponse response, Jsonizable o, Properties options) + throws IOException, JSONException { + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + Writer w = response.getWriter(); + JSONWriter writer = new JSONWriter(w); + + o.write(writer, options); + w.flush(); + w.close(); + } + + static protected void respondException(HttpServletResponse response, Exception e) + throws IOException { + + logger.warn("Exception caught", e); + + try { + JSONObject o = new JSONObject(); + o.put("code", "error"); + o.put("message", e.getMessage()); + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + e.printStackTrace(pw); + pw.flush(); + sw.flush(); + + o.put("stack", sw.toString()); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + respond(response, o.toString()); + } catch (JSONException e1) { + e.printStackTrace(response.getWriter()); + } + } + + static protected void redirect(HttpServletResponse response, String url) throws IOException { + response.sendRedirect(url); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/commands/EngineDependentCommand.java b/src/main/java/com/metaweb/gridworks/commands/EngineDependentCommand.java index a8ba8079c..3882ad722 100644 --- a/src/main/java/com/metaweb/gridworks/commands/EngineDependentCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/EngineDependentCommand.java @@ -1,56 +1,56 @@ -package com.metaweb.gridworks.commands; - -import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONObject; - -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.process.Process; - -/** - * Convenient super class for commands that perform abstract operations on - * only the filtered rows based on the faceted browsing engine's configuration - * on the client side. - * - * The engine's configuration is passed over as a POST body parameter. It is - * retrieved, de-serialized, and used to construct the abstract operation. - * The operation is then used to construct a process. The process is then - * queued for execution. If the process is not long running and there is no - * other queued process, then it gets executed right away, resulting in some - * change to the history. Otherwise, it is pending. The client side can - * decide how to update its UI depending on whether the process is done or - * still pending. - * - * Note that there are interactions on the client side that change only - * individual cells or individual rows (such as starring one row or editing - * the text of one cell). These interactions do not depend on the faceted - * browsing engine's configuration, and so they don't invoke commands that - * subclass this class. See AnnotateOneRowCommand and EditOneCellCommand as - * examples. - */ -abstract public class EngineDependentCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - - AbstractOperation op = createOperation(project, request, getEngineConfig(request)); - Process process = op.createProcess(project, new Properties()); - - performProcessAndRespond(request, response, project, process); - } catch (Exception e) { - respondException(response, e); - } - } - - abstract protected AbstractOperation createOperation( - Project project, HttpServletRequest request, JSONObject engineConfig) throws Exception; -} +package com.metaweb.gridworks.commands; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONObject; + +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.process.Process; + +/** + * Convenient super class for commands that perform abstract operations on + * only the filtered rows based on the faceted browsing engine's configuration + * on the client side. + * + * The engine's configuration is passed over as a POST body parameter. It is + * retrieved, de-serialized, and used to construct the abstract operation. + * The operation is then used to construct a process. The process is then + * queued for execution. If the process is not long running and there is no + * other queued process, then it gets executed right away, resulting in some + * change to the history. Otherwise, it is pending. The client side can + * decide how to update its UI depending on whether the process is done or + * still pending. + * + * Note that there are interactions on the client side that change only + * individual cells or individual rows (such as starring one row or editing + * the text of one cell). These interactions do not depend on the faceted + * browsing engine's configuration, and so they don't invoke commands that + * subclass this class. See AnnotateOneRowCommand and EditOneCellCommand as + * examples. + */ +abstract public class EngineDependentCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + AbstractOperation op = createOperation(project, request, getEngineConfig(request)); + Process process = op.createProcess(project, new Properties()); + + performProcessAndRespond(request, response, project, process); + } catch (Exception e) { + respondException(response, e); + } + } + + abstract protected AbstractOperation createOperation( + Project project, HttpServletRequest request, JSONObject engineConfig) throws Exception; +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/AddColumnCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/AddColumnCommand.java index 0af000f41..8db4315ab 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/AddColumnCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/AddColumnCommand.java @@ -1,34 +1,34 @@ -package com.metaweb.gridworks.commands.edit; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.ColumnAdditionOperation; -import com.metaweb.gridworks.operations.TextTransformOperation; - -public class AddColumnCommand extends EngineDependentCommand { - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - String baseColumnName = request.getParameter("baseColumnName"); - String expression = request.getParameter("expression"); - String newColumnName = request.getParameter("newColumnName"); - int columnInsertIndex = Integer.parseInt(request.getParameter("columnInsertIndex")); - String onError = request.getParameter("onError"); - - return new ColumnAdditionOperation( - engineConfig, - baseColumnName, - expression, - TextTransformOperation.stringToOnError(onError), - newColumnName, - columnInsertIndex - ); - } - -} +package com.metaweb.gridworks.commands.edit; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.ColumnAdditionOperation; +import com.metaweb.gridworks.operations.TextTransformOperation; + +public class AddColumnCommand extends EngineDependentCommand { + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String baseColumnName = request.getParameter("baseColumnName"); + String expression = request.getParameter("expression"); + String newColumnName = request.getParameter("newColumnName"); + int columnInsertIndex = Integer.parseInt(request.getParameter("columnInsertIndex")); + String onError = request.getParameter("onError"); + + return new ColumnAdditionOperation( + engineConfig, + baseColumnName, + expression, + TextTransformOperation.stringToOnError(onError), + newColumnName, + columnInsertIndex + ); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/AnnotateOneRowCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/AnnotateOneRowCommand.java index 27096f998..fe6523b33 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/AnnotateOneRowCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/AnnotateOneRowCommand.java @@ -1,120 +1,120 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.changes.RowFlagChange; -import com.metaweb.gridworks.model.changes.RowStarChange; -import com.metaweb.gridworks.process.QuickHistoryEntryProcess; - -public class AnnotateOneRowCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - try { - Project project = getProject(request); - - int rowIndex = Integer.parseInt(request.getParameter("row")); - - String starredString = request.getParameter("starred"); - if (starredString != null) { - boolean starred = "true".endsWith(starredString); - String description = (starred ? "Star row " : "Unstar row ") + (rowIndex + 1); - - StarOneRowProcess process = new StarOneRowProcess( - project, - description, - rowIndex, - starred - ); - - performProcessAndRespond(request, response, project, process); - return; - } - - String flaggedString = request.getParameter("flagged"); - if (flaggedString != null) { - boolean flagged = "true".endsWith(flaggedString); - String description = (flagged ? "Flag row " : "Unflag row ") + (rowIndex + 1); - - FlagOneRowProcess process = new FlagOneRowProcess( - project, - description, - rowIndex, - flagged - ); - - performProcessAndRespond(request, response, project, process); - return; - } - - respond(response, "{ \"code\" : \"error\", \"message\" : \"invalid command parameters\" }"); - - } catch (Exception e) { - respondException(response, e); - } - } - - protected static class StarOneRowProcess extends QuickHistoryEntryProcess { - final int rowIndex; - final boolean starred; - - StarOneRowProcess( - Project project, - String briefDescription, - int rowIndex, - boolean starred - ) { - super(project, briefDescription); - - this.rowIndex = rowIndex; - this.starred = starred; - } - - protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception { - return new HistoryEntry( - historyEntryID, - _project, - (starred ? "Star row " : "Unstar row ") + (rowIndex + 1), - null, - new RowStarChange(rowIndex, starred) - ); - } - } - protected static class FlagOneRowProcess extends QuickHistoryEntryProcess { - final int rowIndex; - final boolean flagged; - - FlagOneRowProcess( - Project project, - String briefDescription, - int rowIndex, - boolean flagged - ) { - super(project, briefDescription); - - this.rowIndex = rowIndex; - this.flagged = flagged; - } - - protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception { - return new HistoryEntry( - historyEntryID, - _project, - (flagged ? "Flag row " : "Unflag row ") + (rowIndex + 1), - null, - new RowFlagChange(rowIndex, flagged) - ); - } - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.changes.RowFlagChange; +import com.metaweb.gridworks.model.changes.RowStarChange; +import com.metaweb.gridworks.process.QuickHistoryEntryProcess; + +public class AnnotateOneRowCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + try { + Project project = getProject(request); + + int rowIndex = Integer.parseInt(request.getParameter("row")); + + String starredString = request.getParameter("starred"); + if (starredString != null) { + boolean starred = "true".endsWith(starredString); + String description = (starred ? "Star row " : "Unstar row ") + (rowIndex + 1); + + StarOneRowProcess process = new StarOneRowProcess( + project, + description, + rowIndex, + starred + ); + + performProcessAndRespond(request, response, project, process); + return; + } + + String flaggedString = request.getParameter("flagged"); + if (flaggedString != null) { + boolean flagged = "true".endsWith(flaggedString); + String description = (flagged ? "Flag row " : "Unflag row ") + (rowIndex + 1); + + FlagOneRowProcess process = new FlagOneRowProcess( + project, + description, + rowIndex, + flagged + ); + + performProcessAndRespond(request, response, project, process); + return; + } + + respond(response, "{ \"code\" : \"error\", \"message\" : \"invalid command parameters\" }"); + + } catch (Exception e) { + respondException(response, e); + } + } + + protected static class StarOneRowProcess extends QuickHistoryEntryProcess { + final int rowIndex; + final boolean starred; + + StarOneRowProcess( + Project project, + String briefDescription, + int rowIndex, + boolean starred + ) { + super(project, briefDescription); + + this.rowIndex = rowIndex; + this.starred = starred; + } + + protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception { + return new HistoryEntry( + historyEntryID, + _project, + (starred ? "Star row " : "Unstar row ") + (rowIndex + 1), + null, + new RowStarChange(rowIndex, starred) + ); + } + } + protected static class FlagOneRowProcess extends QuickHistoryEntryProcess { + final int rowIndex; + final boolean flagged; + + FlagOneRowProcess( + Project project, + String briefDescription, + int rowIndex, + boolean flagged + ) { + super(project, briefDescription); + + this.rowIndex = rowIndex; + this.flagged = flagged; + } + + protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception { + return new HistoryEntry( + historyEntryID, + _project, + (flagged ? "Flag row " : "Unflag row ") + (rowIndex + 1), + null, + new RowFlagChange(rowIndex, flagged) + ); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/AnnotateRowsCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/AnnotateRowsCommand.java index 165ea10d4..1e6b2803c 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/AnnotateRowsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/AnnotateRowsCommand.java @@ -1,34 +1,34 @@ -package com.metaweb.gridworks.commands.edit; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.RowFlagOperation; -import com.metaweb.gridworks.operations.RowStarOperation; - -public class AnnotateRowsCommand extends EngineDependentCommand { - - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - String starredString = request.getParameter("starred"); - if (starredString != null) { - boolean starred = "true".endsWith(starredString); - - return new RowStarOperation(engineConfig, starred); - } - - String flaggedString = request.getParameter("flagged"); - if (flaggedString != null) { - boolean flagged = "true".endsWith(flaggedString); - - return new RowFlagOperation(engineConfig, flagged); - } - return null; - } -} +package com.metaweb.gridworks.commands.edit; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.RowFlagOperation; +import com.metaweb.gridworks.operations.RowStarOperation; + +public class AnnotateRowsCommand extends EngineDependentCommand { + + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String starredString = request.getParameter("starred"); + if (starredString != null) { + boolean starred = "true".endsWith(starredString); + + return new RowStarOperation(engineConfig, starred); + } + + String flaggedString = request.getParameter("flagged"); + if (flaggedString != null) { + boolean flagged = "true".endsWith(flaggedString); + + return new RowFlagOperation(engineConfig, flagged); + } + return null; + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/ApplyOperationsCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/ApplyOperationsCommand.java index 2103e4d23..a0b663ff0 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/ApplyOperationsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/ApplyOperationsCommand.java @@ -1,60 +1,60 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.OperationRegistry; -import com.metaweb.gridworks.process.Process; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class ApplyOperationsCommand extends Command { - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - Project project = getProject(request); - String jsonString = request.getParameter("operations"); - try { - JSONArray a = ParsingUtilities.evaluateJsonStringToArray(jsonString); - int count = a.length(); - for (int i = 0; i < count; i++) { - JSONObject obj = a.getJSONObject(i); - - reconstructOperation(project, obj); - } - - if (project.processManager.hasPending()) { - respond(response, "{ \"code\" : \"pending\" }"); - } else { - respond(response, "{ \"code\" : \"ok\" }"); - } - } catch (JSONException e) { - respondException(response, e); - } - } - - protected void reconstructOperation(Project project, JSONObject obj) { - AbstractOperation operation = OperationRegistry.reconstruct(project, obj); - if (operation != null) { - try { - Process process = operation.createProcess(project, new Properties()); - - project.processManager.queueProcess(process); - } catch (Exception e) { - e.printStackTrace(); - } - } - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.OperationRegistry; +import com.metaweb.gridworks.process.Process; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class ApplyOperationsCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + Project project = getProject(request); + String jsonString = request.getParameter("operations"); + try { + JSONArray a = ParsingUtilities.evaluateJsonStringToArray(jsonString); + int count = a.length(); + for (int i = 0; i < count; i++) { + JSONObject obj = a.getJSONObject(i); + + reconstructOperation(project, obj); + } + + if (project.processManager.hasPending()) { + respond(response, "{ \"code\" : \"pending\" }"); + } else { + respond(response, "{ \"code\" : \"ok\" }"); + } + } catch (JSONException e) { + respondException(response, e); + } + } + + protected void reconstructOperation(Project project, JSONObject obj) { + AbstractOperation operation = OperationRegistry.reconstruct(project, obj); + if (operation != null) { + try { + Process process = operation.createProcess(project, new Properties()); + + project.processManager.queueProcess(process); + } catch (Exception e) { + e.printStackTrace(); + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java index 182c2088f..956c484eb 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java @@ -1,486 +1,486 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.Serializable; -import java.io.UnsupportedEncodingException; -import java.net.URL; -import java.net.URLConnection; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Map.Entry; -import java.util.zip.GZIPInputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.commons.fileupload.FileItemIterator; -import org.apache.commons.fileupload.FileItemStream; -import org.apache.commons.fileupload.servlet.ServletFileUpload; -import org.apache.commons.fileupload.util.Streams; -import org.apache.tools.bzip2.CBZip2InputStream; -import org.apache.tools.tar.TarEntry; -import org.apache.tools.tar.TarInputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.ibm.icu.text.CharsetDetector; -import com.ibm.icu.text.CharsetMatch; -import com.metaweb.gridworks.Gridworks; -import com.metaweb.gridworks.ProjectManager; -import com.metaweb.gridworks.ProjectMetadata; -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.importers.ExcelImporter; -import com.metaweb.gridworks.importers.Importer; -import com.metaweb.gridworks.importers.MarcImporter; -import com.metaweb.gridworks.importers.TsvCsvImporter; -import com.metaweb.gridworks.importers.XmlImporter; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.util.IOUtils; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class CreateProjectCommand extends Command { - - final static Logger logger = LoggerFactory.getLogger("create-project_command"); - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - /* - * The uploaded file is in the POST body as a "file part". If - * we call request.getParameter() then the POST body will get - * read and we won't have a chance to parse the body ourselves. - * This is why we have to parse the URL for parameters ourselves. - * Don't call request.getParameter() before calling internalImport(). - */ - Properties options = ParsingUtilities.parseUrlParameters(request); - - Project project = new Project(); - - internalImport(request, project, options); - - /* - * The import process above populates options with parameters - * in the POST body. That's why we're constructing the project - * metadata object after calling internalImport(). - */ - ProjectMetadata pm = new ProjectMetadata(); - pm.setName(options.getProperty("project-name")); - pm.setPassword(options.getProperty("project-password")); - pm.setEncoding(options.getProperty("encoding")); - pm.setEncodingConfidence(options.getProperty("encoding_confidence")); - ProjectManager.singleton.registerProject(project, pm); - - project.columnModel.update(); - project.recomputeRowContextDependencies(); - - redirect(response, "/project.html?project=" + project.id); - } catch (Exception e) { - redirect(response, "/error.html?redirect=index.html&msg=" + - ParsingUtilities.encode("Failed to import file: " + e.getLocalizedMessage()) - ); - e.printStackTrace(); - } - } - - protected void internalImport( - HttpServletRequest request, - Project project, - Properties options - ) throws Exception { - - ServletFileUpload upload = new ServletFileUpload(); - String url = null; - - FileItemIterator iter = upload.getItemIterator(request); - while (iter.hasNext()) { - FileItemStream item = iter.next(); - String name = item.getFieldName().toLowerCase(); - InputStream stream = item.openStream(); - if (item.isFormField()) { - if (name.equals("raw-text")) { - Reader reader = new InputStreamReader(stream,"UTF-8"); - try { - internalInvokeImporter(project, new TsvCsvImporter(), options, reader); - } finally { - reader.close(); - } - } else if (name.equals("url")) { - url = Streams.asString(stream); - } else { - options.put(name, Streams.asString(stream)); - } - } else { - String fileName = item.getName().toLowerCase(); - try { - internalImportFile(project, options, fileName, stream); - } finally { - stream.close(); - } - } - } - - if (url != null && url.length() > 0) { - internalImportURL(request, project, options, url); - } - } - - static class SafeInputStream extends FilterInputStream { - public SafeInputStream(InputStream stream) { - super(stream); - } - - @Override - public void close() { - // some libraries attempt to close the input stream while they can't - // read anymore from it... unfortunately this behavior prevents - // the zip input stream from functioning correctly so we just have - // to ignore those close() calls and just close it ourselves - // forcefully later - } - - public void reallyClose() throws IOException { - super.close(); - } - } - - protected void internalImportFile( - Project project, - Properties options, - String fileName, - InputStream inputStream - ) throws Exception { - - logger.info("Importing '{}'", fileName); - - if (fileName.endsWith(".zip") || fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz") || fileName.endsWith(".tar.bz2")) { - - // first, save the file on disk, since we need two passes and we might - // not have enough memory to keep it all in there - File file = save(inputStream); - - // in the first pass, gather statistics about what files are in there - // unfortunately, we have to rely on files extensions, which is horrible but - // better than nothing - HashMap ext_map = new HashMap(); - - InputStream is = getStream(fileName, new FileInputStream(file)); - - // NOTE(SM): unfortunately, java.io does not provide any generalized class for - // archive-like input streams so while both TarInputStream and ZipInputStream - // behave precisely the same, there is no polymorphic behavior so we have - // to treat each instance explicitly... one of those times you wish you had - // closures - try { - if (is instanceof TarInputStream) { - TarInputStream tis = (TarInputStream) is; - TarEntry te; - while ((te = tis.getNextEntry()) != null) { - if (!te.isDirectory()) { - mapExtension(te.getName(),ext_map); - } - } - } else if (is instanceof ZipInputStream) { - ZipInputStream zis = (ZipInputStream) is; - ZipEntry ze; - while ((ze = zis.getNextEntry()) != null) { - if (!ze.isDirectory()) { - mapExtension(ze.getName(),ext_map); - } - } - } - } finally { - try { - is.close(); - } catch (IOException e) {} - } - - // sort extensions by how often they appear - List> values = new ArrayList>(ext_map.entrySet()); - Collections.sort(values, new ValuesComparator()); - - if (values.size() == 0) { - throw new RuntimeException("The archive contains no files."); - } - - // this will contain the set of extensions we'll load from the archive - HashSet exts = new HashSet(); - - // find the extension that is most frequent or those who share the highest frequency value - if (values.size() == 1) { - exts.add(values.get(0).getKey()); - } else { - Entry most_frequent = values.get(0); - Entry second_most_frequent = values.get(1); - if (most_frequent.getValue() > second_most_frequent.getValue()) { // we have a winner - exts.add(most_frequent.getKey()); - } else { // multiple extensions have the same frequency - int winning_frequency = most_frequent.getValue(); - for (Entry e : values) { - if (e.getValue() == winning_frequency) { - exts.add(e.getKey()); - } - } - } - } - - logger.info("Most frequent extensions: {}", exts.toString()); - - // second pass, load the data for real - is = getStream(fileName, new FileInputStream(file)); - SafeInputStream sis = new SafeInputStream(is); - try { - if (is instanceof TarInputStream) { - TarInputStream tis = (TarInputStream) is; - TarEntry te; - while ((te = tis.getNextEntry()) != null) { - if (!te.isDirectory()) { - String name = te.getName(); - String ext = getExtension(name)[1]; - if (exts.contains(ext)) { - internalImportFile(project, options, name, sis); - } - } - } - } else if (is instanceof ZipInputStream) { - ZipInputStream zis = (ZipInputStream) is; - ZipEntry ze; - while ((ze = zis.getNextEntry()) != null) { - if (!ze.isDirectory()) { - String name = ze.getName(); - String ext = getExtension(name)[1]; - if (exts.contains(ext)) { - internalImportFile(project, options, name, sis); - } - } - } - } - } finally { - try { - sis.reallyClose(); - } catch (IOException e) {} - } - - } else if (fileName.endsWith(".gz")) { - internalImportFile(project, options, getExtension(fileName)[0], new GZIPInputStream(inputStream)); - } else if (fileName.endsWith(".bz2")) { - internalImportFile(project, options, getExtension(fileName)[0], new CBZip2InputStream(inputStream)); - } else { - load(project, options, fileName, inputStream); - } - } - - public static class ValuesComparator implements Comparator>, Serializable { - private static final long serialVersionUID = 8845863616149837657L; - - public int compare(Entry o1, Entry o2) { - return o2.getValue() - o1.getValue(); - } - } - - private void load(Project project, Properties options, String fileName, InputStream inputStream) throws Exception { - Importer importer = guessImporter(options, null, fileName); - internalInvokeImporter(project, importer, options, inputStream, null); - } - - private File save(InputStream is) throws IOException { - File temp = Gridworks.getTempFile(Long.toString(System.currentTimeMillis())); - temp.deleteOnExit(); - IOUtils.copy(is,temp); - is.close(); - return temp; - } - - private void mapExtension(String name, Map ext_map) { - String ext = getExtension(name)[1]; - if (ext_map.containsKey(ext)) { - ext_map.put(ext, ext_map.get(ext) + 1); - } else { - ext_map.put(ext, 1); - } - } - - private InputStream getStream(String fileName, InputStream is) throws IOException { - if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) { - return new TarInputStream(new GZIPInputStream(is)); - } else if (fileName.endsWith(".tar.bz2")) { - return new TarInputStream(new CBZip2InputStream(is)); - } else { - return new ZipInputStream(is); - } - } - - private String[] getExtension(String filename) { - String[] result = new String[2]; - int ext_index = filename.lastIndexOf('.'); - result[0] = (ext_index == -1) ? filename : filename.substring(0,ext_index); - result[1] = (ext_index == -1) ? "" : filename.substring(ext_index + 1); - return result; - } - - protected void internalImportURL( - HttpServletRequest request, - Project project, - Properties options, - String urlString - ) throws Exception { - URL url = new URL(urlString); - URLConnection connection = null; - - try { - connection = url.openConnection(); - connection.setConnectTimeout(5000); - connection.connect(); - } catch (Exception e) { - throw new Exception("Cannot connect to " + urlString, e); - } - - InputStream inputStream = null; - try { - inputStream = connection.getInputStream(); - } catch (Exception e) { - throw new Exception("Cannot retrieve content from " + url, e); - } - - try { - Importer importer = guessImporter( - options, - connection.getContentType(), - url.getPath() - ); - - internalInvokeImporter(project, importer, options, inputStream, connection.getContentEncoding()); - } finally { - inputStream.close(); - } - } - - protected void internalInvokeImporter( - Project project, - Importer importer, - Properties options, - InputStream rawInputStream, - String encoding - ) throws Exception { - if (importer.takesReader()) { - - BufferedInputStream inputStream = new BufferedInputStream(rawInputStream); - - // NOTE(SM): The ICU4J char detection code requires the input stream to support mark/reset. - // Unfortunately, not all ServletInputStream implementations are marking, so we need do - // this memory-expensive wrapping to make it work. It's far from ideal but I don't have - // a more efficient solution. - byte[] bytes = new byte[1024 * 4]; - inputStream.mark(bytes.length); - inputStream.read(bytes); - inputStream.reset(); - - CharsetDetector detector = new CharsetDetector(); - detector.setDeclaredEncoding("utf8"); // most of the content on the web is encoded in UTF-8 so start with that - - Reader reader = null; - CharsetMatch[] charsetMatches = detector.setText(bytes).detectAll(); - for (CharsetMatch charsetMatch : charsetMatches) { - try { - reader = new InputStreamReader(inputStream, charsetMatch.getName()); - - options.setProperty("encoding", charsetMatch.getName()); - options.setProperty("encoding_confidence", Integer.toString(charsetMatch.getConfidence())); - - logger.info("Best encoding guess: {} [confidence: {}]", charsetMatch.getName(), charsetMatch.getConfidence()); - - break; - } catch (UnsupportedEncodingException e) { - // silent - } - } - - if (reader == null) { // when all else fails - reader = encoding != null ? - new InputStreamReader(inputStream, encoding) : - new InputStreamReader(inputStream); - } - - importer.read(reader, project, options); - } else { - importer.read(rawInputStream, project, options); - } - } - - protected void internalInvokeImporter( - Project project, - Importer importer, - Properties options, - Reader reader - ) throws Exception { - importer.read(reader, project, options); - } - - protected Importer guessImporter( - Properties options, String contentType, String fileName) { - - if (contentType != null) { - contentType = contentType.toLowerCase().trim(); - - if ("application/msexcel".equals(contentType) || - "application/x-msexcel".equals(contentType) || - "application/x-ms-excel".equals(contentType) || - "application/vnd.ms-excel".equals(contentType) || - "application/x-excel".equals(contentType) || - "application/xls".equals(contentType)) { - - return new ExcelImporter(false); - } else if("application/x-xls".equals(contentType)) { - return new ExcelImporter(true); - } else if("application/xml".equals(contentType) || - "text/xml".equals(contentType) || - "application/rss+xml".equals(contentType) || - "application/atom+xml".equals(contentType) || - "application/rdf+xml".equals(contentType)) { - return new XmlImporter(); - } else if ("application/marc".equals(contentType)) { - return new MarcImporter(); - } - } else if (fileName != null) { - fileName = fileName.toLowerCase(); - if (fileName.endsWith(".xls")) { - return new ExcelImporter(false); - } else if (fileName.endsWith(".xlsx")) { - return new ExcelImporter(true); - } else if ( - fileName.endsWith(".xml") || - fileName.endsWith(".rdf") || - fileName.endsWith(".atom") || - fileName.endsWith(".rss") - ) { - return new XmlImporter(); - } else if ( - fileName.endsWith(".mrc") || - fileName.endsWith(".marc") || - fileName.contains(".mrc.") || - fileName.contains(".marc.") - ) { - return new MarcImporter(); - } - } - - return new TsvCsvImporter(); - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.Serializable; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Map.Entry; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.fileupload.FileItemIterator; +import org.apache.commons.fileupload.FileItemStream; +import org.apache.commons.fileupload.servlet.ServletFileUpload; +import org.apache.commons.fileupload.util.Streams; +import org.apache.tools.bzip2.CBZip2InputStream; +import org.apache.tools.tar.TarEntry; +import org.apache.tools.tar.TarInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.ibm.icu.text.CharsetDetector; +import com.ibm.icu.text.CharsetMatch; +import com.metaweb.gridworks.Gridworks; +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.ProjectMetadata; +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.importers.ExcelImporter; +import com.metaweb.gridworks.importers.Importer; +import com.metaweb.gridworks.importers.MarcImporter; +import com.metaweb.gridworks.importers.TsvCsvImporter; +import com.metaweb.gridworks.importers.XmlImporter; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.util.IOUtils; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class CreateProjectCommand extends Command { + + final static Logger logger = LoggerFactory.getLogger("create-project_command"); + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + /* + * The uploaded file is in the POST body as a "file part". If + * we call request.getParameter() then the POST body will get + * read and we won't have a chance to parse the body ourselves. + * This is why we have to parse the URL for parameters ourselves. + * Don't call request.getParameter() before calling internalImport(). + */ + Properties options = ParsingUtilities.parseUrlParameters(request); + + Project project = new Project(); + + internalImport(request, project, options); + + /* + * The import process above populates options with parameters + * in the POST body. That's why we're constructing the project + * metadata object after calling internalImport(). + */ + ProjectMetadata pm = new ProjectMetadata(); + pm.setName(options.getProperty("project-name")); + pm.setPassword(options.getProperty("project-password")); + pm.setEncoding(options.getProperty("encoding")); + pm.setEncodingConfidence(options.getProperty("encoding_confidence")); + ProjectManager.singleton.registerProject(project, pm); + + project.columnModel.update(); + project.recomputeRowContextDependencies(); + + redirect(response, "/project.html?project=" + project.id); + } catch (Exception e) { + redirect(response, "/error.html?redirect=index.html&msg=" + + ParsingUtilities.encode("Failed to import file: " + e.getLocalizedMessage()) + ); + e.printStackTrace(); + } + } + + protected void internalImport( + HttpServletRequest request, + Project project, + Properties options + ) throws Exception { + + ServletFileUpload upload = new ServletFileUpload(); + String url = null; + + FileItemIterator iter = upload.getItemIterator(request); + while (iter.hasNext()) { + FileItemStream item = iter.next(); + String name = item.getFieldName().toLowerCase(); + InputStream stream = item.openStream(); + if (item.isFormField()) { + if (name.equals("raw-text")) { + Reader reader = new InputStreamReader(stream,"UTF-8"); + try { + internalInvokeImporter(project, new TsvCsvImporter(), options, reader); + } finally { + reader.close(); + } + } else if (name.equals("url")) { + url = Streams.asString(stream); + } else { + options.put(name, Streams.asString(stream)); + } + } else { + String fileName = item.getName().toLowerCase(); + try { + internalImportFile(project, options, fileName, stream); + } finally { + stream.close(); + } + } + } + + if (url != null && url.length() > 0) { + internalImportURL(request, project, options, url); + } + } + + static class SafeInputStream extends FilterInputStream { + public SafeInputStream(InputStream stream) { + super(stream); + } + + @Override + public void close() { + // some libraries attempt to close the input stream while they can't + // read anymore from it... unfortunately this behavior prevents + // the zip input stream from functioning correctly so we just have + // to ignore those close() calls and just close it ourselves + // forcefully later + } + + public void reallyClose() throws IOException { + super.close(); + } + } + + protected void internalImportFile( + Project project, + Properties options, + String fileName, + InputStream inputStream + ) throws Exception { + + logger.info("Importing '{}'", fileName); + + if (fileName.endsWith(".zip") || fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz") || fileName.endsWith(".tar.bz2")) { + + // first, save the file on disk, since we need two passes and we might + // not have enough memory to keep it all in there + File file = save(inputStream); + + // in the first pass, gather statistics about what files are in there + // unfortunately, we have to rely on files extensions, which is horrible but + // better than nothing + HashMap ext_map = new HashMap(); + + InputStream is = getStream(fileName, new FileInputStream(file)); + + // NOTE(SM): unfortunately, java.io does not provide any generalized class for + // archive-like input streams so while both TarInputStream and ZipInputStream + // behave precisely the same, there is no polymorphic behavior so we have + // to treat each instance explicitly... one of those times you wish you had + // closures + try { + if (is instanceof TarInputStream) { + TarInputStream tis = (TarInputStream) is; + TarEntry te; + while ((te = tis.getNextEntry()) != null) { + if (!te.isDirectory()) { + mapExtension(te.getName(),ext_map); + } + } + } else if (is instanceof ZipInputStream) { + ZipInputStream zis = (ZipInputStream) is; + ZipEntry ze; + while ((ze = zis.getNextEntry()) != null) { + if (!ze.isDirectory()) { + mapExtension(ze.getName(),ext_map); + } + } + } + } finally { + try { + is.close(); + } catch (IOException e) {} + } + + // sort extensions by how often they appear + List> values = new ArrayList>(ext_map.entrySet()); + Collections.sort(values, new ValuesComparator()); + + if (values.size() == 0) { + throw new RuntimeException("The archive contains no files."); + } + + // this will contain the set of extensions we'll load from the archive + HashSet exts = new HashSet(); + + // find the extension that is most frequent or those who share the highest frequency value + if (values.size() == 1) { + exts.add(values.get(0).getKey()); + } else { + Entry most_frequent = values.get(0); + Entry second_most_frequent = values.get(1); + if (most_frequent.getValue() > second_most_frequent.getValue()) { // we have a winner + exts.add(most_frequent.getKey()); + } else { // multiple extensions have the same frequency + int winning_frequency = most_frequent.getValue(); + for (Entry e : values) { + if (e.getValue() == winning_frequency) { + exts.add(e.getKey()); + } + } + } + } + + logger.info("Most frequent extensions: {}", exts.toString()); + + // second pass, load the data for real + is = getStream(fileName, new FileInputStream(file)); + SafeInputStream sis = new SafeInputStream(is); + try { + if (is instanceof TarInputStream) { + TarInputStream tis = (TarInputStream) is; + TarEntry te; + while ((te = tis.getNextEntry()) != null) { + if (!te.isDirectory()) { + String name = te.getName(); + String ext = getExtension(name)[1]; + if (exts.contains(ext)) { + internalImportFile(project, options, name, sis); + } + } + } + } else if (is instanceof ZipInputStream) { + ZipInputStream zis = (ZipInputStream) is; + ZipEntry ze; + while ((ze = zis.getNextEntry()) != null) { + if (!ze.isDirectory()) { + String name = ze.getName(); + String ext = getExtension(name)[1]; + if (exts.contains(ext)) { + internalImportFile(project, options, name, sis); + } + } + } + } + } finally { + try { + sis.reallyClose(); + } catch (IOException e) {} + } + + } else if (fileName.endsWith(".gz")) { + internalImportFile(project, options, getExtension(fileName)[0], new GZIPInputStream(inputStream)); + } else if (fileName.endsWith(".bz2")) { + internalImportFile(project, options, getExtension(fileName)[0], new CBZip2InputStream(inputStream)); + } else { + load(project, options, fileName, inputStream); + } + } + + public static class ValuesComparator implements Comparator>, Serializable { + private static final long serialVersionUID = 8845863616149837657L; + + public int compare(Entry o1, Entry o2) { + return o2.getValue() - o1.getValue(); + } + } + + private void load(Project project, Properties options, String fileName, InputStream inputStream) throws Exception { + Importer importer = guessImporter(options, null, fileName); + internalInvokeImporter(project, importer, options, inputStream, null); + } + + private File save(InputStream is) throws IOException { + File temp = Gridworks.getTempFile(Long.toString(System.currentTimeMillis())); + temp.deleteOnExit(); + IOUtils.copy(is,temp); + is.close(); + return temp; + } + + private void mapExtension(String name, Map ext_map) { + String ext = getExtension(name)[1]; + if (ext_map.containsKey(ext)) { + ext_map.put(ext, ext_map.get(ext) + 1); + } else { + ext_map.put(ext, 1); + } + } + + private InputStream getStream(String fileName, InputStream is) throws IOException { + if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) { + return new TarInputStream(new GZIPInputStream(is)); + } else if (fileName.endsWith(".tar.bz2")) { + return new TarInputStream(new CBZip2InputStream(is)); + } else { + return new ZipInputStream(is); + } + } + + private String[] getExtension(String filename) { + String[] result = new String[2]; + int ext_index = filename.lastIndexOf('.'); + result[0] = (ext_index == -1) ? filename : filename.substring(0,ext_index); + result[1] = (ext_index == -1) ? "" : filename.substring(ext_index + 1); + return result; + } + + protected void internalImportURL( + HttpServletRequest request, + Project project, + Properties options, + String urlString + ) throws Exception { + URL url = new URL(urlString); + URLConnection connection = null; + + try { + connection = url.openConnection(); + connection.setConnectTimeout(5000); + connection.connect(); + } catch (Exception e) { + throw new Exception("Cannot connect to " + urlString, e); + } + + InputStream inputStream = null; + try { + inputStream = connection.getInputStream(); + } catch (Exception e) { + throw new Exception("Cannot retrieve content from " + url, e); + } + + try { + Importer importer = guessImporter( + options, + connection.getContentType(), + url.getPath() + ); + + internalInvokeImporter(project, importer, options, inputStream, connection.getContentEncoding()); + } finally { + inputStream.close(); + } + } + + protected void internalInvokeImporter( + Project project, + Importer importer, + Properties options, + InputStream rawInputStream, + String encoding + ) throws Exception { + if (importer.takesReader()) { + + BufferedInputStream inputStream = new BufferedInputStream(rawInputStream); + + // NOTE(SM): The ICU4J char detection code requires the input stream to support mark/reset. + // Unfortunately, not all ServletInputStream implementations are marking, so we need do + // this memory-expensive wrapping to make it work. It's far from ideal but I don't have + // a more efficient solution. + byte[] bytes = new byte[1024 * 4]; + inputStream.mark(bytes.length); + inputStream.read(bytes); + inputStream.reset(); + + CharsetDetector detector = new CharsetDetector(); + detector.setDeclaredEncoding("utf8"); // most of the content on the web is encoded in UTF-8 so start with that + + Reader reader = null; + CharsetMatch[] charsetMatches = detector.setText(bytes).detectAll(); + for (CharsetMatch charsetMatch : charsetMatches) { + try { + reader = new InputStreamReader(inputStream, charsetMatch.getName()); + + options.setProperty("encoding", charsetMatch.getName()); + options.setProperty("encoding_confidence", Integer.toString(charsetMatch.getConfidence())); + + logger.info("Best encoding guess: {} [confidence: {}]", charsetMatch.getName(), charsetMatch.getConfidence()); + + break; + } catch (UnsupportedEncodingException e) { + // silent + } + } + + if (reader == null) { // when all else fails + reader = encoding != null ? + new InputStreamReader(inputStream, encoding) : + new InputStreamReader(inputStream); + } + + importer.read(reader, project, options); + } else { + importer.read(rawInputStream, project, options); + } + } + + protected void internalInvokeImporter( + Project project, + Importer importer, + Properties options, + Reader reader + ) throws Exception { + importer.read(reader, project, options); + } + + protected Importer guessImporter( + Properties options, String contentType, String fileName) { + + if (contentType != null) { + contentType = contentType.toLowerCase().trim(); + + if ("application/msexcel".equals(contentType) || + "application/x-msexcel".equals(contentType) || + "application/x-ms-excel".equals(contentType) || + "application/vnd.ms-excel".equals(contentType) || + "application/x-excel".equals(contentType) || + "application/xls".equals(contentType)) { + + return new ExcelImporter(false); + } else if("application/x-xls".equals(contentType)) { + return new ExcelImporter(true); + } else if("application/xml".equals(contentType) || + "text/xml".equals(contentType) || + "application/rss+xml".equals(contentType) || + "application/atom+xml".equals(contentType) || + "application/rdf+xml".equals(contentType)) { + return new XmlImporter(); + } else if ("application/marc".equals(contentType)) { + return new MarcImporter(); + } + } else if (fileName != null) { + fileName = fileName.toLowerCase(); + if (fileName.endsWith(".xls")) { + return new ExcelImporter(false); + } else if (fileName.endsWith(".xlsx")) { + return new ExcelImporter(true); + } else if ( + fileName.endsWith(".xml") || + fileName.endsWith(".rdf") || + fileName.endsWith(".atom") || + fileName.endsWith(".rss") + ) { + return new XmlImporter(); + } else if ( + fileName.endsWith(".mrc") || + fileName.endsWith(".marc") || + fileName.contains(".mrc.") || + fileName.contains(".marc.") + ) { + return new MarcImporter(); + } + } + + return new TsvCsvImporter(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/DenormalizeCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/DenormalizeCommand.java index e96961ae5..7e6b89552 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/DenormalizeCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/DenormalizeCommand.java @@ -1,32 +1,32 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.DenormalizeOperation; -import com.metaweb.gridworks.process.Process; - -public class DenormalizeCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - - AbstractOperation op = new DenormalizeOperation(); - Process process = op.createProcess(project, new Properties()); - - performProcessAndRespond(request, response, project, process); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.DenormalizeOperation; +import com.metaweb.gridworks.process.Process; + +public class DenormalizeCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + AbstractOperation op = new DenormalizeOperation(); + Process process = op.createProcess(project, new Properties()); + + performProcessAndRespond(request, response, project, process); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/EditOneCellCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/EditOneCellCommand.java index b7adfb782..4743e58c4 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/EditOneCellCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/EditOneCellCommand.java @@ -1,125 +1,125 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.IOException; -import java.io.Serializable; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONWriter; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.changes.CellChange; -import com.metaweb.gridworks.process.QuickHistoryEntryProcess; -import com.metaweb.gridworks.util.ParsingUtilities; -import com.metaweb.gridworks.util.Pool; - -public class EditOneCellCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - - int rowIndex = Integer.parseInt(request.getParameter("row")); - int cellIndex = Integer.parseInt(request.getParameter("cell")); - - String type = request.getParameter("type"); - String valueString = request.getParameter("value"); - Serializable value = null; - - if ("number".equals(type)) { - value = Double.parseDouble(valueString); - } else if ("boolean".equals(type)) { - value = "true".equalsIgnoreCase(valueString); - } else if ("date".equals(type)) { - value = ParsingUtilities.stringToDate(valueString); - } else { - value = valueString; - } - - EditOneCellProcess process = new EditOneCellProcess( - project, - "Edit single cell", - rowIndex, - cellIndex, - value - ); - - HistoryEntry historyEntry = project.processManager.queueProcess(process); - if (historyEntry != null) { - /* - * If the operation has been done, return the new cell's data - * so the client side can update the cell's rendering right away. - */ - JSONWriter writer = new JSONWriter(response.getWriter()); - - Pool pool = new Pool(); - Properties options = new Properties(); - options.put("pool", pool); - - writer.object(); - writer.key("code"); writer.value("ok"); - writer.key("historyEntry"); historyEntry.write(writer, options); - writer.key("cell"); process.newCell.write(writer, options); - writer.key("pool"); pool.write(writer, options); - writer.endObject(); - } else { - respond(response, "{ \"code\" : \"pending\" }"); - } - } catch (Exception e) { - respondException(response, e); - } - } - - protected static class EditOneCellProcess extends QuickHistoryEntryProcess { - final int rowIndex; - final int cellIndex; - final Serializable value; - Cell newCell; - - EditOneCellProcess( - Project project, - String briefDescription, - int rowIndex, - int cellIndex, - Serializable value - ) { - super(project, briefDescription); - - this.rowIndex = rowIndex; - this.cellIndex = cellIndex; - this.value = value; - } - - protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception { - Cell cell = _project.rows.get(rowIndex).getCell(cellIndex); - Column column = _project.columnModel.getColumnByCellIndex(cellIndex); - if (column == null) { - throw new Exception("No such column"); - } - - newCell = new Cell( - value, - cell != null ? cell.recon : null - ); - - String description = - "Edit single cell on row " + (rowIndex + 1) + - ", column " + column.getName(); - - Change change = new CellChange(rowIndex, cellIndex, cell, newCell); - - return new HistoryEntry( - historyEntryID, _project, description, null, change); - } - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONWriter; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.changes.CellChange; +import com.metaweb.gridworks.process.QuickHistoryEntryProcess; +import com.metaweb.gridworks.util.ParsingUtilities; +import com.metaweb.gridworks.util.Pool; + +public class EditOneCellCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + int rowIndex = Integer.parseInt(request.getParameter("row")); + int cellIndex = Integer.parseInt(request.getParameter("cell")); + + String type = request.getParameter("type"); + String valueString = request.getParameter("value"); + Serializable value = null; + + if ("number".equals(type)) { + value = Double.parseDouble(valueString); + } else if ("boolean".equals(type)) { + value = "true".equalsIgnoreCase(valueString); + } else if ("date".equals(type)) { + value = ParsingUtilities.stringToDate(valueString); + } else { + value = valueString; + } + + EditOneCellProcess process = new EditOneCellProcess( + project, + "Edit single cell", + rowIndex, + cellIndex, + value + ); + + HistoryEntry historyEntry = project.processManager.queueProcess(process); + if (historyEntry != null) { + /* + * If the operation has been done, return the new cell's data + * so the client side can update the cell's rendering right away. + */ + JSONWriter writer = new JSONWriter(response.getWriter()); + + Pool pool = new Pool(); + Properties options = new Properties(); + options.put("pool", pool); + + writer.object(); + writer.key("code"); writer.value("ok"); + writer.key("historyEntry"); historyEntry.write(writer, options); + writer.key("cell"); process.newCell.write(writer, options); + writer.key("pool"); pool.write(writer, options); + writer.endObject(); + } else { + respond(response, "{ \"code\" : \"pending\" }"); + } + } catch (Exception e) { + respondException(response, e); + } + } + + protected static class EditOneCellProcess extends QuickHistoryEntryProcess { + final int rowIndex; + final int cellIndex; + final Serializable value; + Cell newCell; + + EditOneCellProcess( + Project project, + String briefDescription, + int rowIndex, + int cellIndex, + Serializable value + ) { + super(project, briefDescription); + + this.rowIndex = rowIndex; + this.cellIndex = cellIndex; + this.value = value; + } + + protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception { + Cell cell = _project.rows.get(rowIndex).getCell(cellIndex); + Column column = _project.columnModel.getColumnByCellIndex(cellIndex); + if (column == null) { + throw new Exception("No such column"); + } + + newCell = new Cell( + value, + cell != null ? cell.recon : null + ); + + String description = + "Edit single cell on row " + (rowIndex + 1) + + ", column " + column.getName(); + + Change change = new CellChange(rowIndex, cellIndex, cell, newCell); + + return new HistoryEntry( + historyEntryID, _project, description, null, change); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/ExportProjectCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/ExportProjectCommand.java index ead87d61d..aad5c2a82 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/ExportProjectCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/ExportProjectCommand.java @@ -1,104 +1,104 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.zip.GZIPOutputStream; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.tools.tar.TarEntry; -import org.apache.tools.tar.TarOutputStream; - -import com.metaweb.gridworks.ProjectManager; -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; - -public class ExportProjectCommand extends Command { - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - ProjectManager.singleton.ensureProjectSaved(project.id); - - response.setHeader("Content-Type", "application/x-gzip"); - - OutputStream os = response.getOutputStream(); - try { - gzipTarToOutputStream( - ProjectManager.singleton.getProjectDir(project.id), - os - ); - } finally { - os.close(); - } - } catch (Exception e) { - respondException(response, e); - } - } - - protected void gzipTarToOutputStream(File dir, OutputStream os) throws IOException { - GZIPOutputStream gos = new GZIPOutputStream(os); - try { - tarToOutputStream(dir, gos); - } finally { - gos.close(); - } - } - - protected void tarToOutputStream(File dir, OutputStream os) throws IOException { - TarOutputStream tos = new TarOutputStream(os); - try { - tarDir("", dir, tos); - } finally { - tos.close(); - } - } - - protected void tarDir(String relative, File dir, TarOutputStream tos) throws IOException { - File[] files = dir.listFiles(); - for (File file : files) { - if (!file.isHidden()) { - String path = relative + file.getName(); - - if (file.isDirectory()) { - tarDir(path + File.separator, file, tos); - } else { - TarEntry entry = new TarEntry(path); - - entry.setMode(TarEntry.DEFAULT_FILE_MODE); - entry.setSize(file.length()); - entry.setModTime(file.lastModified()); - - tos.putNextEntry(entry); - - copyFile(file, tos); - - tos.closeEntry(); - } - } - } - } - - protected void copyFile(File file, OutputStream os) throws IOException { - final int buffersize = 4096; - - FileInputStream fis = new FileInputStream(file); - try { - byte[] buf = new byte[buffersize]; - int count; - - while((count = fis.read(buf, 0, buffersize)) != -1) { - os.write(buf, 0, count); - } - } finally { - fis.close(); - } - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.zip.GZIPOutputStream; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.tools.tar.TarEntry; +import org.apache.tools.tar.TarOutputStream; + +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; + +public class ExportProjectCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + ProjectManager.singleton.ensureProjectSaved(project.id); + + response.setHeader("Content-Type", "application/x-gzip"); + + OutputStream os = response.getOutputStream(); + try { + gzipTarToOutputStream( + ProjectManager.singleton.getProjectDir(project.id), + os + ); + } finally { + os.close(); + } + } catch (Exception e) { + respondException(response, e); + } + } + + protected void gzipTarToOutputStream(File dir, OutputStream os) throws IOException { + GZIPOutputStream gos = new GZIPOutputStream(os); + try { + tarToOutputStream(dir, gos); + } finally { + gos.close(); + } + } + + protected void tarToOutputStream(File dir, OutputStream os) throws IOException { + TarOutputStream tos = new TarOutputStream(os); + try { + tarDir("", dir, tos); + } finally { + tos.close(); + } + } + + protected void tarDir(String relative, File dir, TarOutputStream tos) throws IOException { + File[] files = dir.listFiles(); + for (File file : files) { + if (!file.isHidden()) { + String path = relative + file.getName(); + + if (file.isDirectory()) { + tarDir(path + File.separator, file, tos); + } else { + TarEntry entry = new TarEntry(path); + + entry.setMode(TarEntry.DEFAULT_FILE_MODE); + entry.setSize(file.length()); + entry.setModTime(file.lastModified()); + + tos.putNextEntry(entry); + + copyFile(file, tos); + + tos.closeEntry(); + } + } + } + } + + protected void copyFile(File file, OutputStream os) throws IOException { + final int buffersize = 4096; + + FileInputStream fis = new FileInputStream(file); + try { + byte[] buf = new byte[buffersize]; + int count; + + while((count = fis.read(buf, 0, buffersize)) != -1) { + os.write(buf, 0, count); + } + } finally { + fis.close(); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/ExtendDataCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/ExtendDataCommand.java index 927f12036..01167c8c6 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/ExtendDataCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/ExtendDataCommand.java @@ -1,32 +1,32 @@ -package com.metaweb.gridworks.commands.edit; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.ExtendDataOperation; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class ExtendDataCommand extends EngineDependentCommand { - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - String baseColumnName = request.getParameter("baseColumnName"); - int columnInsertIndex = Integer.parseInt(request.getParameter("columnInsertIndex")); - - String jsonString = request.getParameter("extension"); - JSONObject extension = ParsingUtilities.evaluateJsonStringToObject(jsonString); - - return new ExtendDataOperation( - engineConfig, - baseColumnName, - extension, - columnInsertIndex - ); - } - -} +package com.metaweb.gridworks.commands.edit; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.ExtendDataOperation; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class ExtendDataCommand extends EngineDependentCommand { + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String baseColumnName = request.getParameter("baseColumnName"); + int columnInsertIndex = Integer.parseInt(request.getParameter("columnInsertIndex")); + + String jsonString = request.getParameter("extension"); + JSONObject extension = ParsingUtilities.evaluateJsonStringToObject(jsonString); + + return new ExtendDataOperation( + engineConfig, + baseColumnName, + extension, + columnInsertIndex + ); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/ImportProjectCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/ImportProjectCommand.java index 3f6d22078..9e11d55da 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/ImportProjectCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/ImportProjectCommand.java @@ -1,172 +1,172 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.net.URLConnection; -import java.util.Properties; -import java.util.zip.GZIPInputStream; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.commons.fileupload.FileItemIterator; -import org.apache.commons.fileupload.FileItemStream; -import org.apache.commons.fileupload.servlet.ServletFileUpload; -import org.apache.commons.fileupload.util.Streams; -import org.apache.tools.tar.TarEntry; -import org.apache.tools.tar.TarInputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.metaweb.gridworks.ProjectManager; -import com.metaweb.gridworks.ProjectMetadata; -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class ImportProjectCommand extends Command { - - final static Logger logger = LoggerFactory.getLogger("import-project_command"); - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Properties options = ParsingUtilities.parseUrlParameters(request); - - long projectID = Project.generateID(); - logger.info("Importing existing project using new ID {}", projectID); - - internalImport(request, options, projectID); - - ProjectManager.singleton.importProject(projectID); - - ProjectMetadata pm = ProjectManager.singleton.getProjectMetadata(projectID); - if (pm != null) { - if (options.containsKey("project-name")) { - String projectName = options.getProperty("project-name"); - if (projectName != null && projectName.length() > 0) { - pm.setName(projectName); - } - } - - redirect(response, "/project.html?project=" + projectID); - } else { - redirect(response, "/error.html?redirect=index.html&msg=" + - ParsingUtilities.encode("Failed to import project") - ); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - protected void internalImport( - HttpServletRequest request, - Properties options, - long projectID - ) throws Exception { - - String url = null; - - ServletFileUpload upload = new ServletFileUpload(); - - FileItemIterator iter = upload.getItemIterator(request); - while (iter.hasNext()) { - FileItemStream item = iter.next(); - String name = item.getFieldName().toLowerCase(); - InputStream stream = item.openStream(); - if (item.isFormField()) { - if (name.equals("url")) { - url = Streams.asString(stream); - } else { - options.put(name, Streams.asString(stream)); - } - } else { - String fileName = item.getName().toLowerCase(); - try { - internalImportInputStream(projectID, stream, !fileName.endsWith(".tar")); - } finally { - stream.close(); - } - } - } - - if (url != null && url.length() > 0) { - internalImportURL(request, options, projectID, url); - } - } - - protected void internalImportURL( - HttpServletRequest request, - Properties options, - long projectID, - String urlString - ) throws Exception { - URL url = new URL(urlString); - URLConnection connection = null; - - try { - connection = url.openConnection(); - connection.setConnectTimeout(5000); - connection.connect(); - } catch (Exception e) { - throw new Exception("Cannot connect to " + urlString, e); - } - - InputStream inputStream = null; - try { - inputStream = connection.getInputStream(); - } catch (Exception e) { - throw new Exception("Cannot retrieve content from " + url, e); - } - - try { - internalImportInputStream(projectID, inputStream, !urlString.endsWith(".tar")); - } finally { - inputStream.close(); - } - } - - protected void internalImportInputStream(long projectID, InputStream inputStream, boolean gziped) throws IOException { - File destDir = ProjectManager.singleton.getProjectDir(projectID); - destDir.mkdirs(); - - if (gziped) { - GZIPInputStream gis = new GZIPInputStream(inputStream); - untar(destDir, gis); - } else { - untar(destDir, inputStream); - } - } - - protected void untar(File destDir, InputStream inputStream) throws IOException { - TarInputStream tin = new TarInputStream(inputStream); - TarEntry tarEntry = null; - - while ((tarEntry = tin.getNextEntry()) != null) { - File destEntry = new File(destDir, tarEntry.getName()); - File parent = destEntry.getParentFile(); - - if (!parent.exists()) { - parent.mkdirs(); - } - - if (tarEntry.isDirectory()) { - destEntry.mkdirs(); - } else { - FileOutputStream fout = new FileOutputStream(destEntry); - try { - tin.copyEntryContents(fout); - } finally { - fout.close(); - } - } - } - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.util.Properties; +import java.util.zip.GZIPInputStream; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.fileupload.FileItemIterator; +import org.apache.commons.fileupload.FileItemStream; +import org.apache.commons.fileupload.servlet.ServletFileUpload; +import org.apache.commons.fileupload.util.Streams; +import org.apache.tools.tar.TarEntry; +import org.apache.tools.tar.TarInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.ProjectMetadata; +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class ImportProjectCommand extends Command { + + final static Logger logger = LoggerFactory.getLogger("import-project_command"); + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Properties options = ParsingUtilities.parseUrlParameters(request); + + long projectID = Project.generateID(); + logger.info("Importing existing project using new ID {}", projectID); + + internalImport(request, options, projectID); + + ProjectManager.singleton.importProject(projectID); + + ProjectMetadata pm = ProjectManager.singleton.getProjectMetadata(projectID); + if (pm != null) { + if (options.containsKey("project-name")) { + String projectName = options.getProperty("project-name"); + if (projectName != null && projectName.length() > 0) { + pm.setName(projectName); + } + } + + redirect(response, "/project.html?project=" + projectID); + } else { + redirect(response, "/error.html?redirect=index.html&msg=" + + ParsingUtilities.encode("Failed to import project") + ); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + protected void internalImport( + HttpServletRequest request, + Properties options, + long projectID + ) throws Exception { + + String url = null; + + ServletFileUpload upload = new ServletFileUpload(); + + FileItemIterator iter = upload.getItemIterator(request); + while (iter.hasNext()) { + FileItemStream item = iter.next(); + String name = item.getFieldName().toLowerCase(); + InputStream stream = item.openStream(); + if (item.isFormField()) { + if (name.equals("url")) { + url = Streams.asString(stream); + } else { + options.put(name, Streams.asString(stream)); + } + } else { + String fileName = item.getName().toLowerCase(); + try { + internalImportInputStream(projectID, stream, !fileName.endsWith(".tar")); + } finally { + stream.close(); + } + } + } + + if (url != null && url.length() > 0) { + internalImportURL(request, options, projectID, url); + } + } + + protected void internalImportURL( + HttpServletRequest request, + Properties options, + long projectID, + String urlString + ) throws Exception { + URL url = new URL(urlString); + URLConnection connection = null; + + try { + connection = url.openConnection(); + connection.setConnectTimeout(5000); + connection.connect(); + } catch (Exception e) { + throw new Exception("Cannot connect to " + urlString, e); + } + + InputStream inputStream = null; + try { + inputStream = connection.getInputStream(); + } catch (Exception e) { + throw new Exception("Cannot retrieve content from " + url, e); + } + + try { + internalImportInputStream(projectID, inputStream, !urlString.endsWith(".tar")); + } finally { + inputStream.close(); + } + } + + protected void internalImportInputStream(long projectID, InputStream inputStream, boolean gziped) throws IOException { + File destDir = ProjectManager.singleton.getProjectDir(projectID); + destDir.mkdirs(); + + if (gziped) { + GZIPInputStream gis = new GZIPInputStream(inputStream); + untar(destDir, gis); + } else { + untar(destDir, inputStream); + } + } + + protected void untar(File destDir, InputStream inputStream) throws IOException { + TarInputStream tin = new TarInputStream(inputStream); + TarEntry tarEntry = null; + + while ((tarEntry = tin.getNextEntry()) != null) { + File destEntry = new File(destDir, tarEntry.getName()); + File parent = destEntry.getParentFile(); + + if (!parent.exists()) { + parent.mkdirs(); + } + + if (tarEntry.isDirectory()) { + destEntry.mkdirs(); + } else { + FileOutputStream fout = new FileOutputStream(destEntry); + try { + tin.copyEntryContents(fout); + } finally { + fout.close(); + } + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/JoinMultiValueCellsCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/JoinMultiValueCellsCommand.java index 5238fc362..9606388a9 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/JoinMultiValueCellsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/JoinMultiValueCellsCommand.java @@ -1,36 +1,36 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.MultiValuedCellJoinOperation; -import com.metaweb.gridworks.process.Process; - -public class JoinMultiValueCellsCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - - String columnName = request.getParameter("columnName"); - String keyColumnName = request.getParameter("keyColumnName"); - String separator = request.getParameter("separator"); - - AbstractOperation op = new MultiValuedCellJoinOperation(columnName, keyColumnName, separator); - Process process = op.createProcess(project, new Properties()); - - performProcessAndRespond(request, response, project, process); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.MultiValuedCellJoinOperation; +import com.metaweb.gridworks.process.Process; + +public class JoinMultiValueCellsCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + String columnName = request.getParameter("columnName"); + String keyColumnName = request.getParameter("keyColumnName"); + String separator = request.getParameter("separator"); + + AbstractOperation op = new MultiValuedCellJoinOperation(columnName, keyColumnName, separator); + Process process = op.createProcess(project, new Properties()); + + performProcessAndRespond(request, response, project, process); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/MassEditCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/MassEditCommand.java index 3714cb4e3..a63a5c16b 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/MassEditCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/MassEditCommand.java @@ -1,29 +1,29 @@ -package com.metaweb.gridworks.commands.edit; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.MassEditOperation; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class MassEditCommand extends EngineDependentCommand { - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - String columnName = request.getParameter("columnName"); - String expression = request.getParameter("expression"); - String editsString = request.getParameter("edits"); - - return new MassEditOperation( - engineConfig, - columnName, - expression, - MassEditOperation.reconstructEdits(ParsingUtilities.evaluateJsonStringToArray(editsString)) - ); - } -} +package com.metaweb.gridworks.commands.edit; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.MassEditOperation; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class MassEditCommand extends EngineDependentCommand { + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String columnName = request.getParameter("columnName"); + String expression = request.getParameter("expression"); + String editsString = request.getParameter("edits"); + + return new MassEditOperation( + engineConfig, + columnName, + expression, + MassEditOperation.reconstructEdits(ParsingUtilities.evaluateJsonStringToArray(editsString)) + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/RemoveColumnCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/RemoveColumnCommand.java index c8ffab8d7..4119db05a 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/RemoveColumnCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/RemoveColumnCommand.java @@ -1,34 +1,34 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.ColumnRemovalOperation; -import com.metaweb.gridworks.process.Process; - -public class RemoveColumnCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - - String columnName = request.getParameter("columnName"); - - AbstractOperation op = new ColumnRemovalOperation(columnName); - Process process = op.createProcess(project, new Properties()); - - performProcessAndRespond(request, response, project, process); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.ColumnRemovalOperation; +import com.metaweb.gridworks.process.Process; + +public class RemoveColumnCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + String columnName = request.getParameter("columnName"); + + AbstractOperation op = new ColumnRemovalOperation(columnName); + Process process = op.createProcess(project, new Properties()); + + performProcessAndRespond(request, response, project, process); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/RemoveRowsCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/RemoveRowsCommand.java index 1d5f2f4aa..54e82447d 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/RemoveRowsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/RemoveRowsCommand.java @@ -1,20 +1,20 @@ -package com.metaweb.gridworks.commands.edit; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.RowRemovalOperation; - -public class RemoveRowsCommand extends EngineDependentCommand { - - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - return new RowRemovalOperation(engineConfig); - } -} +package com.metaweb.gridworks.commands.edit; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.RowRemovalOperation; + +public class RemoveRowsCommand extends EngineDependentCommand { + + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + return new RowRemovalOperation(engineConfig); + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/RenameColumnCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/RenameColumnCommand.java index c2e0e8ce0..674e708ba 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/RenameColumnCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/RenameColumnCommand.java @@ -1,35 +1,35 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.ColumnRenameOperation; -import com.metaweb.gridworks.process.Process; - -public class RenameColumnCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - - String oldColumnName = request.getParameter("oldColumnName"); - String newColumnName = request.getParameter("newColumnName"); - - AbstractOperation op = new ColumnRenameOperation(oldColumnName, newColumnName); - Process process = op.createProcess(project, new Properties()); - - performProcessAndRespond(request, response, project, process); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.ColumnRenameOperation; +import com.metaweb.gridworks.process.Process; + +public class RenameColumnCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + String oldColumnName = request.getParameter("oldColumnName"); + String newColumnName = request.getParameter("newColumnName"); + + AbstractOperation op = new ColumnRenameOperation(oldColumnName, newColumnName); + Process process = op.createProcess(project, new Properties()); + + performProcessAndRespond(request, response, project, process); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/SaveProtographCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/SaveProtographCommand.java index b391d3b9d..7906b4056 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/SaveProtographCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/SaveProtographCommand.java @@ -1,40 +1,40 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.SaveProtographOperation; -import com.metaweb.gridworks.process.Process; -import com.metaweb.gridworks.protograph.Protograph; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class SaveProtographCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - - String jsonString = request.getParameter("protograph"); - JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); - Protograph protograph = Protograph.reconstruct(json); - - AbstractOperation op = new SaveProtographOperation(protograph); - Process process = op.createProcess(project, new Properties()); - - performProcessAndRespond(request, response, project, process); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.SaveProtographOperation; +import com.metaweb.gridworks.process.Process; +import com.metaweb.gridworks.protograph.Protograph; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class SaveProtographCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + String jsonString = request.getParameter("protograph"); + JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); + Protograph protograph = Protograph.reconstruct(json); + + AbstractOperation op = new SaveProtographOperation(protograph); + Process process = op.createProcess(project, new Properties()); + + performProcessAndRespond(request, response, project, process); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/SplitColumnCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/SplitColumnCommand.java index dbdadb36a..f21d01f87 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/SplitColumnCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/SplitColumnCommand.java @@ -1,54 +1,54 @@ -package com.metaweb.gridworks.commands.edit; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONArray; -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.ColumnSplitOperation; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class SplitColumnCommand extends EngineDependentCommand { - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - String columnName = request.getParameter("columnName"); - boolean guessCellType = Boolean.parseBoolean(request.getParameter("guessCellType")); - boolean removeOriginalColumn = Boolean.parseBoolean(request.getParameter("removeOriginalColumn")); - String mode = request.getParameter("mode"); - if ("separator".equals(mode)) { - String maxColumns = request.getParameter("maxColumns"); - - return new ColumnSplitOperation( - engineConfig, - columnName, - guessCellType, - removeOriginalColumn, - request.getParameter("separator"), - Boolean.parseBoolean(request.getParameter("regex")), - maxColumns != null && maxColumns.length() > 0 ? Integer.parseInt(maxColumns) : 0 - ); - } else { - String s = request.getParameter("fieldLengths"); - - JSONArray a = ParsingUtilities.evaluateJsonStringToArray(s); - int[] fieldLengths = new int[a.length()]; - - for (int i = 0; i < fieldLengths.length; i++) { - fieldLengths[i] = a.getInt(i); - } - - return new ColumnSplitOperation( - engineConfig, - columnName, - guessCellType, - removeOriginalColumn, - fieldLengths - ); - } - } -} +package com.metaweb.gridworks.commands.edit; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONArray; +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.ColumnSplitOperation; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class SplitColumnCommand extends EngineDependentCommand { + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String columnName = request.getParameter("columnName"); + boolean guessCellType = Boolean.parseBoolean(request.getParameter("guessCellType")); + boolean removeOriginalColumn = Boolean.parseBoolean(request.getParameter("removeOriginalColumn")); + String mode = request.getParameter("mode"); + if ("separator".equals(mode)) { + String maxColumns = request.getParameter("maxColumns"); + + return new ColumnSplitOperation( + engineConfig, + columnName, + guessCellType, + removeOriginalColumn, + request.getParameter("separator"), + Boolean.parseBoolean(request.getParameter("regex")), + maxColumns != null && maxColumns.length() > 0 ? Integer.parseInt(maxColumns) : 0 + ); + } else { + String s = request.getParameter("fieldLengths"); + + JSONArray a = ParsingUtilities.evaluateJsonStringToArray(s); + int[] fieldLengths = new int[a.length()]; + + for (int i = 0; i < fieldLengths.length; i++) { + fieldLengths[i] = a.getInt(i); + } + + return new ColumnSplitOperation( + engineConfig, + columnName, + guessCellType, + removeOriginalColumn, + fieldLengths + ); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/SplitMultiValueCellsCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/SplitMultiValueCellsCommand.java index f1a12fa75..1134e47ed 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/SplitMultiValueCellsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/SplitMultiValueCellsCommand.java @@ -1,37 +1,37 @@ -package com.metaweb.gridworks.commands.edit; - - import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.MultiValuedCellSplitOperation; -import com.metaweb.gridworks.process.Process; - -public class SplitMultiValueCellsCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - - String columnName = request.getParameter("columnName"); - String keyColumnName = request.getParameter("keyColumnName"); - String separator = request.getParameter("separator"); - String mode = request.getParameter("mode"); - - AbstractOperation op = new MultiValuedCellSplitOperation(columnName, keyColumnName, separator, mode); - Process process = op.createProcess(project, new Properties()); - - performProcessAndRespond(request, response, project, process); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.edit; + + import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.MultiValuedCellSplitOperation; +import com.metaweb.gridworks.process.Process; + +public class SplitMultiValueCellsCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + String columnName = request.getParameter("columnName"); + String keyColumnName = request.getParameter("keyColumnName"); + String separator = request.getParameter("separator"); + String mode = request.getParameter("mode"); + + AbstractOperation op = new MultiValuedCellSplitOperation(columnName, keyColumnName, separator, mode); + Process process = op.createProcess(project, new Properties()); + + performProcessAndRespond(request, response, project, process); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/TextTransformCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/TextTransformCommand.java index 2ded3fc09..0c0ed5d23 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/TextTransformCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/TextTransformCommand.java @@ -1,38 +1,38 @@ -package com.metaweb.gridworks.commands.edit; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.TextTransformOperation; - -public class TextTransformCommand extends EngineDependentCommand { - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - String columnName = request.getParameter("columnName"); - String expression = request.getParameter("expression"); - String onError = request.getParameter("onError"); - boolean repeat = "true".equals(request.getParameter("repeat")); - - int repeatCount = 10; - String repeatCountString = request.getParameter("repeatCount"); - try { - repeatCount = Math.max(Math.min(Integer.parseInt(repeatCountString), 10), 0); - } catch (Exception e) { - } - - return new TextTransformOperation( - engineConfig, - columnName, - expression, - TextTransformOperation.stringToOnError(onError), - repeat, - repeatCount - ); - } -} +package com.metaweb.gridworks.commands.edit; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.TextTransformOperation; + +public class TextTransformCommand extends EngineDependentCommand { + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String columnName = request.getParameter("columnName"); + String expression = request.getParameter("expression"); + String onError = request.getParameter("onError"); + boolean repeat = "true".equals(request.getParameter("repeat")); + + int repeatCount = 10; + String repeatCountString = request.getParameter("repeatCount"); + try { + repeatCount = Math.max(Math.min(Integer.parseInt(repeatCountString), 10), 0); + } catch (Exception e) { + } + + return new TextTransformOperation( + engineConfig, + columnName, + expression, + TextTransformOperation.stringToOnError(onError), + repeat, + repeatCount + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/edit/UndoRedoCommand.java b/src/main/java/com/metaweb/gridworks/commands/edit/UndoRedoCommand.java index 9aea924be..457819402 100644 --- a/src/main/java/com/metaweb/gridworks/commands/edit/UndoRedoCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/edit/UndoRedoCommand.java @@ -1,40 +1,40 @@ -package com.metaweb.gridworks.commands.edit; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.history.HistoryProcess; -import com.metaweb.gridworks.model.Project; - -public class UndoRedoCommand extends Command { - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - Project project = getProject(request); - - long lastDoneID = -1; - String lastDoneIDString = request.getParameter("lastDoneID"); - if (lastDoneIDString != null) { - lastDoneID = Long.parseLong(lastDoneIDString); - } else { - String undoIDString = request.getParameter("undoID"); - if (undoIDString != null) { - long undoID = Long.parseLong(undoIDString); - - lastDoneID = project.history.getPrecedingEntryID(undoID); - } - } - - boolean done = lastDoneID == -1 || - project.processManager.queueProcess( - new HistoryProcess(project, lastDoneID)); - - respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }"); - } -} +package com.metaweb.gridworks.commands.edit; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.history.HistoryProcess; +import com.metaweb.gridworks.model.Project; + +public class UndoRedoCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + Project project = getProject(request); + + long lastDoneID = -1; + String lastDoneIDString = request.getParameter("lastDoneID"); + if (lastDoneIDString != null) { + lastDoneID = Long.parseLong(lastDoneIDString); + } else { + String undoIDString = request.getParameter("undoID"); + if (undoIDString != null) { + long undoID = Long.parseLong(undoIDString); + + lastDoneID = project.history.getPrecedingEntryID(undoID); + } + } + + boolean done = lastDoneID == -1 || + project.processManager.queueProcess( + new HistoryProcess(project, lastDoneID)); + + respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }"); + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/info/ComputeFacetsCommand.java b/src/main/java/com/metaweb/gridworks/commands/info/ComputeFacetsCommand.java index 7db1bf36b..7dd0ae1fd 100644 --- a/src/main/java/com/metaweb/gridworks/commands/info/ComputeFacetsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/info/ComputeFacetsCommand.java @@ -1,29 +1,29 @@ -package com.metaweb.gridworks.commands.info; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; - -public class ComputeFacetsCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - Engine engine = getEngine(request, project); - - engine.computeFacets(); - - respondJSON(response, engine); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.info; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; + +public class ComputeFacetsCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + Engine engine = getEngine(request, project); + + engine.computeFacets(); + + respondJSON(response, engine); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/info/ExportRowsCommand.java b/src/main/java/com/metaweb/gridworks/commands/info/ExportRowsCommand.java index 4cf88bbc2..6d355b0bb 100644 --- a/src/main/java/com/metaweb/gridworks/commands/info/ExportRowsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/info/ExportRowsCommand.java @@ -1,58 +1,58 @@ -package com.metaweb.gridworks.commands.info; - - import java.io.IOException; -import java.io.PrintWriter; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.exporters.Exporter; -import com.metaweb.gridworks.exporters.HtmlTableExporter; -import com.metaweb.gridworks.exporters.TripleloaderExporter; -import com.metaweb.gridworks.exporters.TsvExporter; -import com.metaweb.gridworks.exporters.XlsExporter; -import com.metaweb.gridworks.model.Project; - -public class ExportRowsCommand extends Command { - - static final protected Map s_formatToExporter = new HashMap(); - - static { - s_formatToExporter.put("tripleloader", new TripleloaderExporter()); - s_formatToExporter.put("html", new HtmlTableExporter()); - s_formatToExporter.put("xls", new XlsExporter()); - } - - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - Engine engine = getEngine(request, project); - String format = request.getParameter("format"); - - Exporter exporter = s_formatToExporter.get(format.toLowerCase()); - if (exporter == null){ - exporter = new TsvExporter(); - } - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", exporter.getContentType()); - - if (exporter.takeWriter()) { - PrintWriter writer = response.getWriter(); - exporter.export(project, new Properties(), engine, writer); - } else { - exporter.export(project, new Properties(), engine, response.getOutputStream()); - } - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.info; + + import java.io.IOException; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.exporters.Exporter; +import com.metaweb.gridworks.exporters.HtmlTableExporter; +import com.metaweb.gridworks.exporters.TripleloaderExporter; +import com.metaweb.gridworks.exporters.TsvExporter; +import com.metaweb.gridworks.exporters.XlsExporter; +import com.metaweb.gridworks.model.Project; + +public class ExportRowsCommand extends Command { + + static final protected Map s_formatToExporter = new HashMap(); + + static { + s_formatToExporter.put("tripleloader", new TripleloaderExporter()); + s_formatToExporter.put("html", new HtmlTableExporter()); + s_formatToExporter.put("xls", new XlsExporter()); + } + + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + Engine engine = getEngine(request, project); + String format = request.getParameter("format"); + + Exporter exporter = s_formatToExporter.get(format.toLowerCase()); + if (exporter == null){ + exporter = new TsvExporter(); + } + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", exporter.getContentType()); + + if (exporter.takeWriter()) { + PrintWriter writer = response.getWriter(); + exporter.export(project, new Properties(), engine, writer); + } else { + exporter.export(project, new Properties(), engine, response.getOutputStream()); + } + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/info/GetAllProjectMetadataCommand.java b/src/main/java/com/metaweb/gridworks/commands/info/GetAllProjectMetadataCommand.java index 67336ec9e..c595b422d 100644 --- a/src/main/java/com/metaweb/gridworks/commands/info/GetAllProjectMetadataCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/info/GetAllProjectMetadataCommand.java @@ -1,48 +1,48 @@ -package com.metaweb.gridworks.commands.info; - -import java.io.IOException; -import java.util.Map; -import java.util.Properties; -import java.util.Map.Entry; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.ProjectManager; -import com.metaweb.gridworks.ProjectMetadata; -import com.metaweb.gridworks.commands.Command; - -public class GetAllProjectMetadataCommand extends Command { - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - JSONWriter writer = new JSONWriter(response.getWriter()); - Properties options = new Properties(); - - writer.object(); - writer.key("projects"); - writer.object(); - Map m = ProjectManager.singleton.getAllProjectMetadata(); - for (Entry e : m.entrySet()) { - ProjectMetadata pm = e.getValue(); - if (pm != null) { - writer.key(e.getKey().toString()); - e.getValue().write(writer, options); - } - } - writer.endObject(); - writer.endObject(); - } catch (JSONException e) { - respondException(response, e); - } - } +package com.metaweb.gridworks.commands.info; + +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import java.util.Map.Entry; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.ProjectMetadata; +import com.metaweb.gridworks.commands.Command; + +public class GetAllProjectMetadataCommand extends Command { + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + JSONWriter writer = new JSONWriter(response.getWriter()); + Properties options = new Properties(); + + writer.object(); + writer.key("projects"); + writer.object(); + Map m = ProjectManager.singleton.getAllProjectMetadata(); + for (Entry e : m.entrySet()) { + ProjectMetadata pm = e.getValue(); + if (pm != null) { + writer.key(e.getKey().toString()); + e.getValue().write(writer, options); + } + } + writer.endObject(); + writer.endObject(); + } catch (JSONException e) { + respondException(response, e); + } + } } \ No newline at end of file diff --git a/src/main/java/com/metaweb/gridworks/commands/info/GetExpressionHistoryCommand.java b/src/main/java/com/metaweb/gridworks/commands/info/GetExpressionHistoryCommand.java index 794abcce0..af2de7e1e 100644 --- a/src/main/java/com/metaweb/gridworks/commands/info/GetExpressionHistoryCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/info/GetExpressionHistoryCommand.java @@ -1,61 +1,61 @@ -package com.metaweb.gridworks.commands.info; - -import java.io.IOException; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONWriter; - -import com.metaweb.gridworks.ProjectManager; -import com.metaweb.gridworks.ProjectMetadata; -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; - -public class GetExpressionHistoryCommand extends Command { - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - ProjectMetadata pm = ProjectManager.singleton.getProjectMetadata(project.id); - - List localExpressions = pm.getExpressions(); - List globalExpressions = ProjectManager.singleton.getExpressions(); - Set done = new HashSet(); - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - JSONWriter writer = new JSONWriter(response.getWriter()); - writer.object(); - writer.key("expressions"); - writer.array(); - for (String s : localExpressions) { - writer.object(); - writer.key("code"); writer.value(s); - writer.key("global"); writer.value(false); - writer.endObject(); - done.add(s); - } - for (String s : globalExpressions) { - if (!done.contains(s)) { - writer.object(); - writer.key("code"); writer.value(s); - writer.key("global"); writer.value(true); - writer.endObject(); - } - } - writer.endArray(); - writer.endObject(); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.info; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONWriter; + +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.ProjectMetadata; +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; + +public class GetExpressionHistoryCommand extends Command { + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + ProjectMetadata pm = ProjectManager.singleton.getProjectMetadata(project.id); + + List localExpressions = pm.getExpressions(); + List globalExpressions = ProjectManager.singleton.getExpressions(); + Set done = new HashSet(); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + JSONWriter writer = new JSONWriter(response.getWriter()); + writer.object(); + writer.key("expressions"); + writer.array(); + for (String s : localExpressions) { + writer.object(); + writer.key("code"); writer.value(s); + writer.key("global"); writer.value(false); + writer.endObject(); + done.add(s); + } + for (String s : globalExpressions) { + if (!done.contains(s)) { + writer.object(); + writer.key("code"); writer.value(s); + writer.key("global"); writer.value(true); + writer.endObject(); + } + } + writer.endArray(); + writer.endObject(); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/info/GetHistoryCommand.java b/src/main/java/com/metaweb/gridworks/commands/info/GetHistoryCommand.java index 0400bc55b..ca005ccd6 100644 --- a/src/main/java/com/metaweb/gridworks/commands/info/GetHistoryCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/info/GetHistoryCommand.java @@ -1,27 +1,27 @@ -package com.metaweb.gridworks.commands.info; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONException; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; - -public class GetHistoryCommand extends Command { - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - Project project = getProject(request); - try { - respondJSON(response, project.history); - } catch (JSONException e) { - respondException(response, e); - } - } - -} +package com.metaweb.gridworks.commands.info; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONException; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; + +public class GetHistoryCommand extends Command { + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + Project project = getProject(request); + try { + respondJSON(response, project.history); + } catch (JSONException e) { + respondException(response, e); + } + } + +} diff --git a/src/main/java/com/metaweb/gridworks/commands/info/GetModelsCommand.java b/src/main/java/com/metaweb/gridworks/commands/info/GetModelsCommand.java index e1cdbdff6..d762f8b53 100644 --- a/src/main/java/com/metaweb/gridworks/commands/info/GetModelsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/info/GetModelsCommand.java @@ -1,44 +1,44 @@ -package com.metaweb.gridworks.commands.info; - -import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; - -public class GetModelsCommand extends Command { - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - Project project = getProject(request); - - try { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - Properties options = new Properties(); - JSONWriter writer = new JSONWriter(response.getWriter()); - - writer.object(); - writer.key("columnModel"); project.columnModel.write(writer, options); - writer.key("protograph"); - if (project.protograph == null) { - writer.value(null); - } else { - project.protograph.write(writer, options); - } - writer.endObject(); - } catch (JSONException e) { - respondException(response, e); - } - } - -} +package com.metaweb.gridworks.commands.info; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; + +public class GetModelsCommand extends Command { + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + Project project = getProject(request); + + try { + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + Properties options = new Properties(); + JSONWriter writer = new JSONWriter(response.getWriter()); + + writer.object(); + writer.key("columnModel"); project.columnModel.write(writer, options); + writer.key("protograph"); + if (project.protograph == null) { + writer.value(null); + } else { + project.protograph.write(writer, options); + } + writer.endObject(); + } catch (JSONException e) { + respondException(response, e); + } + } + +} diff --git a/src/main/java/com/metaweb/gridworks/commands/info/GetOperationsCommand.java b/src/main/java/com/metaweb/gridworks/commands/info/GetOperationsCommand.java index 06e5b05e3..7829997e8 100644 --- a/src/main/java/com/metaweb/gridworks/commands/info/GetOperationsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/info/GetOperationsCommand.java @@ -1,50 +1,50 @@ -package com.metaweb.gridworks.commands.info; - -import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.Project; - -public class GetOperationsCommand extends Command { - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - Project project = getProject(request); - - try { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - Properties options = new Properties(); - JSONWriter writer = new JSONWriter(response.getWriter()); - - writer.object(); - writer.key("entries"); writer.array(); - - for (HistoryEntry entry : project.history.getLastPastEntries(-1)) { - writer.object(); - writer.key("description"); writer.value(entry.description); - if (entry.operation != null) { - writer.key("operation"); - entry.operation.write(writer, options); - } - writer.endObject(); - } - writer.endArray(); - writer.endObject(); - } catch (JSONException e) { - respondException(response, e); - } - } - -} +package com.metaweb.gridworks.commands.info; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.Project; + +public class GetOperationsCommand extends Command { + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + Project project = getProject(request); + + try { + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + Properties options = new Properties(); + JSONWriter writer = new JSONWriter(response.getWriter()); + + writer.object(); + writer.key("entries"); writer.array(); + + for (HistoryEntry entry : project.history.getLastPastEntries(-1)) { + writer.object(); + writer.key("description"); writer.value(entry.description); + if (entry.operation != null) { + writer.key("operation"); + entry.operation.write(writer, options); + } + writer.endObject(); + } + writer.endArray(); + writer.endObject(); + } catch (JSONException e) { + respondException(response, e); + } + } + +} diff --git a/src/main/java/com/metaweb/gridworks/commands/info/GetProcessesCommand.java b/src/main/java/com/metaweb/gridworks/commands/info/GetProcessesCommand.java index 9761036c2..9da0f5760 100644 --- a/src/main/java/com/metaweb/gridworks/commands/info/GetProcessesCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/info/GetProcessesCommand.java @@ -1,27 +1,27 @@ -package com.metaweb.gridworks.commands.info; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONException; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; - -public class GetProcessesCommand extends Command { - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - Project project = getProject(request); - - try { - respondJSON(response, project.processManager); - } catch (JSONException e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.info; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONException; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; + +public class GetProcessesCommand extends Command { + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + Project project = getProject(request); + + try { + respondJSON(response, project.processManager); + } catch (JSONException e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/info/GetProjectMetadataCommand.java b/src/main/java/com/metaweb/gridworks/commands/info/GetProjectMetadataCommand.java index 38337329d..dad117187 100644 --- a/src/main/java/com/metaweb/gridworks/commands/info/GetProjectMetadataCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/info/GetProjectMetadataCommand.java @@ -1,28 +1,28 @@ -package com.metaweb.gridworks.commands.info; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONException; - -import com.metaweb.gridworks.ProjectManager; -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; - -public class GetProjectMetadataCommand extends Command { - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - Project project = getProject(request); - - try { - respondJSON(response, ProjectManager.singleton.getProjectMetadata(project.id)); - } catch (JSONException e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.info; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONException; + +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; + +public class GetProjectMetadataCommand extends Command { + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + Project project = getProject(request); + + try { + respondJSON(response, ProjectManager.singleton.getProjectMetadata(project.id)); + } catch (JSONException e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/info/GetRowsCommand.java b/src/main/java/com/metaweb/gridworks/commands/info/GetRowsCommand.java index af1750188..09ba491b2 100644 --- a/src/main/java/com/metaweb/gridworks/commands/info/GetRowsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/info/GetRowsCommand.java @@ -1,129 +1,129 @@ -package com.metaweb.gridworks.commands.info; - -import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.util.Pool; - -public class GetRowsCommand extends Command { - - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - Engine engine = getEngine(request, project); - - int start = Math.min(project.rows.size(), Math.max(0, getIntegerParameter(request, "start", 0))); - int limit = Math.min(project.rows.size() - start, Math.max(0, getIntegerParameter(request, "limit", 20))); - - Pool pool = new Pool(); - Properties options = new Properties(); - options.put("reconCandidateOmitTypes", true); - options.put("pool", pool); - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - JSONWriter writer = new JSONWriter(response.getWriter()); - writer.object(); - - { - RowAccumulator acc = new RowAccumulator(start, limit) { - JSONWriter writer; - Properties options; - Properties extra; - - public RowAccumulator init(JSONWriter writer, Properties options) { - this.writer = writer; - this.options = options; - - this.extra = new Properties(); - this.extra.put("contextual", true); - - return this; - } - - @Override - public boolean internalVisit(int rowIndex, Row row, boolean contextual, boolean dependent) { - try { - /* - * Whatever that's in the "extra" field will be written out - * by the row as well. This is how we can customize what the row - * writes, in a limited way. - */ - if (contextual) { - options.put("extra", extra); - } else { - options.remove("extra"); - } - options.put("rowIndex", rowIndex); - - row.write(writer, options); - } catch (JSONException e) { - } - return false; - } - }.init(writer, options); - - FilteredRows filteredRows = engine.getAllFilteredRows(true); - - writer.key("rows"); writer.array(); - filteredRows.accept(project, acc); - writer.endArray(); - - writer.key("filtered"); writer.value(acc.total); - } - - writer.key("start"); writer.value(start); - writer.key("limit"); writer.value(limit); - writer.key("total"); writer.value(project.rows.size()); - writer.key("pool"); pool.write(writer, options); - - writer.endObject(); - } catch (Exception e) { - respondException(response, e); - } - } - - static protected class RowAccumulator implements RowVisitor { - final public int start; - final public int limit; - - public int total; - - public RowAccumulator(int start, int limit) { - this.start = start; - this.limit = limit; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean contextual, boolean dependent) { - boolean r = false; - - if (total >= start && total < start + limit) { - r = internalVisit(rowIndex, row, contextual, dependent); - } - if (!contextual) { - total++; - } - return r; - } - - protected boolean internalVisit(int rowIndex, Row row, boolean contextual, boolean dependent) { - return false; - } - } -} +package com.metaweb.gridworks.commands.info; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.util.Pool; + +public class GetRowsCommand extends Command { + + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + Engine engine = getEngine(request, project); + + int start = Math.min(project.rows.size(), Math.max(0, getIntegerParameter(request, "start", 0))); + int limit = Math.min(project.rows.size() - start, Math.max(0, getIntegerParameter(request, "limit", 20))); + + Pool pool = new Pool(); + Properties options = new Properties(); + options.put("reconCandidateOmitTypes", true); + options.put("pool", pool); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + JSONWriter writer = new JSONWriter(response.getWriter()); + writer.object(); + + { + RowAccumulator acc = new RowAccumulator(start, limit) { + JSONWriter writer; + Properties options; + Properties extra; + + public RowAccumulator init(JSONWriter writer, Properties options) { + this.writer = writer; + this.options = options; + + this.extra = new Properties(); + this.extra.put("contextual", true); + + return this; + } + + @Override + public boolean internalVisit(int rowIndex, Row row, boolean contextual, boolean dependent) { + try { + /* + * Whatever that's in the "extra" field will be written out + * by the row as well. This is how we can customize what the row + * writes, in a limited way. + */ + if (contextual) { + options.put("extra", extra); + } else { + options.remove("extra"); + } + options.put("rowIndex", rowIndex); + + row.write(writer, options); + } catch (JSONException e) { + } + return false; + } + }.init(writer, options); + + FilteredRows filteredRows = engine.getAllFilteredRows(true); + + writer.key("rows"); writer.array(); + filteredRows.accept(project, acc); + writer.endArray(); + + writer.key("filtered"); writer.value(acc.total); + } + + writer.key("start"); writer.value(start); + writer.key("limit"); writer.value(limit); + writer.key("total"); writer.value(project.rows.size()); + writer.key("pool"); pool.write(writer, options); + + writer.endObject(); + } catch (Exception e) { + respondException(response, e); + } + } + + static protected class RowAccumulator implements RowVisitor { + final public int start; + final public int limit; + + public int total; + + public RowAccumulator(int start, int limit) { + this.start = start; + this.limit = limit; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean contextual, boolean dependent) { + boolean r = false; + + if (total >= start && total < start + limit) { + r = internalVisit(rowIndex, row, contextual, dependent); + } + if (!contextual) { + total++; + } + return r; + } + + protected boolean internalVisit(int rowIndex, Row row, boolean contextual, boolean dependent) { + return false; + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/recon/ReconDiscardJudgmentsCommand.java b/src/main/java/com/metaweb/gridworks/commands/recon/ReconDiscardJudgmentsCommand.java index cf43999d3..1c2086865 100644 --- a/src/main/java/com/metaweb/gridworks/commands/recon/ReconDiscardJudgmentsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/recon/ReconDiscardJudgmentsCommand.java @@ -1,21 +1,21 @@ -package com.metaweb.gridworks.commands.recon; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.ReconDiscardJudgmentsOperation; - -public class ReconDiscardJudgmentsCommand extends EngineDependentCommand { - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - String columnName = request.getParameter("columnName"); - - return new ReconDiscardJudgmentsOperation(engineConfig, columnName); - } -} +package com.metaweb.gridworks.commands.recon; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.ReconDiscardJudgmentsOperation; + +public class ReconDiscardJudgmentsCommand extends EngineDependentCommand { + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String columnName = request.getParameter("columnName"); + + return new ReconDiscardJudgmentsOperation(engineConfig, columnName); + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/recon/ReconJudgeOneCellCommand.java b/src/main/java/com/metaweb/gridworks/commands/recon/ReconJudgeOneCellCommand.java index 70ca797dd..f128903c3 100644 --- a/src/main/java/com/metaweb/gridworks/commands/recon/ReconJudgeOneCellCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/recon/ReconJudgeOneCellCommand.java @@ -1,205 +1,205 @@ -package com.metaweb.gridworks.commands.recon; - -import java.io.IOException; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONWriter; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.model.ReconStats; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.model.changes.CellChange; -import com.metaweb.gridworks.model.changes.ReconChange; -import com.metaweb.gridworks.process.QuickHistoryEntryProcess; -import com.metaweb.gridworks.util.Pool; - -public class ReconJudgeOneCellCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - - int rowIndex = Integer.parseInt(request.getParameter("row")); - int cellIndex = Integer.parseInt(request.getParameter("cell")); - Judgment judgment = Recon.stringToJudgment(request.getParameter("judgment")); - - ReconCandidate match = null; - String topicID = request.getParameter("topicID"); - if (topicID != null) { - String scoreString = request.getParameter("score"); - - match = new ReconCandidate( - topicID, - request.getParameter("topicGUID"), - request.getParameter("topicName"), - request.getParameter("types").split(","), - scoreString != null ? Double.parseDouble(scoreString) : 100 - ); - } - - JudgeOneCellProcess process = new JudgeOneCellProcess( - project, - "Judge one cell's recon result", - judgment, - rowIndex, - cellIndex, - match - ); - - HistoryEntry historyEntry = project.processManager.queueProcess(process); - if (historyEntry != null) { - /* - * If the process is done, write back the cell's data so that the - * client side can update its UI right away. - */ - JSONWriter writer = new JSONWriter(response.getWriter()); - - Pool pool = new Pool(); - Properties options = new Properties(); - options.put("pool", pool); - - writer.object(); - writer.key("code"); writer.value("ok"); - writer.key("historyEntry"); historyEntry.write(writer, options); - writer.key("cell"); process.newCell.write(writer, options); - writer.key("pool"); pool.write(writer, options); - writer.endObject(); - } else { - respond(response, "{ \"code\" : \"pending\" }"); - } - } catch (Exception e) { - respondException(response, e); - } - } - - protected static class JudgeOneCellProcess extends QuickHistoryEntryProcess { - - final int rowIndex; - final int cellIndex; - final Judgment judgment; - final ReconCandidate match; - Cell newCell; - - JudgeOneCellProcess( - Project project, - String briefDescription, - Judgment judgment, - int rowIndex, - int cellIndex, - ReconCandidate match - ) { - super(project, briefDescription); - - this.judgment = judgment; - this.rowIndex = rowIndex; - this.cellIndex = cellIndex; - this.match = match; - } - - protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception { - Cell cell = _project.rows.get(rowIndex).getCell(cellIndex); - if (cell == null || !ExpressionUtils.isNonBlankData(cell.value)) { - throw new Exception("Cell is blank or error"); - } - - Column column = _project.columnModel.getColumnByCellIndex(cellIndex); - if (column == null) { - throw new Exception("No such column"); - } - - Judgment oldJudgment = cell.recon == null ? Judgment.None : cell.recon.judgment; - - newCell = new Cell( - cell.value, - cell.recon == null ? new Recon(historyEntryID) : cell.recon.dup(historyEntryID) - ); - - String cellDescription = - "single cell on row " + (rowIndex + 1) + - ", column " + column.getName() + - ", containing \"" + cell.value + "\""; - - String description = null; - - newCell.recon.matchRank = -1; - newCell.recon.judgmentAction = "single"; - newCell.recon.judgmentBatchSize = 1; - - if (judgment == Judgment.None) { - newCell.recon.judgment = Recon.Judgment.None; - newCell.recon.match = null; - - description = "Discard recon judgment for " + cellDescription; - } else if (judgment == Judgment.New) { - newCell.recon.judgment = Recon.Judgment.New; - newCell.recon.match = null; - - description = "Mark to create new topic for " + cellDescription; - } else { - newCell.recon.judgment = Recon.Judgment.Matched; - newCell.recon.match = this.match; - - for (int m = 0; m < newCell.recon.candidates.size(); m++) { - if (newCell.recon.candidates.get(m).topicGUID.equals(this.match.topicGUID)) { - newCell.recon.matchRank = m; - break; - } - } - - description = "Match " + this.match.topicName + - " (" + match.topicID + ") to " + - cellDescription; - } - - ReconStats stats = column.getReconStats(); - if (stats == null) { - stats = ReconStats.create(_project, cellIndex); - } else { - int newChange = 0; - int matchChange = 0; - - if (oldJudgment == Judgment.New) { - newChange--; - } - if (oldJudgment == Judgment.Matched) { - matchChange--; - } - if (newCell.recon.judgment == Judgment.New) { - newChange++; - } - if (newCell.recon.judgment == Judgment.Matched) { - matchChange++; - } - - stats = new ReconStats( - stats.nonBlanks, - stats.newTopics + newChange, - stats.matchedTopics + matchChange); - } - - Change change = new ReconChange( - new CellChange(rowIndex, cellIndex, cell, newCell), - column.getName(), - column.getReconConfig(), - stats - ); - - return new HistoryEntry( - historyEntryID, _project, description, null, change); - } - } -} +package com.metaweb.gridworks.commands.recon; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONWriter; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.ReconStats; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.model.changes.CellChange; +import com.metaweb.gridworks.model.changes.ReconChange; +import com.metaweb.gridworks.process.QuickHistoryEntryProcess; +import com.metaweb.gridworks.util.Pool; + +public class ReconJudgeOneCellCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + int rowIndex = Integer.parseInt(request.getParameter("row")); + int cellIndex = Integer.parseInt(request.getParameter("cell")); + Judgment judgment = Recon.stringToJudgment(request.getParameter("judgment")); + + ReconCandidate match = null; + String topicID = request.getParameter("topicID"); + if (topicID != null) { + String scoreString = request.getParameter("score"); + + match = new ReconCandidate( + topicID, + request.getParameter("topicGUID"), + request.getParameter("topicName"), + request.getParameter("types").split(","), + scoreString != null ? Double.parseDouble(scoreString) : 100 + ); + } + + JudgeOneCellProcess process = new JudgeOneCellProcess( + project, + "Judge one cell's recon result", + judgment, + rowIndex, + cellIndex, + match + ); + + HistoryEntry historyEntry = project.processManager.queueProcess(process); + if (historyEntry != null) { + /* + * If the process is done, write back the cell's data so that the + * client side can update its UI right away. + */ + JSONWriter writer = new JSONWriter(response.getWriter()); + + Pool pool = new Pool(); + Properties options = new Properties(); + options.put("pool", pool); + + writer.object(); + writer.key("code"); writer.value("ok"); + writer.key("historyEntry"); historyEntry.write(writer, options); + writer.key("cell"); process.newCell.write(writer, options); + writer.key("pool"); pool.write(writer, options); + writer.endObject(); + } else { + respond(response, "{ \"code\" : \"pending\" }"); + } + } catch (Exception e) { + respondException(response, e); + } + } + + protected static class JudgeOneCellProcess extends QuickHistoryEntryProcess { + + final int rowIndex; + final int cellIndex; + final Judgment judgment; + final ReconCandidate match; + Cell newCell; + + JudgeOneCellProcess( + Project project, + String briefDescription, + Judgment judgment, + int rowIndex, + int cellIndex, + ReconCandidate match + ) { + super(project, briefDescription); + + this.judgment = judgment; + this.rowIndex = rowIndex; + this.cellIndex = cellIndex; + this.match = match; + } + + protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception { + Cell cell = _project.rows.get(rowIndex).getCell(cellIndex); + if (cell == null || !ExpressionUtils.isNonBlankData(cell.value)) { + throw new Exception("Cell is blank or error"); + } + + Column column = _project.columnModel.getColumnByCellIndex(cellIndex); + if (column == null) { + throw new Exception("No such column"); + } + + Judgment oldJudgment = cell.recon == null ? Judgment.None : cell.recon.judgment; + + newCell = new Cell( + cell.value, + cell.recon == null ? new Recon(historyEntryID) : cell.recon.dup(historyEntryID) + ); + + String cellDescription = + "single cell on row " + (rowIndex + 1) + + ", column " + column.getName() + + ", containing \"" + cell.value + "\""; + + String description = null; + + newCell.recon.matchRank = -1; + newCell.recon.judgmentAction = "single"; + newCell.recon.judgmentBatchSize = 1; + + if (judgment == Judgment.None) { + newCell.recon.judgment = Recon.Judgment.None; + newCell.recon.match = null; + + description = "Discard recon judgment for " + cellDescription; + } else if (judgment == Judgment.New) { + newCell.recon.judgment = Recon.Judgment.New; + newCell.recon.match = null; + + description = "Mark to create new topic for " + cellDescription; + } else { + newCell.recon.judgment = Recon.Judgment.Matched; + newCell.recon.match = this.match; + + for (int m = 0; m < newCell.recon.candidates.size(); m++) { + if (newCell.recon.candidates.get(m).topicGUID.equals(this.match.topicGUID)) { + newCell.recon.matchRank = m; + break; + } + } + + description = "Match " + this.match.topicName + + " (" + match.topicID + ") to " + + cellDescription; + } + + ReconStats stats = column.getReconStats(); + if (stats == null) { + stats = ReconStats.create(_project, cellIndex); + } else { + int newChange = 0; + int matchChange = 0; + + if (oldJudgment == Judgment.New) { + newChange--; + } + if (oldJudgment == Judgment.Matched) { + matchChange--; + } + if (newCell.recon.judgment == Judgment.New) { + newChange++; + } + if (newCell.recon.judgment == Judgment.Matched) { + matchChange++; + } + + stats = new ReconStats( + stats.nonBlanks, + stats.newTopics + newChange, + stats.matchedTopics + matchChange); + } + + Change change = new ReconChange( + new CellChange(rowIndex, cellIndex, cell, newCell), + column.getName(), + column.getReconConfig(), + stats + ); + + return new HistoryEntry( + historyEntryID, _project, description, null, change); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/recon/ReconJudgeSimilarCellsCommand.java b/src/main/java/com/metaweb/gridworks/commands/recon/ReconJudgeSimilarCellsCommand.java index 07fd1bc71..403907871 100644 --- a/src/main/java/com/metaweb/gridworks/commands/recon/ReconJudgeSimilarCellsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/recon/ReconJudgeSimilarCellsCommand.java @@ -1,50 +1,50 @@ -package com.metaweb.gridworks.commands.recon; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.operations.ReconJudgeSimilarCellsOperation; - -public class ReconJudgeSimilarCellsCommand extends EngineDependentCommand { - - @Override - protected AbstractOperation createOperation( - Project project, HttpServletRequest request, JSONObject engineConfig) throws Exception { - - String columnName = request.getParameter("columnName"); - String similarValue = request.getParameter("similarValue"); - Judgment judgment = Recon.stringToJudgment(request.getParameter("judgment")); - - ReconCandidate match = null; - String topicID = request.getParameter("topicID"); - if (topicID != null) { - String scoreString = request.getParameter("score"); - - match = new ReconCandidate( - topicID, - request.getParameter("topicGUID"), - request.getParameter("topicName"), - request.getParameter("types").split(","), - scoreString != null ? Double.parseDouble(scoreString) : 100 - ); - } - - String shareNewTopics = request.getParameter("shareNewTopics"); - - return new ReconJudgeSimilarCellsOperation( - engineConfig, - columnName, - similarValue, - judgment, - match, - "true".equals(shareNewTopics) - ); - } -} +package com.metaweb.gridworks.commands.recon; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.operations.ReconJudgeSimilarCellsOperation; + +public class ReconJudgeSimilarCellsCommand extends EngineDependentCommand { + + @Override + protected AbstractOperation createOperation( + Project project, HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String columnName = request.getParameter("columnName"); + String similarValue = request.getParameter("similarValue"); + Judgment judgment = Recon.stringToJudgment(request.getParameter("judgment")); + + ReconCandidate match = null; + String topicID = request.getParameter("topicID"); + if (topicID != null) { + String scoreString = request.getParameter("score"); + + match = new ReconCandidate( + topicID, + request.getParameter("topicGUID"), + request.getParameter("topicName"), + request.getParameter("types").split(","), + scoreString != null ? Double.parseDouble(scoreString) : 100 + ); + } + + String shareNewTopics = request.getParameter("shareNewTopics"); + + return new ReconJudgeSimilarCellsOperation( + engineConfig, + columnName, + similarValue, + judgment, + match, + "true".equals(shareNewTopics) + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/recon/ReconMarkNewTopicsCommand.java b/src/main/java/com/metaweb/gridworks/commands/recon/ReconMarkNewTopicsCommand.java index 391e23737..6613ecdf1 100644 --- a/src/main/java/com/metaweb/gridworks/commands/recon/ReconMarkNewTopicsCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/recon/ReconMarkNewTopicsCommand.java @@ -1,24 +1,24 @@ -package com.metaweb.gridworks.commands.recon; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.ReconMarkNewTopicsOperation; - -public class ReconMarkNewTopicsCommand extends EngineDependentCommand { - - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - return new ReconMarkNewTopicsOperation( - engineConfig, - request.getParameter("columnName"), - "true".equals(request.getParameter("shareNewTopics")) - ); - } -} +package com.metaweb.gridworks.commands.recon; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.ReconMarkNewTopicsOperation; + +public class ReconMarkNewTopicsCommand extends EngineDependentCommand { + + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + return new ReconMarkNewTopicsOperation( + engineConfig, + request.getParameter("columnName"), + "true".equals(request.getParameter("shareNewTopics")) + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/recon/ReconMatchBestCandidatesCommand.java b/src/main/java/com/metaweb/gridworks/commands/recon/ReconMatchBestCandidatesCommand.java index abd6dbd57..a9ea558a8 100644 --- a/src/main/java/com/metaweb/gridworks/commands/recon/ReconMatchBestCandidatesCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/recon/ReconMatchBestCandidatesCommand.java @@ -1,22 +1,22 @@ -package com.metaweb.gridworks.commands.recon; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.ReconMatchBestCandidatesOperation; - -public class ReconMatchBestCandidatesCommand extends EngineDependentCommand { - - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - String columnName = request.getParameter("columnName"); - - return new ReconMatchBestCandidatesOperation(engineConfig, columnName); - } -} +package com.metaweb.gridworks.commands.recon; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.ReconMatchBestCandidatesOperation; + +public class ReconMatchBestCandidatesCommand extends EngineDependentCommand { + + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String columnName = request.getParameter("columnName"); + + return new ReconMatchBestCandidatesOperation(engineConfig, columnName); + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/recon/ReconMatchSpecificTopicCommand.java b/src/main/java/com/metaweb/gridworks/commands/recon/ReconMatchSpecificTopicCommand.java index c42fe79ba..e00a82ee8 100644 --- a/src/main/java/com/metaweb/gridworks/commands/recon/ReconMatchSpecificTopicCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/recon/ReconMatchSpecificTopicCommand.java @@ -1,30 +1,30 @@ -package com.metaweb.gridworks.commands.recon; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.operations.ReconMatchSpecificTopicOperation; - -public class ReconMatchSpecificTopicCommand extends EngineDependentCommand { - - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - String columnName = request.getParameter("columnName"); - ReconCandidate match = new ReconCandidate( - request.getParameter("topicID"), - request.getParameter("topicGUID"), - request.getParameter("topicName"), - request.getParameter("types").split(","), - 100 - ); - - return new ReconMatchSpecificTopicOperation(engineConfig, columnName, match); - } -} +package com.metaweb.gridworks.commands.recon; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.operations.ReconMatchSpecificTopicOperation; + +public class ReconMatchSpecificTopicCommand extends EngineDependentCommand { + + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String columnName = request.getParameter("columnName"); + ReconCandidate match = new ReconCandidate( + request.getParameter("topicID"), + request.getParameter("topicGUID"), + request.getParameter("topicName"), + request.getParameter("types").split(","), + 100 + ); + + return new ReconMatchSpecificTopicOperation(engineConfig, columnName, match); + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/recon/ReconcileCommand.java b/src/main/java/com/metaweb/gridworks/commands/recon/ReconcileCommand.java index f61c99ef5..984acbe79 100644 --- a/src/main/java/com/metaweb/gridworks/commands/recon/ReconcileCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/recon/ReconcileCommand.java @@ -1,28 +1,28 @@ -package com.metaweb.gridworks.commands.recon; - -import javax.servlet.http.HttpServletRequest; - -import org.json.JSONObject; -import org.json.JSONTokener; - -import com.metaweb.gridworks.commands.EngineDependentCommand; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.recon.ReconConfig; -import com.metaweb.gridworks.operations.ReconOperation; - -public class ReconcileCommand extends EngineDependentCommand { - - @Override - protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { - - String columnName = request.getParameter("columnName"); - String configString = request.getParameter("config"); - - JSONTokener t = new JSONTokener(configString); - JSONObject config = (JSONObject) t.nextValue(); - - return new ReconOperation(engineConfig, columnName, ReconConfig.reconstruct(config)); - } -} +package com.metaweb.gridworks.commands.recon; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; +import org.json.JSONTokener; + +import com.metaweb.gridworks.commands.EngineDependentCommand; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.recon.ReconConfig; +import com.metaweb.gridworks.operations.ReconOperation; + +public class ReconcileCommand extends EngineDependentCommand { + + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String columnName = request.getParameter("columnName"); + String configString = request.getParameter("config"); + + JSONTokener t = new JSONTokener(configString); + JSONObject config = (JSONObject) t.nextValue(); + + return new ReconOperation(engineConfig, columnName, ReconConfig.reconstruct(config)); + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/util/CancelProcessesCommand.java b/src/main/java/com/metaweb/gridworks/commands/util/CancelProcessesCommand.java index 203ff7903..eb067b0b9 100644 --- a/src/main/java/com/metaweb/gridworks/commands/util/CancelProcessesCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/util/CancelProcessesCommand.java @@ -1,29 +1,29 @@ -package com.metaweb.gridworks.commands.util; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; - -public class CancelProcessesCommand extends Command { - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - project.processManager.cancelAll(); - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - response.getWriter().write("{ \"code\" : \"ok\" }"); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.util; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; + +public class CancelProcessesCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + project.processManager.cancelAll(); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + response.getWriter().write("{ \"code\" : \"ok\" }"); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/util/GetExpressionLanguageInfoCommand.java b/src/main/java/com/metaweb/gridworks/commands/util/GetExpressionLanguageInfoCommand.java index 53abd67c5..5b3c743a1 100644 --- a/src/main/java/com/metaweb/gridworks/commands/util/GetExpressionLanguageInfoCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/util/GetExpressionLanguageInfoCommand.java @@ -1,58 +1,58 @@ -package com.metaweb.gridworks.commands.util; - -import java.io.IOException; -import java.util.Properties; -import java.util.Map.Entry; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONWriter; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.gel.Control; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class GetExpressionLanguageInfoCommand extends Command { - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - JSONWriter writer = new JSONWriter(response.getWriter()); - Properties options = new Properties(); - - writer.object(); - - writer.key("functions"); - writer.object(); - { - for (Entry entry : ControlFunctionRegistry.getFunctionMapping()) { - writer.key(entry.getKey()); - entry.getValue().write(writer, options); - } - } - writer.endObject(); - - writer.key("controls"); - writer.object(); - { - for (Entry entry : ControlFunctionRegistry.getControlMapping()) { - writer.key(entry.getKey()); - entry.getValue().write(writer, options); - } - } - writer.endObject(); - - writer.endObject(); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.util; + +import java.io.IOException; +import java.util.Properties; +import java.util.Map.Entry; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONWriter; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.gel.Control; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class GetExpressionLanguageInfoCommand extends Command { + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + JSONWriter writer = new JSONWriter(response.getWriter()); + Properties options = new Properties(); + + writer.object(); + + writer.key("functions"); + writer.object(); + { + for (Entry entry : ControlFunctionRegistry.getFunctionMapping()) { + writer.key(entry.getKey()); + entry.getValue().write(writer, options); + } + } + writer.endObject(); + + writer.key("controls"); + writer.object(); + { + for (Entry entry : ControlFunctionRegistry.getControlMapping()) { + writer.key(entry.getKey()); + entry.getValue().write(writer, options); + } + } + writer.endObject(); + + writer.endObject(); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/util/GuessTypesOfColumnCommand.java b/src/main/java/com/metaweb/gridworks/commands/util/GuessTypesOfColumnCommand.java index 3c725db1a..8bfbe47bb 100644 --- a/src/main/java/com/metaweb/gridworks/commands/util/GuessTypesOfColumnCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/util/GuessTypesOfColumnCommand.java @@ -1,219 +1,219 @@ -package com.metaweb.gridworks.commands.util; - -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; -import java.net.URL; -import java.net.URLConnection; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class GuessTypesOfColumnCommand extends Command { - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - String columnName = request.getParameter("columnName"); - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - JSONWriter writer = new JSONWriter(response.getWriter()); - writer.object(); - - Column column = project.columnModel.getColumnByName(columnName); - if (column == null) { - writer.key("code"); writer.value("error"); - writer.key("message"); writer.value("No such column"); - } else { - try { - writer.key("code"); writer.value("ok"); - writer.key("types"); writer.array(); - - List typeGroups = guessTypes(project, column); - for (TypeGroup tg : typeGroups) { - writer.object(); - writer.key("id"); writer.value(tg.id); - writer.key("name"); writer.value(tg.name); - writer.key("score"); writer.value(tg.score); - writer.key("count"); writer.value(tg.count); - writer.endObject(); - } - - writer.endArray(); - } catch (Exception e) { - writer.key("code"); writer.value("error"); - } - } - - writer.endObject(); - } catch (Exception e) { - respondException(response, e); - } - } - - final static int s_sampleSize = 20; - - /** - * Run relevance searches for the first n cells in the given column and - * count the types of the results. Return a sorted list of types, from most - * frequent to least. - * - * @param project - * @param column - * @return - */ - protected List guessTypes(Project project, Column column) { - Map map = new HashMap(); - - int cellIndex = column.getCellIndex(); - - List samples = new ArrayList(s_sampleSize); - Set sampleSet = new HashSet(); - - for (Row row : project.rows) { - Object value = row.getCellValue(cellIndex); - if (ExpressionUtils.isNonBlankData(value)) { - String s = value.toString().trim(); - if (!sampleSet.contains(s)) { - samples.add(s); - sampleSet.add(s); - if (samples.size() >= s_sampleSize) { - break; - } - } - } - } - - try { - StringWriter stringWriter = new StringWriter(); - JSONWriter jsonWriter = new JSONWriter(stringWriter); - - jsonWriter.object(); - for (int i = 0; i < samples.size(); i++) { - jsonWriter.key("q" + i + ":search"); - jsonWriter.object(); - - jsonWriter.key("query"); jsonWriter.value(samples.get(i)); - jsonWriter.key("limit"); jsonWriter.value(3); - - jsonWriter.endObject(); - } - jsonWriter.endObject(); - - StringBuffer sb = new StringBuffer(1024); - sb.append("http://api.freebase.com/api/service/search?queries="); - sb.append(ParsingUtilities.encode(stringWriter.toString())); - - URL url = new URL(sb.toString()); - URLConnection connection = url.openConnection(); - connection.setConnectTimeout(5000); - connection.connect(); - - InputStream is = connection.getInputStream(); - try { - String s = ParsingUtilities.inputStreamToString(is); - JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); - - for (int i = 0; i < samples.size(); i++) { - String key = "q" + i + ":search"; - if (!o.has(key)) { - continue; - } - - JSONObject o2 = o.getJSONObject(key); - if (!(o2.has("result"))) { - continue; - } - - JSONArray results = o2.getJSONArray("result"); - int count = results.length(); - - for (int j = 0; j < count; j++) { - JSONObject result = results.getJSONObject(j); - double score = 1.0 / (1 + j); // score by each result's rank - - JSONArray types = result.getJSONArray("type"); - int typeCount = types.length(); - - for (int t = 0; t < typeCount; t++) { - JSONObject type = types.getJSONObject(t); - String id = type.getString("id"); - if (id.equals("/common/topic") || - id.equals("/base/ontologies/ontology_instance") || - (id.startsWith("/base/") && id.endsWith("/topic")) || - id.startsWith("/user/") || - id.startsWith("/freebase/") - ) { - continue; - } - - if (map.containsKey(id)) { - TypeGroup tg = map.get(id); - tg.score += score; - tg.count++; - } else { - map.put(id, new TypeGroup(id, type.getString("name"), score)); - } - } - } - } - } finally { - is.close(); - } - } catch (Exception e) { - e.printStackTrace(); - } - - List types = new ArrayList(map.values()); - Collections.sort(types, new Comparator() { - public int compare(TypeGroup o1, TypeGroup o2) { - int c = Math.min(s_sampleSize, o2.count) - Math.min(s_sampleSize, o1.count); - if (c != 0) { - return c; - } - return (int) Math.signum(o2.score / o2.count - o1.score / o1.count); - } - }); - - return types; - } - - static protected class TypeGroup { - String id; - String name; - int count; - double score; - - TypeGroup(String id, String name, double score) { - this.id = id; - this.name = name; - this.score = score; - this.count = 1; - } - } -} +package com.metaweb.gridworks.commands.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class GuessTypesOfColumnCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + String columnName = request.getParameter("columnName"); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + JSONWriter writer = new JSONWriter(response.getWriter()); + writer.object(); + + Column column = project.columnModel.getColumnByName(columnName); + if (column == null) { + writer.key("code"); writer.value("error"); + writer.key("message"); writer.value("No such column"); + } else { + try { + writer.key("code"); writer.value("ok"); + writer.key("types"); writer.array(); + + List typeGroups = guessTypes(project, column); + for (TypeGroup tg : typeGroups) { + writer.object(); + writer.key("id"); writer.value(tg.id); + writer.key("name"); writer.value(tg.name); + writer.key("score"); writer.value(tg.score); + writer.key("count"); writer.value(tg.count); + writer.endObject(); + } + + writer.endArray(); + } catch (Exception e) { + writer.key("code"); writer.value("error"); + } + } + + writer.endObject(); + } catch (Exception e) { + respondException(response, e); + } + } + + final static int s_sampleSize = 20; + + /** + * Run relevance searches for the first n cells in the given column and + * count the types of the results. Return a sorted list of types, from most + * frequent to least. + * + * @param project + * @param column + * @return + */ + protected List guessTypes(Project project, Column column) { + Map map = new HashMap(); + + int cellIndex = column.getCellIndex(); + + List samples = new ArrayList(s_sampleSize); + Set sampleSet = new HashSet(); + + for (Row row : project.rows) { + Object value = row.getCellValue(cellIndex); + if (ExpressionUtils.isNonBlankData(value)) { + String s = value.toString().trim(); + if (!sampleSet.contains(s)) { + samples.add(s); + sampleSet.add(s); + if (samples.size() >= s_sampleSize) { + break; + } + } + } + } + + try { + StringWriter stringWriter = new StringWriter(); + JSONWriter jsonWriter = new JSONWriter(stringWriter); + + jsonWriter.object(); + for (int i = 0; i < samples.size(); i++) { + jsonWriter.key("q" + i + ":search"); + jsonWriter.object(); + + jsonWriter.key("query"); jsonWriter.value(samples.get(i)); + jsonWriter.key("limit"); jsonWriter.value(3); + + jsonWriter.endObject(); + } + jsonWriter.endObject(); + + StringBuffer sb = new StringBuffer(1024); + sb.append("http://api.freebase.com/api/service/search?queries="); + sb.append(ParsingUtilities.encode(stringWriter.toString())); + + URL url = new URL(sb.toString()); + URLConnection connection = url.openConnection(); + connection.setConnectTimeout(5000); + connection.connect(); + + InputStream is = connection.getInputStream(); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + + for (int i = 0; i < samples.size(); i++) { + String key = "q" + i + ":search"; + if (!o.has(key)) { + continue; + } + + JSONObject o2 = o.getJSONObject(key); + if (!(o2.has("result"))) { + continue; + } + + JSONArray results = o2.getJSONArray("result"); + int count = results.length(); + + for (int j = 0; j < count; j++) { + JSONObject result = results.getJSONObject(j); + double score = 1.0 / (1 + j); // score by each result's rank + + JSONArray types = result.getJSONArray("type"); + int typeCount = types.length(); + + for (int t = 0; t < typeCount; t++) { + JSONObject type = types.getJSONObject(t); + String id = type.getString("id"); + if (id.equals("/common/topic") || + id.equals("/base/ontologies/ontology_instance") || + (id.startsWith("/base/") && id.endsWith("/topic")) || + id.startsWith("/user/") || + id.startsWith("/freebase/") + ) { + continue; + } + + if (map.containsKey(id)) { + TypeGroup tg = map.get(id); + tg.score += score; + tg.count++; + } else { + map.put(id, new TypeGroup(id, type.getString("name"), score)); + } + } + } + } + } finally { + is.close(); + } + } catch (Exception e) { + e.printStackTrace(); + } + + List types = new ArrayList(map.values()); + Collections.sort(types, new Comparator() { + public int compare(TypeGroup o1, TypeGroup o2) { + int c = Math.min(s_sampleSize, o2.count) - Math.min(s_sampleSize, o1.count); + if (c != 0) { + return c; + } + return (int) Math.signum(o2.score / o2.count - o1.score / o1.count); + } + }); + + return types; + } + + static protected class TypeGroup { + String id; + String name; + int count; + double score; + + TypeGroup(String id, String name, double score) { + this.id = id; + this.name = name; + this.score = score; + this.count = 1; + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/util/LogExpressionCommand.java b/src/main/java/com/metaweb/gridworks/commands/util/LogExpressionCommand.java index cc810a457..02d71231f 100644 --- a/src/main/java/com/metaweb/gridworks/commands/util/LogExpressionCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/util/LogExpressionCommand.java @@ -1,31 +1,31 @@ -package com.metaweb.gridworks.commands.util; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; - -public class LogExpressionCommand extends Command { - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - String expression = request.getParameter("expression"); - - project.getMetadata().addLatestExpression(expression); - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - response.getWriter().write("{ \"code\" : \"ok\" }"); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.util; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; + +public class LogExpressionCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + String expression = request.getParameter("expression"); + + project.getMetadata().addLatestExpression(expression); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + response.getWriter().write("{ \"code\" : \"ok\" }"); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/util/PreviewExpressionCommand.java b/src/main/java/com/metaweb/gridworks/commands/util/PreviewExpressionCommand.java index cfa8d2ec6..c5390cce1 100644 --- a/src/main/java/com/metaweb/gridworks/commands/util/PreviewExpressionCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/util/PreviewExpressionCommand.java @@ -1,194 +1,194 @@ -package com.metaweb.gridworks.commands.util; - -import java.io.IOException; -import java.io.Serializable; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.expr.HasFields; -import com.metaweb.gridworks.expr.MetaParser; -import com.metaweb.gridworks.expr.ParsingException; -import com.metaweb.gridworks.expr.WrappedCell; -import com.metaweb.gridworks.expr.WrappedRow; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class PreviewExpressionCommand extends Command { - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - - int cellIndex = Integer.parseInt(request.getParameter("cellIndex")); - String columnName = cellIndex < 0 ? "" : project.columnModel.getColumnByCellIndex(cellIndex).getName(); - - String expression = request.getParameter("expression"); - String rowIndicesString = request.getParameter("rowIndices"); - if (rowIndicesString == null) { - respond(response, "{ \"code\" : \"error\", \"message\" : \"No row indices specified\" }"); - return; - } - - boolean repeat = "true".equals(request.getParameter("repeat")); - int repeatCount = 10; - if (repeat) { - String repeatCountString = request.getParameter("repeatCount"); - try { - repeatCount = Math.max(Math.min(Integer.parseInt(repeatCountString), 10), 0); - } catch (Exception e) { - } - } - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString); - int length = rowIndices.length(); - - JSONWriter writer = new JSONWriter(response.getWriter()); - writer.object(); - - try { - Evaluable eval = MetaParser.parse(expression); - - writer.key("code"); writer.value("ok"); - writer.key("results"); writer.array(); - - Properties bindings = ExpressionUtils.createBindings(project); - for (int i = 0; i < length; i++) { - Object result = null; - - int rowIndex = rowIndices.getInt(i); - if (rowIndex >= 0 && rowIndex < project.rows.size()) { - Row row = project.rows.get(rowIndex); - Cell cell = row.getCell(cellIndex); - - try { - ExpressionUtils.bind(bindings, row, rowIndex, columnName, cell); - result = eval.evaluate(bindings); - - if (repeat) { - for (int r = 0; r < repeatCount && ExpressionUtils.isStorable(result); r++) { - Cell newCell = new Cell((Serializable) result, (cell != null) ? cell.recon : null); - ExpressionUtils.bind(bindings, row, rowIndex, columnName, newCell); - - Object newResult = eval.evaluate(bindings); - if (ExpressionUtils.isError(newResult)) { - break; - } else if (ExpressionUtils.sameValue(result, newResult)) { - break; - } else { - result = newResult; - } - } - } - } catch (Exception e) { - // ignore - } - } - - if (result == null) { - writer.value(null); - } else if (ExpressionUtils.isError(result)) { - writer.object(); - writer.key("message"); writer.value(((EvalError) result).message); - writer.endObject(); - } else { - StringBuffer sb = new StringBuffer(); - - writeValue(sb, result, false); - - writer.value(sb.toString()); - } - } - writer.endArray(); - } catch (ParsingException e) { - writer.key("code"); writer.value("error"); - writer.key("type"); writer.value("parser"); - writer.key("message"); writer.value(e.getMessage()); - } catch (Exception e) { - writer.key("code"); writer.value("error"); - writer.key("type"); writer.value("other"); - writer.key("message"); writer.value(e.getMessage()); - } - - writer.endObject(); - } catch (Exception e) { - respondException(response, e); - } - } - - static protected void writeValue(StringBuffer sb, Object v, boolean quote) throws JSONException { - if (ExpressionUtils.isError(v)) { - sb.append("[error: " + ((EvalError) v).message + "]"); - } else { - if (v == null) { - sb.append("null"); - } else { - if (v instanceof WrappedCell) { - sb.append("[object Cell]"); - } else if (v instanceof WrappedRow) { - sb.append("[object Row]"); - } else if (ExpressionUtils.isArray(v)) { - Object[] a = (Object[]) v; - sb.append("[ "); - for (int i = 0; i < a.length; i++) { - if (i > 0) { - sb.append(", "); - } - writeValue(sb, a[i], true); - } - sb.append(" ]"); - } else if (ExpressionUtils.isArrayOrList(v)) { - List list = ExpressionUtils.toObjectList(v); - sb.append("[ "); - for (int i = 0; i < list.size(); i++) { - if (i > 0) { - sb.append(", "); - } - writeValue(sb, list.get(i), true); - } - sb.append(" ]"); - } else if (v instanceof HasFields) { - sb.append("[object " + v.getClass().getSimpleName() + "]"); - } else if (v instanceof Calendar) { - Calendar c = (Calendar) v; - - sb.append("[date " + - ParsingUtilities.dateToString(c.getTime()) +"]"); - } else if (v instanceof Date) { - sb.append("[date " + - ParsingUtilities.dateToString((Date) v) +"]"); - } else if (v instanceof String) { - if (quote) { - sb.append(JSONObject.quote((String) v)); - } else { - sb.append((String) v); - } - } else { - sb.append(v.toString()); - } - } - } - } -} +package com.metaweb.gridworks.commands.util; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.expr.HasFields; +import com.metaweb.gridworks.expr.MetaParser; +import com.metaweb.gridworks.expr.ParsingException; +import com.metaweb.gridworks.expr.WrappedCell; +import com.metaweb.gridworks.expr.WrappedRow; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class PreviewExpressionCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + int cellIndex = Integer.parseInt(request.getParameter("cellIndex")); + String columnName = cellIndex < 0 ? "" : project.columnModel.getColumnByCellIndex(cellIndex).getName(); + + String expression = request.getParameter("expression"); + String rowIndicesString = request.getParameter("rowIndices"); + if (rowIndicesString == null) { + respond(response, "{ \"code\" : \"error\", \"message\" : \"No row indices specified\" }"); + return; + } + + boolean repeat = "true".equals(request.getParameter("repeat")); + int repeatCount = 10; + if (repeat) { + String repeatCountString = request.getParameter("repeatCount"); + try { + repeatCount = Math.max(Math.min(Integer.parseInt(repeatCountString), 10), 0); + } catch (Exception e) { + } + } + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString); + int length = rowIndices.length(); + + JSONWriter writer = new JSONWriter(response.getWriter()); + writer.object(); + + try { + Evaluable eval = MetaParser.parse(expression); + + writer.key("code"); writer.value("ok"); + writer.key("results"); writer.array(); + + Properties bindings = ExpressionUtils.createBindings(project); + for (int i = 0; i < length; i++) { + Object result = null; + + int rowIndex = rowIndices.getInt(i); + if (rowIndex >= 0 && rowIndex < project.rows.size()) { + Row row = project.rows.get(rowIndex); + Cell cell = row.getCell(cellIndex); + + try { + ExpressionUtils.bind(bindings, row, rowIndex, columnName, cell); + result = eval.evaluate(bindings); + + if (repeat) { + for (int r = 0; r < repeatCount && ExpressionUtils.isStorable(result); r++) { + Cell newCell = new Cell((Serializable) result, (cell != null) ? cell.recon : null); + ExpressionUtils.bind(bindings, row, rowIndex, columnName, newCell); + + Object newResult = eval.evaluate(bindings); + if (ExpressionUtils.isError(newResult)) { + break; + } else if (ExpressionUtils.sameValue(result, newResult)) { + break; + } else { + result = newResult; + } + } + } + } catch (Exception e) { + // ignore + } + } + + if (result == null) { + writer.value(null); + } else if (ExpressionUtils.isError(result)) { + writer.object(); + writer.key("message"); writer.value(((EvalError) result).message); + writer.endObject(); + } else { + StringBuffer sb = new StringBuffer(); + + writeValue(sb, result, false); + + writer.value(sb.toString()); + } + } + writer.endArray(); + } catch (ParsingException e) { + writer.key("code"); writer.value("error"); + writer.key("type"); writer.value("parser"); + writer.key("message"); writer.value(e.getMessage()); + } catch (Exception e) { + writer.key("code"); writer.value("error"); + writer.key("type"); writer.value("other"); + writer.key("message"); writer.value(e.getMessage()); + } + + writer.endObject(); + } catch (Exception e) { + respondException(response, e); + } + } + + static protected void writeValue(StringBuffer sb, Object v, boolean quote) throws JSONException { + if (ExpressionUtils.isError(v)) { + sb.append("[error: " + ((EvalError) v).message + "]"); + } else { + if (v == null) { + sb.append("null"); + } else { + if (v instanceof WrappedCell) { + sb.append("[object Cell]"); + } else if (v instanceof WrappedRow) { + sb.append("[object Row]"); + } else if (ExpressionUtils.isArray(v)) { + Object[] a = (Object[]) v; + sb.append("[ "); + for (int i = 0; i < a.length; i++) { + if (i > 0) { + sb.append(", "); + } + writeValue(sb, a[i], true); + } + sb.append(" ]"); + } else if (ExpressionUtils.isArrayOrList(v)) { + List list = ExpressionUtils.toObjectList(v); + sb.append("[ "); + for (int i = 0; i < list.size(); i++) { + if (i > 0) { + sb.append(", "); + } + writeValue(sb, list.get(i), true); + } + sb.append(" ]"); + } else if (v instanceof HasFields) { + sb.append("[object " + v.getClass().getSimpleName() + "]"); + } else if (v instanceof Calendar) { + Calendar c = (Calendar) v; + + sb.append("[date " + + ParsingUtilities.dateToString(c.getTime()) +"]"); + } else if (v instanceof Date) { + sb.append("[date " + + ParsingUtilities.dateToString((Date) v) +"]"); + } else if (v instanceof String) { + if (quote) { + sb.append(JSONObject.quote((String) v)); + } else { + sb.append((String) v); + } + } else { + sb.append(v.toString()); + } + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/util/PreviewExtendDataCommand.java b/src/main/java/com/metaweb/gridworks/commands/util/PreviewExtendDataCommand.java index 69423ced0..5cdac177a 100644 --- a/src/main/java/com/metaweb/gridworks/commands/util/PreviewExtendDataCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/util/PreviewExtendDataCommand.java @@ -1,158 +1,158 @@ -package com.metaweb.gridworks.commands.util; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.util.FreebaseDataExtensionJob; -import com.metaweb.gridworks.util.ParsingUtilities; -import com.metaweb.gridworks.util.FreebaseDataExtensionJob.ColumnInfo; -import com.metaweb.gridworks.util.FreebaseDataExtensionJob.DataExtension; - -public class PreviewExtendDataCommand extends Command { - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - String columnName = request.getParameter("columnName"); - - String rowIndicesString = request.getParameter("rowIndices"); - if (rowIndicesString == null) { - respond(response, "{ \"code\" : \"error\", \"message\" : \"No row indices specified\" }"); - return; - } - - String jsonString = request.getParameter("extension"); - JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); - - JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString); - int length = rowIndices.length(); - int cellIndex = project.columnModel.getColumnByName(columnName).getCellIndex(); - - List topicNames = new ArrayList(); - List topicGuids = new ArrayList(); - Set guids = new HashSet(); - for (int i = 0; i < length; i++) { - int rowIndex = rowIndices.getInt(i); - if (rowIndex >= 0 && rowIndex < project.rows.size()) { - Row row = project.rows.get(rowIndex); - Cell cell = row.getCell(cellIndex); - if (cell != null && cell.recon != null && cell.recon.match != null) { - topicNames.add(cell.recon.match.topicName); - topicGuids.add(cell.recon.match.topicGUID); - guids.add(cell.recon.match.topicGUID); - } else { - topicNames.add(null); - topicGuids.add(null); - guids.add(null); - } - } - } - - Map reconCandidateMap = new HashMap(); - FreebaseDataExtensionJob job = new FreebaseDataExtensionJob(json); - Map map = job.extend(guids, reconCandidateMap); - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - JSONWriter writer = new JSONWriter(response.getWriter()); - writer.object(); - writer.key("code"); writer.value("ok"); - writer.key("columns"); - writer.array(); - for (ColumnInfo info : job.columns) { - writer.object(); - writer.key("names"); - writer.array(); - for (String name : info.names) { - writer.value(name); - } - writer.endArray(); - writer.key("path"); - writer.array(); - for (String id : info.path) { - writer.value(id); - } - writer.endArray(); - writer.endObject(); - } - writer.endArray(); - - writer.key("rows"); - writer.array(); - for (int r = 0; r < topicNames.size(); r++) { - String guid = topicGuids.get(r); - String topicName = topicNames.get(r); - - if (guid != null && map.containsKey(guid)) { - DataExtension ext = map.get(guid); - boolean first = true; - - if (ext.data.length > 0) { - for (Object[] row : ext.data) { - writer.array(); - if (first) { - writer.value(topicName); - first = false; - } else { - writer.value(null); - } - - for (Object cell : row) { - if (cell != null && cell instanceof ReconCandidate) { - ReconCandidate rc = (ReconCandidate) cell; - writer.object(); - writer.key("id"); writer.value(rc.topicID); - writer.key("name"); writer.value(rc.topicName); - writer.endObject(); - } else { - writer.value(cell); - } - } - - writer.endArray(); - } - continue; - } - } - - writer.array(); - if (guid != null) { - writer.object(); - writer.key("id"); writer.value("/guid/" + guid.substring(1)); - writer.key("name"); writer.value(topicName); - writer.endObject(); - } else { - writer.value(""); - } - writer.endArray(); - } - writer.endArray(); - - writer.endObject(); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.util.FreebaseDataExtensionJob; +import com.metaweb.gridworks.util.ParsingUtilities; +import com.metaweb.gridworks.util.FreebaseDataExtensionJob.ColumnInfo; +import com.metaweb.gridworks.util.FreebaseDataExtensionJob.DataExtension; + +public class PreviewExtendDataCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + String columnName = request.getParameter("columnName"); + + String rowIndicesString = request.getParameter("rowIndices"); + if (rowIndicesString == null) { + respond(response, "{ \"code\" : \"error\", \"message\" : \"No row indices specified\" }"); + return; + } + + String jsonString = request.getParameter("extension"); + JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); + + JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString); + int length = rowIndices.length(); + int cellIndex = project.columnModel.getColumnByName(columnName).getCellIndex(); + + List topicNames = new ArrayList(); + List topicGuids = new ArrayList(); + Set guids = new HashSet(); + for (int i = 0; i < length; i++) { + int rowIndex = rowIndices.getInt(i); + if (rowIndex >= 0 && rowIndex < project.rows.size()) { + Row row = project.rows.get(rowIndex); + Cell cell = row.getCell(cellIndex); + if (cell != null && cell.recon != null && cell.recon.match != null) { + topicNames.add(cell.recon.match.topicName); + topicGuids.add(cell.recon.match.topicGUID); + guids.add(cell.recon.match.topicGUID); + } else { + topicNames.add(null); + topicGuids.add(null); + guids.add(null); + } + } + } + + Map reconCandidateMap = new HashMap(); + FreebaseDataExtensionJob job = new FreebaseDataExtensionJob(json); + Map map = job.extend(guids, reconCandidateMap); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + JSONWriter writer = new JSONWriter(response.getWriter()); + writer.object(); + writer.key("code"); writer.value("ok"); + writer.key("columns"); + writer.array(); + for (ColumnInfo info : job.columns) { + writer.object(); + writer.key("names"); + writer.array(); + for (String name : info.names) { + writer.value(name); + } + writer.endArray(); + writer.key("path"); + writer.array(); + for (String id : info.path) { + writer.value(id); + } + writer.endArray(); + writer.endObject(); + } + writer.endArray(); + + writer.key("rows"); + writer.array(); + for (int r = 0; r < topicNames.size(); r++) { + String guid = topicGuids.get(r); + String topicName = topicNames.get(r); + + if (guid != null && map.containsKey(guid)) { + DataExtension ext = map.get(guid); + boolean first = true; + + if (ext.data.length > 0) { + for (Object[] row : ext.data) { + writer.array(); + if (first) { + writer.value(topicName); + first = false; + } else { + writer.value(null); + } + + for (Object cell : row) { + if (cell != null && cell instanceof ReconCandidate) { + ReconCandidate rc = (ReconCandidate) cell; + writer.object(); + writer.key("id"); writer.value(rc.topicID); + writer.key("name"); writer.value(rc.topicName); + writer.endObject(); + } else { + writer.value(cell); + } + } + + writer.endArray(); + } + continue; + } + } + + writer.array(); + if (guid != null) { + writer.object(); + writer.key("id"); writer.value("/guid/" + guid.substring(1)); + writer.key("name"); writer.value(topicName); + writer.endObject(); + } else { + writer.value(""); + } + writer.endArray(); + } + writer.endArray(); + + writer.endObject(); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/commands/util/PreviewProtographCommand.java b/src/main/java/com/metaweb/gridworks/commands/util/PreviewProtographCommand.java index 78228fd50..feb450c2a 100644 --- a/src/main/java/com/metaweb/gridworks/commands/util/PreviewProtographCommand.java +++ b/src/main/java/com/metaweb/gridworks/commands/util/PreviewProtographCommand.java @@ -1,68 +1,68 @@ -package com.metaweb.gridworks.commands.util; - -import java.io.IOException; -import java.io.StringWriter; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONArray; -import org.json.JSONObject; - -import com.metaweb.gridworks.commands.Command; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.protograph.Protograph; -import com.metaweb.gridworks.protograph.transpose.MqlreadLikeTransposedNodeFactory; -import com.metaweb.gridworks.protograph.transpose.Transposer; -import com.metaweb.gridworks.protograph.transpose.TripleLoaderTransposedNodeFactory; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class PreviewProtographCommand extends Command { - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - try { - Project project = getProject(request); - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - String jsonString = request.getParameter("protograph"); - JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); - Protograph protograph = Protograph.reconstruct(json); - - StringBuffer sb = new StringBuffer(2048); - sb.append("{ "); - - { - StringWriter stringWriter = new StringWriter(); - TripleLoaderTransposedNodeFactory nodeFactory = new TripleLoaderTransposedNodeFactory(stringWriter); - - Transposer.transpose(project, protograph, protograph.getRootNode(0), nodeFactory); - nodeFactory.flush(); - - sb.append("\"tripleloader\" : "); - sb.append(JSONObject.quote(stringWriter.toString())); - } - - { - MqlreadLikeTransposedNodeFactory nodeFactory = new MqlreadLikeTransposedNodeFactory(); - - Transposer.transpose(project, protograph, protograph.getRootNode(0), nodeFactory); - - JSONArray results = nodeFactory.getJSON(); - - sb.append(", \"mqllike\" : "); - sb.append(results.toString()); - } - - sb.append(" }"); - - respond(response, sb.toString()); - } catch (Exception e) { - respondException(response, e); - } - } -} +package com.metaweb.gridworks.commands.util; + +import java.io.IOException; +import java.io.StringWriter; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONArray; +import org.json.JSONObject; + +import com.metaweb.gridworks.commands.Command; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.protograph.Protograph; +import com.metaweb.gridworks.protograph.transpose.MqlreadLikeTransposedNodeFactory; +import com.metaweb.gridworks.protograph.transpose.Transposer; +import com.metaweb.gridworks.protograph.transpose.TripleLoaderTransposedNodeFactory; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class PreviewProtographCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + String jsonString = request.getParameter("protograph"); + JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); + Protograph protograph = Protograph.reconstruct(json); + + StringBuffer sb = new StringBuffer(2048); + sb.append("{ "); + + { + StringWriter stringWriter = new StringWriter(); + TripleLoaderTransposedNodeFactory nodeFactory = new TripleLoaderTransposedNodeFactory(stringWriter); + + Transposer.transpose(project, protograph, protograph.getRootNode(0), nodeFactory); + nodeFactory.flush(); + + sb.append("\"tripleloader\" : "); + sb.append(JSONObject.quote(stringWriter.toString())); + } + + { + MqlreadLikeTransposedNodeFactory nodeFactory = new MqlreadLikeTransposedNodeFactory(); + + Transposer.transpose(project, protograph, protograph.getRootNode(0), nodeFactory); + + JSONArray results = nodeFactory.getJSON(); + + sb.append(", \"mqllike\" : "); + sb.append(results.toString()); + } + + sb.append(" }"); + + respond(response, sb.toString()); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/Evaluable.java b/src/main/java/com/metaweb/gridworks/expr/Evaluable.java index 776b31e02..dcaab3be3 100644 --- a/src/main/java/com/metaweb/gridworks/expr/Evaluable.java +++ b/src/main/java/com/metaweb/gridworks/expr/Evaluable.java @@ -1,16 +1,16 @@ -package com.metaweb.gridworks.expr; - -import java.util.Properties; - -/** - * Interface for evaluable expressions in any arbitrary language. - */ -public interface Evaluable { - /** - * Evaluate this expression in the given environment (bindings). - * - * @param bindings - * @return - */ - public Object evaluate(Properties bindings); -} +package com.metaweb.gridworks.expr; + +import java.util.Properties; + +/** + * Interface for evaluable expressions in any arbitrary language. + */ +public interface Evaluable { + /** + * Evaluate this expression in the given environment (bindings). + * + * @param bindings + * @return + */ + public Object evaluate(Properties bindings); +} diff --git a/src/main/java/com/metaweb/gridworks/expr/HasFields.java b/src/main/java/com/metaweb/gridworks/expr/HasFields.java index e346797af..df1262f91 100644 --- a/src/main/java/com/metaweb/gridworks/expr/HasFields.java +++ b/src/main/java/com/metaweb/gridworks/expr/HasFields.java @@ -1,13 +1,13 @@ -package com.metaweb.gridworks.expr; - -import java.util.Properties; - -/** - * Interface for objects that have named fields, which can be retrieved using the - * dot notation or the bracket notation, e.g., cells.Country, cells["Type of Disaster"]. - */ -public interface HasFields { - public Object getField(String name, Properties bindings); - - public boolean fieldAlsoHasFields(String name); -} +package com.metaweb.gridworks.expr; + +import java.util.Properties; + +/** + * Interface for objects that have named fields, which can be retrieved using the + * dot notation or the bracket notation, e.g., cells.Country, cells["Type of Disaster"]. + */ +public interface HasFields { + public Object getField(String name, Properties bindings); + + public boolean fieldAlsoHasFields(String name); +} diff --git a/src/main/java/com/metaweb/gridworks/expr/HasFieldsList.java b/src/main/java/com/metaweb/gridworks/expr/HasFieldsList.java index 1c666bdd9..82f85ff82 100644 --- a/src/main/java/com/metaweb/gridworks/expr/HasFieldsList.java +++ b/src/main/java/com/metaweb/gridworks/expr/HasFieldsList.java @@ -1,10 +1,10 @@ -package com.metaweb.gridworks.expr; - -/** - * Interface for objects each of which is a list of HasFields objects of the - * same kind (e.g., list of cells). Its getField method thus returns either - * another HasFieldsList object or an array or java.util.List of objects. - */ -public interface HasFieldsList extends HasFields { - public int length(); -} +package com.metaweb.gridworks.expr; + +/** + * Interface for objects each of which is a list of HasFields objects of the + * same kind (e.g., list of cells). Its getField method thus returns either + * another HasFieldsList object or an array or java.util.List of objects. + */ +public interface HasFieldsList extends HasFields { + public int length(); +} diff --git a/src/main/java/com/metaweb/gridworks/expr/JythonEvaluable.java b/src/main/java/com/metaweb/gridworks/expr/JythonEvaluable.java index 503e45d03..1125409d5 100644 --- a/src/main/java/com/metaweb/gridworks/expr/JythonEvaluable.java +++ b/src/main/java/com/metaweb/gridworks/expr/JythonEvaluable.java @@ -15,8 +15,8 @@ import org.python.core.PyString; import org.python.util.PythonInterpreter; public class JythonEvaluable implements Evaluable { - private static final String s_functionName = "___temp___"; - + private static final String s_functionName = "___temp___"; + private static PythonInterpreter _engine; static { File libPath = new File("lib/jython"); @@ -77,7 +77,7 @@ public class JythonEvaluable implements Evaluable { } } - return result; + return result; } protected Object unwrap(PyObject po) { diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/Cross.java b/src/main/java/com/metaweb/gridworks/expr/functions/Cross.java index 40a81890a..560882c77 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/Cross.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/Cross.java @@ -1,53 +1,53 @@ -package com.metaweb.gridworks.expr.functions; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.ProjectManager; -import com.metaweb.gridworks.InterProjectModel.ProjectJoin; -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.expr.WrappedCell; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; -import com.metaweb.gridworks.model.Project; - -public class Cross implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 3) { - // from project is implied - - Object wrappedCell = args[0]; // from cell - Object toProjectName = args[1]; - Object toColumnName = args[2]; - - if (wrappedCell != null && wrappedCell instanceof WrappedCell && - toProjectName != null && toProjectName instanceof String && - toColumnName != null && toColumnName instanceof String) { - - ProjectJoin join = ProjectManager.singleton.getInterProjectModel().getJoin( - ProjectManager.singleton.getProjectMetadata( - ((Project) bindings.get("project")).id).getName(), - ((WrappedCell) wrappedCell).columnName, - (String) toProjectName, - (String) toColumnName - ); - - return join.getRows(((WrappedCell) wrappedCell).cell.value); - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a cell, a project name to join with, and a column name in that project"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("TODO"); - writer.key("params"); writer.value("cell c, string projectName, string columnName"); - writer.key("returns"); writer.value("array"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.InterProjectModel.ProjectJoin; +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.expr.WrappedCell; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; +import com.metaweb.gridworks.model.Project; + +public class Cross implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 3) { + // from project is implied + + Object wrappedCell = args[0]; // from cell + Object toProjectName = args[1]; + Object toColumnName = args[2]; + + if (wrappedCell != null && wrappedCell instanceof WrappedCell && + toProjectName != null && toProjectName instanceof String && + toColumnName != null && toColumnName instanceof String) { + + ProjectJoin join = ProjectManager.singleton.getInterProjectModel().getJoin( + ProjectManager.singleton.getProjectMetadata( + ((Project) bindings.get("project")).id).getName(), + ((WrappedCell) wrappedCell).columnName, + (String) toProjectName, + (String) toColumnName + ); + + return join.getRows(((WrappedCell) wrappedCell).cell.value); + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a cell, a project name to join with, and a column name in that project"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("TODO"); + writer.key("params"); writer.value("cell c, string projectName, string columnName"); + writer.key("returns"); writer.value("array"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/Get.java b/src/main/java/com/metaweb/gridworks/expr/functions/Get.java index 1106b0751..68790162f 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/Get.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/Get.java @@ -1,105 +1,105 @@ -package com.metaweb.gridworks.expr.functions; - -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.expr.HasFields; -import com.metaweb.gridworks.gel.Function; - -public class Get implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length > 1 && args.length <= 3) { - Object v = args[0]; - Object from = args[1]; - Object to = (args.length == 3) ? args[2] : null; - - if (v != null && from != null) { - if (v instanceof HasFields) { - if (from instanceof String) { - return ((HasFields) v).getField((String) from, bindings); - } - } else { - if (from instanceof Number && (to == null || to instanceof Number)) { - if (v.getClass().isArray() || v instanceof List) { - int length = v.getClass().isArray() ? - ((Object[]) v).length : - ExpressionUtils.toObjectList(v).size(); - - int start = ((Number) from).intValue(); - if (start < 0) { - start = length + start; - } - start = Math.min(length, Math.max(0, start)); - - if (to == null) { - return start >= length ? null : - (v.getClass().isArray() ? - ((Object[]) v)[start] : - ExpressionUtils.toObjectList(v).get(start)); - } else { - int end = (to != null && to instanceof Number) ? ((Number) to).intValue() : length; - - if (end < 0) { - end = length + end; - } - end = Math.min(length, Math.max(start, end)); - - if (end > start) { - if (v.getClass().isArray()) { - Object[] a2 = new Object[end - start]; - - System.arraycopy((Object[]) v, start, a2, 0, end - start); - - return a2; - } else { - return ExpressionUtils.toObjectList(v).subList(start, end); - } - } - } - } else { - String s = (v instanceof String) ? (String) v : v.toString(); - - int start = ((Number) from).intValue(); - if (start < 0) { - start = s.length() + start; - } - start = Math.min(s.length(), Math.max(0, start)); - - if (to != null && to instanceof Number) { - int end = ((Number) to).intValue(); - if (end < 0) { - end = s.length() + end; - } - end = Math.min(s.length(), Math.max(start, end)); - - return s.substring(start, end); - } else { - return s.substring(start, start + 1); - } - } - } - } - } - } - return null; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value( - "If o has fields, returns the field named 'from' of o. " + - "If o is an array, returns o[from, to]. " + - "if o is a string, returns o.substring(from, to)" - ); - writer.key("params"); writer.value("o, number or string from, optional number to"); - writer.key("returns"); writer.value("Depends on actual arguments"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions; + +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.expr.HasFields; +import com.metaweb.gridworks.gel.Function; + +public class Get implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length > 1 && args.length <= 3) { + Object v = args[0]; + Object from = args[1]; + Object to = (args.length == 3) ? args[2] : null; + + if (v != null && from != null) { + if (v instanceof HasFields) { + if (from instanceof String) { + return ((HasFields) v).getField((String) from, bindings); + } + } else { + if (from instanceof Number && (to == null || to instanceof Number)) { + if (v.getClass().isArray() || v instanceof List) { + int length = v.getClass().isArray() ? + ((Object[]) v).length : + ExpressionUtils.toObjectList(v).size(); + + int start = ((Number) from).intValue(); + if (start < 0) { + start = length + start; + } + start = Math.min(length, Math.max(0, start)); + + if (to == null) { + return start >= length ? null : + (v.getClass().isArray() ? + ((Object[]) v)[start] : + ExpressionUtils.toObjectList(v).get(start)); + } else { + int end = (to != null && to instanceof Number) ? ((Number) to).intValue() : length; + + if (end < 0) { + end = length + end; + } + end = Math.min(length, Math.max(start, end)); + + if (end > start) { + if (v.getClass().isArray()) { + Object[] a2 = new Object[end - start]; + + System.arraycopy((Object[]) v, start, a2, 0, end - start); + + return a2; + } else { + return ExpressionUtils.toObjectList(v).subList(start, end); + } + } + } + } else { + String s = (v instanceof String) ? (String) v : v.toString(); + + int start = ((Number) from).intValue(); + if (start < 0) { + start = s.length() + start; + } + start = Math.min(s.length(), Math.max(0, start)); + + if (to != null && to instanceof Number) { + int end = ((Number) to).intValue(); + if (end < 0) { + end = s.length() + end; + } + end = Math.min(s.length(), Math.max(start, end)); + + return s.substring(start, end); + } else { + return s.substring(start, start + 1); + } + } + } + } + } + } + return null; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value( + "If o has fields, returns the field named 'from' of o. " + + "If o is an array, returns o[from, to]. " + + "if o is a string, returns o.substring(from, to)" + ); + writer.key("params"); writer.value("o, number or string from, optional number to"); + writer.key("returns"); writer.value("Depends on actual arguments"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/Length.java b/src/main/java/com/metaweb/gridworks/expr/functions/Length.java index 011c543ad..747496c27 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/Length.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/Length.java @@ -1,43 +1,43 @@ -package com.metaweb.gridworks.expr.functions; - -import java.util.Collection; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Length implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1) { - Object v = args[0]; - - if (v != null) { - if (v.getClass().isArray()) { - Object[] a = (Object[]) v; - return a.length; - } else if (v instanceof Collection) { - return ((Collection) v).size(); - } else { - String s = (v instanceof String ? (String) v : v.toString()); - return s.length(); - } - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array or a string"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the length of o"); - writer.key("params"); writer.value("array or string o"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions; + +import java.util.Collection; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Length implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1) { + Object v = args[0]; + + if (v != null) { + if (v.getClass().isArray()) { + Object[] a = (Object[]) v; + return a.length; + } else if (v instanceof Collection) { + return ((Collection) v).size(); + } else { + String s = (v instanceof String ? (String) v : v.toString()); + return s.length(); + } + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array or a string"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the length of o"); + writer.key("params"); writer.value("array or string o"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/Slice.java b/src/main/java/com/metaweb/gridworks/expr/functions/Slice.java index 28655f861..2eab6508f 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/Slice.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/Slice.java @@ -1,86 +1,86 @@ -package com.metaweb.gridworks.expr.functions; - -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.gel.Function; - -public class Slice implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length > 1 && args.length <= 3) { - Object v = args[0]; - Object from = args[1]; - Object to = (args.length == 3) ? args[2] : null; - - if (v != null && from != null && from instanceof Number && (to == null || to instanceof Number)) { - if (v.getClass().isArray() || v instanceof List) { - int length = v.getClass().isArray() ? - ((Object[]) v).length : - ExpressionUtils.toObjectList(v).size(); - - int start = ((Number) from).intValue(); - int end = (to != null && to instanceof Number) ? ((Number) to).intValue() : length; - - if (start < 0) { - start = length + start; - } - start = Math.min(length, Math.max(0, start)); - - if (end < 0) { - end = length + end; - } - end = Math.min(length, Math.max(start, end)); - - if (v.getClass().isArray()) { - Object[] a2 = new Object[end - start]; - - System.arraycopy((Object[]) v, start, a2, 0, end - start); - - return a2; - } else { - return ExpressionUtils.toObjectList(v).subList(start, end); - } - } else { - String s = (v instanceof String) ? (String) v : v.toString(); - - int start = ((Number) from).intValue(); - if (start < 0) { - start = s.length() + start; - } - start = Math.min(s.length(), Math.max(0, start)); - - if (to != null && to instanceof Number) { - int end = ((Number) to).intValue(); - if (end < 0) { - end = s.length() + end; - } - end = Math.min(s.length(), Math.max(start, end)); - - return s.substring(start, end); - } else { - return s.substring(start); - } - } - } - } - return null; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value( - "If o is an array, returns o[from, to]. " + - "if o is a string, returns o.substring(from, to)" - ); - writer.key("params"); writer.value("o, number from, optional number to"); - writer.key("returns"); writer.value("Depends on actual arguments"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions; + +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.gel.Function; + +public class Slice implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length > 1 && args.length <= 3) { + Object v = args[0]; + Object from = args[1]; + Object to = (args.length == 3) ? args[2] : null; + + if (v != null && from != null && from instanceof Number && (to == null || to instanceof Number)) { + if (v.getClass().isArray() || v instanceof List) { + int length = v.getClass().isArray() ? + ((Object[]) v).length : + ExpressionUtils.toObjectList(v).size(); + + int start = ((Number) from).intValue(); + int end = (to != null && to instanceof Number) ? ((Number) to).intValue() : length; + + if (start < 0) { + start = length + start; + } + start = Math.min(length, Math.max(0, start)); + + if (end < 0) { + end = length + end; + } + end = Math.min(length, Math.max(start, end)); + + if (v.getClass().isArray()) { + Object[] a2 = new Object[end - start]; + + System.arraycopy((Object[]) v, start, a2, 0, end - start); + + return a2; + } else { + return ExpressionUtils.toObjectList(v).subList(start, end); + } + } else { + String s = (v instanceof String) ? (String) v : v.toString(); + + int start = ((Number) from).intValue(); + if (start < 0) { + start = s.length() + start; + } + start = Math.min(s.length(), Math.max(0, start)); + + if (to != null && to instanceof Number) { + int end = ((Number) to).intValue(); + if (end < 0) { + end = s.length() + end; + } + end = Math.min(s.length(), Math.max(start, end)); + + return s.substring(start, end); + } else { + return s.substring(start); + } + } + } + } + return null; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value( + "If o is an array, returns o[from, to]. " + + "if o is a string, returns o.substring(from, to)" + ); + writer.key("params"); writer.value("o, number from, optional number to"); + writer.key("returns"); writer.value("Depends on actual arguments"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/ToNumber.java b/src/main/java/com/metaweb/gridworks/expr/functions/ToNumber.java index 107dfeaa8..7477a2f47 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/ToNumber.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/ToNumber.java @@ -1,39 +1,39 @@ -package com.metaweb.gridworks.expr.functions; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.Function; - -public class ToNumber implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1 && args[0] != null) { - if (args[0] instanceof Number) { - return args[0]; - } else { - String s = args[0].toString(); - try { - return Double.parseDouble(s); - } catch (NumberFormatException e) { - return new EvalError("Cannot parse to number"); - } - } - } - return null; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns o converted to a number"); - writer.key("params"); writer.value("o"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } - -} +package com.metaweb.gridworks.expr.functions; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.Function; + +public class ToNumber implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1 && args[0] != null) { + if (args[0] instanceof Number) { + return args[0]; + } else { + String s = args[0].toString(); + try { + return Double.parseDouble(s); + } catch (NumberFormatException e) { + return new EvalError("Cannot parse to number"); + } + } + } + return null; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns o converted to a number"); + writer.key("params"); writer.value("o"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/ToString.java b/src/main/java/com/metaweb/gridworks/expr/functions/ToString.java index 5f95be839..3d6dc1a78 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/ToString.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/ToString.java @@ -1,44 +1,44 @@ -package com.metaweb.gridworks.expr.functions; - -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.gel.Function; - -public class ToString implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length >= 1) { - Object o1 = args[0]; - if (o1 != null) { - if (o1 instanceof Calendar) { - if (args.length == 2) { - Object o2 = args[1]; - if (o2 != null && o2 instanceof String) { - SimpleDateFormat formatter = new SimpleDateFormat((String) o2); - return formatter.format(((Calendar) o1).getTime()); - } - } - } else { - return (o1 instanceof String) ? o1 : o1.toString(); - } - } - } - return null; - } - - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns o converted to a string"); - writer.key("params"); writer.value("o, string format (optional)"); - writer.key("returns"); writer.value("string"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.gel.Function; + +public class ToString implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length >= 1) { + Object o1 = args[0]; + if (o1 != null) { + if (o1 instanceof Calendar) { + if (args.length == 2) { + Object o2 = args[1]; + if (o2 != null && o2 instanceof String) { + SimpleDateFormat formatter = new SimpleDateFormat((String) o2); + return formatter.format(((Calendar) o1).getTime()); + } + } + } else { + return (o1 instanceof String) ? o1 : o1.toString(); + } + } + } + return null; + } + + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns o converted to a string"); + writer.key("params"); writer.value("o, string format (optional)"); + writer.key("returns"); writer.value("string"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Join.java b/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Join.java index 731986519..449dfda1d 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Join.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Join.java @@ -1,62 +1,62 @@ -package com.metaweb.gridworks.expr.functions.arrays; - -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Join implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 2) { - Object v = args[0]; - Object s = args[1]; - - if (v != null && (v.getClass().isArray() || v instanceof List) && - s != null && s instanceof String) { - - String separator = (String) s; - - StringBuffer sb = new StringBuffer(); - if (v.getClass().isArray()) { - for (Object o : (Object[]) v) { - if (o != null) { - if (sb.length() > 0) { - sb.append(separator); - } - sb.append(o.toString()); - } - } - } else { - for (Object o : ExpressionUtils.toObjectList(v)) { - if (o != null) { - if (sb.length() > 0) { - sb.append(separator); - } - sb.append(o.toString()); - } - } - } - - return sb.toString(); - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array and a string"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the string obtained by joining the array a with the separator sep"); - writer.key("params"); writer.value("array a, string sep"); - writer.key("returns"); writer.value("string"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.arrays; + +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Join implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 2) { + Object v = args[0]; + Object s = args[1]; + + if (v != null && (v.getClass().isArray() || v instanceof List) && + s != null && s instanceof String) { + + String separator = (String) s; + + StringBuffer sb = new StringBuffer(); + if (v.getClass().isArray()) { + for (Object o : (Object[]) v) { + if (o != null) { + if (sb.length() > 0) { + sb.append(separator); + } + sb.append(o.toString()); + } + } + } else { + for (Object o : ExpressionUtils.toObjectList(v)) { + if (o != null) { + if (sb.length() > 0) { + sb.append(separator); + } + sb.append(o.toString()); + } + } + } + + return sb.toString(); + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array and a string"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the string obtained by joining the array a with the separator sep"); + writer.key("params"); writer.value("array a, string sep"); + writer.key("returns"); writer.value("string"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Reverse.java b/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Reverse.java index f24d29bf0..df20a7328 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Reverse.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Reverse.java @@ -1,52 +1,52 @@ -package com.metaweb.gridworks.expr.functions.arrays; - -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Reverse implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1) { - Object v = args[0]; - - if (v != null && (v.getClass().isArray() || v instanceof List)) { - int length = v.getClass().isArray() ? - ((Object[]) v).length : - ExpressionUtils.toObjectList(v).size(); - - Object[] r = new Object[length]; - if (v.getClass().isArray()) { - Object[] a = (Object[]) v; - for (int i = 0; i < length; i++) { - r[i] = a[r.length - i - 1]; - } - } else { - List a = ExpressionUtils.toObjectList(v); - for (int i = 0; i < length; i++) { - r[i] = a.get(r.length - i - 1); - } - } - return r; - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Reverses array a"); - writer.key("params"); writer.value("array a"); - writer.key("returns"); writer.value("array"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.arrays; + +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Reverse implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1) { + Object v = args[0]; + + if (v != null && (v.getClass().isArray() || v instanceof List)) { + int length = v.getClass().isArray() ? + ((Object[]) v).length : + ExpressionUtils.toObjectList(v).size(); + + Object[] r = new Object[length]; + if (v.getClass().isArray()) { + Object[] a = (Object[]) v; + for (int i = 0; i < length; i++) { + r[i] = a[r.length - i - 1]; + } + } else { + List a = ExpressionUtils.toObjectList(v); + for (int i = 0; i < length; i++) { + r[i] = a.get(r.length - i - 1); + } + } + return r; + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Reverses array a"); + writer.key("params"); writer.value("array a"); + writer.key("returns"); writer.value("array"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Sort.java b/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Sort.java index 0a3faece4..c67cb7e79 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Sort.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/arrays/Sort.java @@ -1,50 +1,50 @@ -package com.metaweb.gridworks.expr.functions.arrays; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Sort implements Function { - - @SuppressWarnings("unchecked") - public Object call(Properties bindings, Object[] args) { - if (args.length == 1) { - Object v = args[0]; - - if (v != null) { - if (v.getClass().isArray()) { - Object[] a = (Object[]) v; - Object[] r = a.clone(); - - Arrays.sort(r, 0, r.length); - - return r; - } else if (v instanceof List) { - List a = (List) v; - Collections.sort(a); - - return a; - } - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Sorts array a"); - writer.key("params"); writer.value("array a"); - writer.key("returns"); writer.value("array"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.arrays; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Sort implements Function { + + @SuppressWarnings("unchecked") + public Object call(Properties bindings, Object[] args) { + if (args.length == 1) { + Object v = args[0]; + + if (v != null) { + if (v.getClass().isArray()) { + Object[] a = (Object[]) v; + Object[] r = a.clone(); + + Arrays.sort(r, 0, r.length); + + return r; + } else if (v instanceof List) { + List a = (List) v; + Collections.sort(a); + + return a; + } + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Sorts array a"); + writer.key("params"); writer.value("array a"); + writer.key("returns"); writer.value("array"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/booleans/And.java b/src/main/java/com/metaweb/gridworks/expr/functions/booleans/And.java index a0256c140..d8ab465fe 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/booleans/And.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/booleans/And.java @@ -1,30 +1,30 @@ -package com.metaweb.gridworks.expr.functions.booleans; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.gel.Function; - -public class And implements Function { - - public Object call(Properties bindings, Object[] args) { - for (Object o : args) { - if (!Not.objectToBoolean(o)) { - return false; - } - } - return true; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("ANDs two boolean values"); - writer.key("params"); writer.value("boolean a, boolean b"); - writer.key("returns"); writer.value("boolean"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.booleans; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.gel.Function; + +public class And implements Function { + + public Object call(Properties bindings, Object[] args) { + for (Object o : args) { + if (!Not.objectToBoolean(o)) { + return false; + } + } + return true; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("ANDs two boolean values"); + writer.key("params"); writer.value("boolean a, boolean b"); + writer.key("returns"); writer.value("boolean"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/booleans/Not.java b/src/main/java/com/metaweb/gridworks/expr/functions/booleans/Not.java index fd329c211..e551c9819 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/booleans/Not.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/booleans/Not.java @@ -1,35 +1,35 @@ -package com.metaweb.gridworks.expr.functions.booleans; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Not implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1) { - return !objectToBoolean(args[0]); - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a boolean"); - } - - public static boolean objectToBoolean(Object o) { - return o == null ? false : ( - (o instanceof Boolean) ? ((Boolean) o).booleanValue() : Boolean.parseBoolean(o.toString())); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the opposite of b"); - writer.key("params"); writer.value("boolean b"); - writer.key("returns"); writer.value("boolean"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.booleans; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Not implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1) { + return !objectToBoolean(args[0]); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a boolean"); + } + + public static boolean objectToBoolean(Object o) { + return o == null ? false : ( + (o instanceof Boolean) ? ((Boolean) o).booleanValue() : Boolean.parseBoolean(o.toString())); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the opposite of b"); + writer.key("params"); writer.value("boolean b"); + writer.key("returns"); writer.value("boolean"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/booleans/Or.java b/src/main/java/com/metaweb/gridworks/expr/functions/booleans/Or.java index e5d9643c1..b3cf5d82c 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/booleans/Or.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/booleans/Or.java @@ -1,30 +1,30 @@ -package com.metaweb.gridworks.expr.functions.booleans; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.gel.Function; - -public class Or implements Function { - - public Object call(Properties bindings, Object[] args) { - for (Object o : args) { - if (Not.objectToBoolean(o)) { - return true; - } - } - return false; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns a OR b"); - writer.key("params"); writer.value("boolean a, boolean b"); - writer.key("returns"); writer.value("boolean"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.booleans; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.gel.Function; + +public class Or implements Function { + + public Object call(Properties bindings, Object[] args) { + for (Object o : args) { + if (Not.objectToBoolean(o)) { + return true; + } + } + return false; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns a OR b"); + writer.key("params"); writer.value("boolean a, boolean b"); + writer.key("returns"); writer.value("boolean"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/math/Ceil.java b/src/main/java/com/metaweb/gridworks/expr/functions/math/Ceil.java index b40873639..08f5692a9 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/math/Ceil.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/math/Ceil.java @@ -1,30 +1,30 @@ -package com.metaweb.gridworks.expr.functions.math; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Ceil implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1 && args[0] != null && args[0] instanceof Number) { - return (long) Math.ceil(((Number) args[0]).doubleValue()); - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a number"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the ceiling of a number"); - writer.key("params"); writer.value("number d"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.math; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Ceil implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1 && args[0] != null && args[0] instanceof Number) { + return (long) Math.ceil(((Number) args[0]).doubleValue()); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a number"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the ceiling of a number"); + writer.key("params"); writer.value("number d"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/math/Floor.java b/src/main/java/com/metaweb/gridworks/expr/functions/math/Floor.java index 6e1e9fec2..739693579 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/math/Floor.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/math/Floor.java @@ -1,31 +1,31 @@ -package com.metaweb.gridworks.expr.functions.math; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Floor implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1 && args[0] != null && args[0] instanceof Number) { - return (long) Math.floor(((Number) args[0]).doubleValue()); - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a number"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the floor of a number"); - writer.key("params"); writer.value("number d"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } - -} +package com.metaweb.gridworks.expr.functions.math; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Floor implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1 && args[0] != null && args[0] instanceof Number) { + return (long) Math.floor(((Number) args[0]).doubleValue()); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a number"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the floor of a number"); + writer.key("params"); writer.value("number d"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/math/Ln.java b/src/main/java/com/metaweb/gridworks/expr/functions/math/Ln.java index c8f94be5a..abf075840 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/math/Ln.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/math/Ln.java @@ -1,30 +1,30 @@ -package com.metaweb.gridworks.expr.functions.math; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Ln implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1 && args[0] != null && args[0] instanceof Number) { - return Math.log(((Number) args[0]).doubleValue()); - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a number"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the natural log of n"); - writer.key("params"); writer.value("number n"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.math; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Ln implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1 && args[0] != null && args[0] instanceof Number) { + return Math.log(((Number) args[0]).doubleValue()); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a number"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the natural log of n"); + writer.key("params"); writer.value("number n"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/math/Log.java b/src/main/java/com/metaweb/gridworks/expr/functions/math/Log.java index de8e909ab..1d1110b77 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/math/Log.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/math/Log.java @@ -1,30 +1,30 @@ -package com.metaweb.gridworks.expr.functions.math; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Log implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1 && args[0] != null && args[0] instanceof Number) { - return Math.log10(((Number) args[0]).doubleValue()); - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a number"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the base 10 log of n"); - writer.key("params"); writer.value("number n"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.math; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Log implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1 && args[0] != null && args[0] instanceof Number) { + return Math.log10(((Number) args[0]).doubleValue()); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a number"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the base 10 log of n"); + writer.key("params"); writer.value("number n"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/math/Max.java b/src/main/java/com/metaweb/gridworks/expr/functions/math/Max.java index c04f1b588..3c304e15b 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/math/Max.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/math/Max.java @@ -1,34 +1,34 @@ -package com.metaweb.gridworks.expr.functions.math; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Max implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 2 && - args[0] != null && args[0] instanceof Number && - args[1] != null && args[1] instanceof Number) { - return Math.max( - ((Number) args[0]).doubleValue(), - ((Number) args[1]).doubleValue()); - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 numbers"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the greater of two numbers"); - writer.key("params"); writer.value("number a, number b"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.math; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Max implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 2 && + args[0] != null && args[0] instanceof Number && + args[1] != null && args[1] instanceof Number) { + return Math.max( + ((Number) args[0]).doubleValue(), + ((Number) args[1]).doubleValue()); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 numbers"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the greater of two numbers"); + writer.key("params"); writer.value("number a, number b"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/math/Min.java b/src/main/java/com/metaweb/gridworks/expr/functions/math/Min.java index 22661f203..7c3bbad16 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/math/Min.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/math/Min.java @@ -1,34 +1,34 @@ -package com.metaweb.gridworks.expr.functions.math; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Min implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 2 && - args[0] != null && args[0] instanceof Number && - args[1] != null && args[1] instanceof Number) { - return Math.min( - ((Number) args[0]).doubleValue(), - ((Number) args[1]).doubleValue()); - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 numbers"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the smaller of two numbers"); - writer.key("params"); writer.value("number a, number b"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.math; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Min implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 2 && + args[0] != null && args[0] instanceof Number && + args[1] != null && args[1] instanceof Number) { + return Math.min( + ((Number) args[0]).doubleValue(), + ((Number) args[1]).doubleValue()); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 numbers"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the smaller of two numbers"); + writer.key("params"); writer.value("number a, number b"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/math/Mod.java b/src/main/java/com/metaweb/gridworks/expr/functions/math/Mod.java index 08909099b..edcea9611 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/math/Mod.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/math/Mod.java @@ -1,35 +1,35 @@ -package com.metaweb.gridworks.expr.functions.math; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Mod implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 2 && - args[0] != null && args[0] instanceof Number && - args[1] != null && args[1] instanceof Number) { - int a = ((Number) args[0]).intValue(); - int b = ((Number) args[0]).intValue(); - - return a % b; - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 numbers"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns a modulus b"); - writer.key("params"); writer.value("number a, number b"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.math; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Mod implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 2 && + args[0] != null && args[0] instanceof Number && + args[1] != null && args[1] instanceof Number) { + int a = ((Number) args[0]).intValue(); + int b = ((Number) args[0]).intValue(); + + return a % b; + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 numbers"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns a modulus b"); + writer.key("params"); writer.value("number a, number b"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/math/Round.java b/src/main/java/com/metaweb/gridworks/expr/functions/math/Round.java index 0b73bf1ea..85edc2cad 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/math/Round.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/math/Round.java @@ -1,30 +1,30 @@ -package com.metaweb.gridworks.expr.functions.math; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Round implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1 && args[0] != null && args[0] instanceof Number) { - return ((Number) args[0]).longValue(); - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a number"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns n rounded"); - writer.key("params"); writer.value("number n"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.math; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Round implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1 && args[0] != null && args[0] instanceof Number) { + return ((Number) args[0]).longValue(); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a number"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns n rounded"); + writer.key("params"); writer.value("number n"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/math/Sum.java b/src/main/java/com/metaweb/gridworks/expr/functions/math/Sum.java index 057c8f339..b9602564c 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/math/Sum.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/math/Sum.java @@ -1,59 +1,59 @@ -package com.metaweb.gridworks.expr.functions.math; - -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Sum implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1) { - Object v = args[0]; - - if (v != null && (v.getClass().isArray() || v instanceof List)) { - int length = v.getClass().isArray() ? - ((Object[]) v).length : - ExpressionUtils.toObjectList(v).size(); - - double total = 0; - - if (v.getClass().isArray()) { - Object[] a = (Object[]) v; - for (int i = 0; i < length; i++) { - Object n = a[length - i - 1]; - if (n instanceof Number) { - total += ((Number) n).doubleValue(); - } - } - } else { - List a = ExpressionUtils.toObjectList(v); - for (int i = 0; i < length; i++) { - Object n = a.get(length - i - 1); - if (n instanceof Number) { - total += ((Number) n).doubleValue(); - } - } - } - return total; - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array of numbers"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Sums numbers in array a"); - writer.key("params"); writer.value("array a"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.math; + +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Sum implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1) { + Object v = args[0]; + + if (v != null && (v.getClass().isArray() || v instanceof List)) { + int length = v.getClass().isArray() ? + ((Object[]) v).length : + ExpressionUtils.toObjectList(v).size(); + + double total = 0; + + if (v.getClass().isArray()) { + Object[] a = (Object[]) v; + for (int i = 0; i < length; i++) { + Object n = a[length - i - 1]; + if (n instanceof Number) { + total += ((Number) n).doubleValue(); + } + } + } else { + List a = ExpressionUtils.toObjectList(v); + for (int i = 0; i < length; i++) { + Object n = a.get(length - i - 1); + if (n instanceof Number) { + total += ((Number) n).doubleValue(); + } + } + } + return total; + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array of numbers"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Sums numbers in array a"); + writer.key("params"); writer.value("array a"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/strings/EndsWith.java b/src/main/java/com/metaweb/gridworks/expr/functions/strings/EndsWith.java index de5d4e13d..4c3322351 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/strings/EndsWith.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/strings/EndsWith.java @@ -1,34 +1,34 @@ -package com.metaweb.gridworks.expr.functions.strings; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class EndsWith implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 2) { - Object s1 = args[0]; - Object s2 = args[1]; - if (s1 != null && s2 != null && s1 instanceof String && s2 instanceof String) { - return ((String) s1).endsWith((String) s2); - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns whether s ends with sub"); - writer.key("params"); writer.value("string s, string sub"); - writer.key("returns"); writer.value("boolean"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.strings; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class EndsWith implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 2) { + Object s1 = args[0]; + Object s2 = args[1]; + if (s1 != null && s2 != null && s1 instanceof String && s2 instanceof String) { + return ((String) s1).endsWith((String) s2); + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns whether s ends with sub"); + writer.key("params"); writer.value("string s, string sub"); + writer.key("returns"); writer.value("boolean"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/strings/IndexOf.java b/src/main/java/com/metaweb/gridworks/expr/functions/strings/IndexOf.java index f434ccab8..faf5d95ff 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/strings/IndexOf.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/strings/IndexOf.java @@ -1,34 +1,34 @@ -package com.metaweb.gridworks.expr.functions.strings; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class IndexOf implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 2) { - Object s1 = args[0]; - Object s2 = args[1]; - if (s1 != null && s2 != null && s1 instanceof String && s2 instanceof String) { - return ((String) s1).indexOf((String) s2); - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the index of sub first ocurring in s"); - writer.key("params"); writer.value("string s, string sub"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.strings; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class IndexOf implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 2) { + Object s1 = args[0]; + Object s2 = args[1]; + if (s1 != null && s2 != null && s1 instanceof String && s2 instanceof String) { + return ((String) s1).indexOf((String) s2); + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the index of sub first ocurring in s"); + writer.key("params"); writer.value("string s, string sub"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/strings/LastIndexOf.java b/src/main/java/com/metaweb/gridworks/expr/functions/strings/LastIndexOf.java index d16fc2559..478c987e6 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/strings/LastIndexOf.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/strings/LastIndexOf.java @@ -1,35 +1,35 @@ -package com.metaweb.gridworks.expr.functions.strings; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class LastIndexOf implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 2) { - Object s1 = args[0]; - Object s2 = args[1]; - if (s1 != null && s2 != null && s1 instanceof String && s2 instanceof String) { - return ((String) s1).lastIndexOf((String) s2); - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the index of sub last ocurring in s"); - writer.key("params"); writer.value("string s, string sub"); - writer.key("returns"); writer.value("number"); - writer.endObject(); - } - -} +package com.metaweb.gridworks.expr.functions.strings; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class LastIndexOf implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 2) { + Object s1 = args[0]; + Object s2 = args[1]; + if (s1 != null && s2 != null && s1 instanceof String && s2 instanceof String) { + return ((String) s1).lastIndexOf((String) s2); + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the index of sub last ocurring in s"); + writer.key("params"); writer.value("string s, string sub"); + writer.key("returns"); writer.value("number"); + writer.endObject(); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/strings/Replace.java b/src/main/java/com/metaweb/gridworks/expr/functions/strings/Replace.java index 10ac2e2f1..a5505ae8f 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/strings/Replace.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/strings/Replace.java @@ -1,44 +1,44 @@ -package com.metaweb.gridworks.expr.functions.strings; - -import java.util.Properties; -import java.util.regex.Pattern; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Replace implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 3) { - Object o1 = args[0]; - Object o2 = args[1]; - Object o3 = args[2]; - if (o1 != null && o2 != null && o3 != null && o3 instanceof String) { - String str = (o1 instanceof String) ? (String) o1 : o1.toString(); - - if (o2 instanceof String) { - return str.replace((String) o2, (String) o3); - } else if (o2 instanceof Pattern) { - Pattern pattern = (Pattern) o2; - return pattern.matcher(str).replaceAll((String) o3); - } - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 3 strings, or 1 string, 1 regex, and 1 string"); - } - - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the string obtained by replacing f with r in s"); - writer.key("params"); writer.value("string s, string or regex f, string r"); - writer.key("returns"); writer.value("string"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.strings; + +import java.util.Properties; +import java.util.regex.Pattern; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Replace implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 3) { + Object o1 = args[0]; + Object o2 = args[1]; + Object o3 = args[2]; + if (o1 != null && o2 != null && o3 != null && o3 instanceof String) { + String str = (o1 instanceof String) ? (String) o1 : o1.toString(); + + if (o2 instanceof String) { + return str.replace((String) o2, (String) o3); + } else if (o2 instanceof Pattern) { + Pattern pattern = (Pattern) o2; + return pattern.matcher(str).replaceAll((String) o3); + } + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 3 strings, or 1 string, 1 regex, and 1 string"); + } + + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the string obtained by replacing f with r in s"); + writer.key("params"); writer.value("string s, string or regex f, string r"); + writer.key("returns"); writer.value("string"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/strings/Split.java b/src/main/java/com/metaweb/gridworks/expr/functions/strings/Split.java index 58298b4a9..f598c74c3 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/strings/Split.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/strings/Split.java @@ -1,42 +1,42 @@ -package com.metaweb.gridworks.expr.functions.strings; - -import java.util.Properties; -import java.util.regex.Pattern; - -import org.apache.commons.lang.StringUtils; -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class Split implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 2) { - Object v = args[0]; - Object split = args[1]; - if (v != null && split != null) { - String str = (v instanceof String ? (String) v : v.toString()); - if (split instanceof String) { - return StringUtils.splitByWholeSeparator(str, (String) split); - } else if (split instanceof Pattern) { - Pattern pattern = (Pattern) split; - return pattern.split(str); - } - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings, or 1 string and 1 regex"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the array of strings obtained by splitting s with separator sep"); - writer.key("params"); writer.value("string s, string or regex sep"); - writer.key("returns"); writer.value("array"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.strings; + +import java.util.Properties; +import java.util.regex.Pattern; + +import org.apache.commons.lang.StringUtils; +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class Split implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 2) { + Object v = args[0]; + Object split = args[1]; + if (v != null && split != null) { + String str = (v instanceof String ? (String) v : v.toString()); + if (split instanceof String) { + return StringUtils.splitByWholeSeparator(str, (String) split); + } else if (split instanceof Pattern) { + Pattern pattern = (Pattern) split; + return pattern.split(str); + } + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings, or 1 string and 1 regex"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the array of strings obtained by splitting s with separator sep"); + writer.key("params"); writer.value("string s, string or regex sep"); + writer.key("returns"); writer.value("array"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/strings/SplitByLengths.java b/src/main/java/com/metaweb/gridworks/expr/functions/strings/SplitByLengths.java index 8bb1dddd2..bdfcc3c7d 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/strings/SplitByLengths.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/strings/SplitByLengths.java @@ -1,48 +1,48 @@ -package com.metaweb.gridworks.expr.functions.strings; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class SplitByLengths implements Function { - public Object call(Properties bindings, Object[] args) { - if (args.length >= 2 && args[0] != null) { - Object o = args[0]; - String s = o instanceof String ? (String) o : o.toString(); - - String[] results = new String[args.length - 1]; - - int lastIndex = 0; - - for (int i = 1; i < args.length; i++) { - int thisIndex = lastIndex; - - Object o2 = args[i]; - if (o2 instanceof Number) { - thisIndex = Math.min(s.length(), lastIndex + Math.max(0, ((Number) o2).intValue())); - } - - results[i - 1] = s.substring(lastIndex, thisIndex); - lastIndex = thisIndex; - } - - return results; - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 1 string and 1 or more numbers"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns the array of strings obtained by splitting s into substrings with the given lengths"); - writer.key("params"); writer.value("string s, number n, ..."); - writer.key("returns"); writer.value("array"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.strings; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class SplitByLengths implements Function { + public Object call(Properties bindings, Object[] args) { + if (args.length >= 2 && args[0] != null) { + Object o = args[0]; + String s = o instanceof String ? (String) o : o.toString(); + + String[] results = new String[args.length - 1]; + + int lastIndex = 0; + + for (int i = 1; i < args.length; i++) { + int thisIndex = lastIndex; + + Object o2 = args[i]; + if (o2 instanceof Number) { + thisIndex = Math.min(s.length(), lastIndex + Math.max(0, ((Number) o2).intValue())); + } + + results[i - 1] = s.substring(lastIndex, thisIndex); + lastIndex = thisIndex; + } + + return results; + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 1 string and 1 or more numbers"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns the array of strings obtained by splitting s into substrings with the given lengths"); + writer.key("params"); writer.value("string s, number n, ..."); + writer.key("returns"); writer.value("array"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/strings/StartsWith.java b/src/main/java/com/metaweb/gridworks/expr/functions/strings/StartsWith.java index 981090570..31becf2b1 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/strings/StartsWith.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/strings/StartsWith.java @@ -1,33 +1,33 @@ -package com.metaweb.gridworks.expr.functions.strings; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class StartsWith implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 2) { - Object s1 = args[0]; - Object s2 = args[1]; - if (s1 != null && s2 != null && s1 instanceof String && s2 instanceof String) { - return ((String) s1).startsWith((String) s2); - } - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings"); - } - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns whether s starts with sub"); - writer.key("params"); writer.value("string s, string sub"); - writer.key("returns"); writer.value("boolean"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.strings; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class StartsWith implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 2) { + Object s1 = args[0]; + Object s2 = args[1]; + if (s1 != null && s2 != null && s1 instanceof String && s2 instanceof String) { + return ((String) s1).startsWith((String) s2); + } + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings"); + } + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns whether s starts with sub"); + writer.key("params"); writer.value("string s, string sub"); + writer.key("returns"); writer.value("boolean"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToLowercase.java b/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToLowercase.java index 2fb8fa6d0..4303687e8 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToLowercase.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToLowercase.java @@ -1,31 +1,31 @@ -package com.metaweb.gridworks.expr.functions.strings; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class ToLowercase implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1 && args[0] != null) { - Object o = args[0]; - return (o instanceof String ? (String) o : o.toString()).toLowerCase(); - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a string"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns s converted to lowercase"); - writer.key("params"); writer.value("string s"); - writer.key("returns"); writer.value("string"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.strings; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class ToLowercase implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1 && args[0] != null) { + Object o = args[0]; + return (o instanceof String ? (String) o : o.toString()).toLowerCase(); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a string"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns s converted to lowercase"); + writer.key("params"); writer.value("string s"); + writer.key("returns"); writer.value("string"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToTitlecase.java b/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToTitlecase.java index 90105755e..0e6e4259c 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToTitlecase.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToTitlecase.java @@ -1,35 +1,35 @@ -package com.metaweb.gridworks.expr.functions.strings; - -import java.util.Properties; - -import org.apache.commons.lang.WordUtils; -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class ToTitlecase implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1 && args[0] != null) { - Object o = args[0]; - String s = o instanceof String ? (String) o : o.toString(); - - return WordUtils.capitalizeFully(s); - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a string"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns s converted to titlecase"); - writer.key("params"); writer.value("string s"); - writer.key("returns"); writer.value("string"); - writer.endObject(); - } - -} +package com.metaweb.gridworks.expr.functions.strings; + +import java.util.Properties; + +import org.apache.commons.lang.WordUtils; +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class ToTitlecase implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1 && args[0] != null) { + Object o = args[0]; + String s = o instanceof String ? (String) o : o.toString(); + + return WordUtils.capitalizeFully(s); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a string"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns s converted to titlecase"); + writer.key("params"); writer.value("string s"); + writer.key("returns"); writer.value("string"); + writer.endObject(); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToUppercase.java b/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToUppercase.java index 37b0729e6..5cad55794 100644 --- a/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToUppercase.java +++ b/src/main/java/com/metaweb/gridworks/expr/functions/strings/ToUppercase.java @@ -1,31 +1,31 @@ -package com.metaweb.gridworks.expr.functions.strings; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.gel.ControlFunctionRegistry; -import com.metaweb.gridworks.gel.Function; - -public class ToUppercase implements Function { - - public Object call(Properties bindings, Object[] args) { - if (args.length == 1 && args[0] != null) { - Object o = args[0]; - return (o instanceof String ? (String) o : o.toString()).toUpperCase(); - } - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a string"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value("Returns s converted to uppercase"); - writer.key("params"); writer.value("string s"); - writer.key("returns"); writer.value("string"); - writer.endObject(); - } -} +package com.metaweb.gridworks.expr.functions.strings; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.gel.ControlFunctionRegistry; +import com.metaweb.gridworks.gel.Function; + +public class ToUppercase implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 1 && args[0] != null) { + Object o = args[0]; + return (o instanceof String ? (String) o : o.toString()).toUpperCase(); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a string"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns s converted to uppercase"); + writer.key("params"); writer.value("string s"); + writer.key("returns"); writer.value("string"); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/Control.java b/src/main/java/com/metaweb/gridworks/gel/Control.java index 4de85e3ad..ebd4a62a9 100644 --- a/src/main/java/com/metaweb/gridworks/gel/Control.java +++ b/src/main/java/com/metaweb/gridworks/gel/Control.java @@ -1,17 +1,17 @@ -package com.metaweb.gridworks.gel; - -import java.util.Properties; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.expr.Evaluable; - -/** - * Interface of GEL controls such as if, forEach, forNonBlank, with. A control can - * decide which part of the code to execute and can affect the environment bindings. - * Functions, on the other hand, can't do either. - */ -public interface Control extends Jsonizable { - public Object call(Properties bindings, Evaluable[] args); - - public String checkArguments(Evaluable[] args); -} +package com.metaweb.gridworks.gel; + +import java.util.Properties; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.expr.Evaluable; + +/** + * Interface of GEL controls such as if, forEach, forNonBlank, with. A control can + * decide which part of the code to execute and can affect the environment bindings. + * Functions, on the other hand, can't do either. + */ +public interface Control extends Jsonizable { + public Object call(Properties bindings, Evaluable[] args); + + public String checkArguments(Evaluable[] args); +} diff --git a/src/main/java/com/metaweb/gridworks/gel/Function.java b/src/main/java/com/metaweb/gridworks/gel/Function.java index a620e1f73..3e7a26d4b 100644 --- a/src/main/java/com/metaweb/gridworks/gel/Function.java +++ b/src/main/java/com/metaweb/gridworks/gel/Function.java @@ -1,13 +1,13 @@ -package com.metaweb.gridworks.gel; - -import java.util.Properties; - -import com.metaweb.gridworks.Jsonizable; - -/** - * Interface for functions. When a function is called, its arguments have already - * been evaluated down into non-error values. - */ -public interface Function extends Jsonizable { - public Object call(Properties bindings, Object[] args); -} +package com.metaweb.gridworks.gel; + +import java.util.Properties; + +import com.metaweb.gridworks.Jsonizable; + +/** + * Interface for functions. When a function is called, its arguments have already + * been evaluated down into non-error values. + */ +public interface Function extends Jsonizable { + public Object call(Properties bindings, Object[] args); +} diff --git a/src/main/java/com/metaweb/gridworks/gel/Parser.java b/src/main/java/com/metaweb/gridworks/gel/Parser.java index 887ad3498..30d5eb0b0 100644 --- a/src/main/java/com/metaweb/gridworks/gel/Parser.java +++ b/src/main/java/com/metaweb/gridworks/gel/Parser.java @@ -1,284 +1,284 @@ -package com.metaweb.gridworks.gel; - -import java.util.LinkedList; -import java.util.List; -import java.util.regex.Pattern; - -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ParsingException; -import com.metaweb.gridworks.gel.Scanner.NumberToken; -import com.metaweb.gridworks.gel.Scanner.RegexToken; -import com.metaweb.gridworks.gel.Scanner.Token; -import com.metaweb.gridworks.gel.Scanner.TokenType; -import com.metaweb.gridworks.gel.ast.ControlCallExpr; -import com.metaweb.gridworks.gel.ast.FieldAccessorExpr; -import com.metaweb.gridworks.gel.ast.FunctionCallExpr; -import com.metaweb.gridworks.gel.ast.LiteralExpr; -import com.metaweb.gridworks.gel.ast.OperatorCallExpr; -import com.metaweb.gridworks.gel.ast.VariableExpr; - -public class Parser { - protected Scanner _scanner; - protected Token _token; - protected Evaluable _root; - - public Parser(String s) throws ParsingException { - this(s, 0, s.length()); - } - - public Parser(String s, int from, int to) throws ParsingException { - _scanner = new Scanner(s, from, to); - _token = _scanner.next(true); - - _root = parseExpression(); - } - - public Evaluable getExpression() { - return _root; - } - - protected void next(boolean regexPossible) { - _token = _scanner.next(regexPossible); - } - - protected ParsingException makeException(String desc) { - int index = _token != null ? _token.start : _scanner.getIndex(); - - return new ParsingException("Parsing error at offset " + index + ": " + desc); - } - - /** - * := - * | [ "<" "<=" ">" ">=" "==" "!=" ] - */ - protected Evaluable parseExpression() throws ParsingException { - Evaluable sub = parseSubExpression(); - - while (_token != null && - _token.type == TokenType.Operator && - ">=<==!=".indexOf(_token.text) >= 0) { - - String op = _token.text; - - next(true); - - Evaluable sub2 = parseSubExpression(); - - sub = new OperatorCallExpr(new Evaluable[] { sub, sub2 }, op); - } - - return sub; - } - - /** - * := - * | [ "+" "-" ] - */ - protected Evaluable parseSubExpression() throws ParsingException { - Evaluable sub = parseTerm(); - - while (_token != null && - _token.type == TokenType.Operator && - "+-".indexOf(_token.text) >= 0) { - - String op = _token.text; - - next(true); - - Evaluable sub2 = parseSubExpression(); - - sub = new OperatorCallExpr(new Evaluable[] { sub, sub2 }, op); - } - - return sub; - } - - /** - * := - * | [ "*" "/" ] - */ - protected Evaluable parseTerm() throws ParsingException { - Evaluable factor = parseFactor(); - - while (_token != null && - _token.type == TokenType.Operator && - "*/".indexOf(_token.text) >= 0) { - - String op = _token.text; - - next(true); - - Evaluable factor2 = parseFactor(); - - factor = new OperatorCallExpr(new Evaluable[] { factor, factor2 }, op); - } - - return factor; - } - - /** - * := ( )* - * := - * | | - | | | - * ( ) - * - * := "[" "]" - * | "." - * | "." "(" ")" - * - */ - protected Evaluable parseFactor() throws ParsingException { - if (_token == null) { - throw makeException("Expecting something more at end of expression"); - } - - Evaluable eval = null; - - if (_token.type == TokenType.String) { - eval = new LiteralExpr(_token.text); - next(false); - } else if (_token.type == TokenType.Regex) { - RegexToken t = (RegexToken) _token; - - try { - Pattern pattern = Pattern.compile(_token.text, t.caseInsensitive ? Pattern.CASE_INSENSITIVE : 0); - eval = new LiteralExpr(pattern); - next(false); - } catch (Exception e) { - throw makeException("Bad regular expression (" + e.getMessage() + ")"); - } - } else if (_token.type == TokenType.Number) { - eval = new LiteralExpr(((NumberToken)_token).value); - next(false); - } else if (_token.type == TokenType.Operator && _token.text.equals("-")) { // unary minus? - next(true); - - if (_token != null && _token.type == TokenType.Number) { - eval = new LiteralExpr(-((NumberToken)_token).value); - next(false); - } else { - throw makeException("Bad negative number"); - } - } else if (_token.type == TokenType.Identifier) { - String text = _token.text; - next(false); - - if (_token == null || _token.type != TokenType.Delimiter || !_token.text.equals("(")) { - eval = new VariableExpr(text); - } else { - Function f = ControlFunctionRegistry.getFunction(text); - Control c = ControlFunctionRegistry.getControl(text); - if (f == null && c == null) { - throw makeException("Unknown function or control named " + text); - } - - next(true); // swallow ( - - List args = parseExpressionList(")"); - - if (c != null) { - Evaluable[] argsA = makeArray(args); - String errorMessage = c.checkArguments(argsA); - if (errorMessage != null) { - throw makeException(errorMessage); - } - eval = new ControlCallExpr(argsA, c); - } else { - eval = new FunctionCallExpr(makeArray(args), f); - } - } - } else if (_token.type == TokenType.Delimiter && _token.text.equals("(")) { - next(true); - - eval = parseExpression(); - - if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(")")) { - next(false); - } else { - throw makeException("Missing )"); - } - } else { - throw makeException("Missing number, string, identifier, regex, or parenthesized expression"); - } - - while (_token != null) { - if (_token.type == TokenType.Operator && _token.text.equals(".")) { - next(false); // swallow . - - if (_token == null || _token.type != TokenType.Identifier) { - throw makeException("Missing function name"); - } - - String identifier = _token.text; - next(false); - - if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals("(")) { - next(true); // swallow ( - - Function f = ControlFunctionRegistry.getFunction(identifier); - if (f == null) { - throw makeException("Unknown function " + identifier); - } - - List args = parseExpressionList(")"); - args.add(0, eval); - - eval = new FunctionCallExpr(makeArray(args), f); - } else { - eval = new FieldAccessorExpr(eval, identifier); - } - } else if (_token.type == TokenType.Delimiter && _token.text.equals("[")) { - next(true); // swallow [ - - List args = parseExpressionList("]"); - args.add(0, eval); - - eval = new FunctionCallExpr(makeArray(args), ControlFunctionRegistry.getFunction("get")); - } else { - break; - } - } - - return eval; - } - - /** - * := - * | ( "," )* - * - */ - protected List parseExpressionList(String closingDelimiter) throws ParsingException { - List l = new LinkedList(); - - if (_token != null && - (_token.type != TokenType.Delimiter || !_token.text.equals(closingDelimiter))) { - - while (_token != null) { - Evaluable eval = parseExpression(); - - l.add(eval); - - if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(",")) { - next(true); // swallow comma, loop back for more - } else { - break; - } - } - } - - if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(closingDelimiter)) { - next(false); // swallow closing delimiter - } else { - throw makeException("Missing " + closingDelimiter); - } - - return l; - } - - protected Evaluable[] makeArray(List l) { - Evaluable[] a = new Evaluable[l.size()]; - l.toArray(a); - - return a; - } -} +package com.metaweb.gridworks.gel; + +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Pattern; + +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ParsingException; +import com.metaweb.gridworks.gel.Scanner.NumberToken; +import com.metaweb.gridworks.gel.Scanner.RegexToken; +import com.metaweb.gridworks.gel.Scanner.Token; +import com.metaweb.gridworks.gel.Scanner.TokenType; +import com.metaweb.gridworks.gel.ast.ControlCallExpr; +import com.metaweb.gridworks.gel.ast.FieldAccessorExpr; +import com.metaweb.gridworks.gel.ast.FunctionCallExpr; +import com.metaweb.gridworks.gel.ast.LiteralExpr; +import com.metaweb.gridworks.gel.ast.OperatorCallExpr; +import com.metaweb.gridworks.gel.ast.VariableExpr; + +public class Parser { + protected Scanner _scanner; + protected Token _token; + protected Evaluable _root; + + public Parser(String s) throws ParsingException { + this(s, 0, s.length()); + } + + public Parser(String s, int from, int to) throws ParsingException { + _scanner = new Scanner(s, from, to); + _token = _scanner.next(true); + + _root = parseExpression(); + } + + public Evaluable getExpression() { + return _root; + } + + protected void next(boolean regexPossible) { + _token = _scanner.next(regexPossible); + } + + protected ParsingException makeException(String desc) { + int index = _token != null ? _token.start : _scanner.getIndex(); + + return new ParsingException("Parsing error at offset " + index + ": " + desc); + } + + /** + * := + * | [ "<" "<=" ">" ">=" "==" "!=" ] + */ + protected Evaluable parseExpression() throws ParsingException { + Evaluable sub = parseSubExpression(); + + while (_token != null && + _token.type == TokenType.Operator && + ">=<==!=".indexOf(_token.text) >= 0) { + + String op = _token.text; + + next(true); + + Evaluable sub2 = parseSubExpression(); + + sub = new OperatorCallExpr(new Evaluable[] { sub, sub2 }, op); + } + + return sub; + } + + /** + * := + * | [ "+" "-" ] + */ + protected Evaluable parseSubExpression() throws ParsingException { + Evaluable sub = parseTerm(); + + while (_token != null && + _token.type == TokenType.Operator && + "+-".indexOf(_token.text) >= 0) { + + String op = _token.text; + + next(true); + + Evaluable sub2 = parseSubExpression(); + + sub = new OperatorCallExpr(new Evaluable[] { sub, sub2 }, op); + } + + return sub; + } + + /** + * := + * | [ "*" "/" ] + */ + protected Evaluable parseTerm() throws ParsingException { + Evaluable factor = parseFactor(); + + while (_token != null && + _token.type == TokenType.Operator && + "*/".indexOf(_token.text) >= 0) { + + String op = _token.text; + + next(true); + + Evaluable factor2 = parseFactor(); + + factor = new OperatorCallExpr(new Evaluable[] { factor, factor2 }, op); + } + + return factor; + } + + /** + * := ( )* + * := + * | | - | | | + * ( ) + * + * := "[" "]" + * | "." + * | "." "(" ")" + * + */ + protected Evaluable parseFactor() throws ParsingException { + if (_token == null) { + throw makeException("Expecting something more at end of expression"); + } + + Evaluable eval = null; + + if (_token.type == TokenType.String) { + eval = new LiteralExpr(_token.text); + next(false); + } else if (_token.type == TokenType.Regex) { + RegexToken t = (RegexToken) _token; + + try { + Pattern pattern = Pattern.compile(_token.text, t.caseInsensitive ? Pattern.CASE_INSENSITIVE : 0); + eval = new LiteralExpr(pattern); + next(false); + } catch (Exception e) { + throw makeException("Bad regular expression (" + e.getMessage() + ")"); + } + } else if (_token.type == TokenType.Number) { + eval = new LiteralExpr(((NumberToken)_token).value); + next(false); + } else if (_token.type == TokenType.Operator && _token.text.equals("-")) { // unary minus? + next(true); + + if (_token != null && _token.type == TokenType.Number) { + eval = new LiteralExpr(-((NumberToken)_token).value); + next(false); + } else { + throw makeException("Bad negative number"); + } + } else if (_token.type == TokenType.Identifier) { + String text = _token.text; + next(false); + + if (_token == null || _token.type != TokenType.Delimiter || !_token.text.equals("(")) { + eval = new VariableExpr(text); + } else { + Function f = ControlFunctionRegistry.getFunction(text); + Control c = ControlFunctionRegistry.getControl(text); + if (f == null && c == null) { + throw makeException("Unknown function or control named " + text); + } + + next(true); // swallow ( + + List args = parseExpressionList(")"); + + if (c != null) { + Evaluable[] argsA = makeArray(args); + String errorMessage = c.checkArguments(argsA); + if (errorMessage != null) { + throw makeException(errorMessage); + } + eval = new ControlCallExpr(argsA, c); + } else { + eval = new FunctionCallExpr(makeArray(args), f); + } + } + } else if (_token.type == TokenType.Delimiter && _token.text.equals("(")) { + next(true); + + eval = parseExpression(); + + if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(")")) { + next(false); + } else { + throw makeException("Missing )"); + } + } else { + throw makeException("Missing number, string, identifier, regex, or parenthesized expression"); + } + + while (_token != null) { + if (_token.type == TokenType.Operator && _token.text.equals(".")) { + next(false); // swallow . + + if (_token == null || _token.type != TokenType.Identifier) { + throw makeException("Missing function name"); + } + + String identifier = _token.text; + next(false); + + if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals("(")) { + next(true); // swallow ( + + Function f = ControlFunctionRegistry.getFunction(identifier); + if (f == null) { + throw makeException("Unknown function " + identifier); + } + + List args = parseExpressionList(")"); + args.add(0, eval); + + eval = new FunctionCallExpr(makeArray(args), f); + } else { + eval = new FieldAccessorExpr(eval, identifier); + } + } else if (_token.type == TokenType.Delimiter && _token.text.equals("[")) { + next(true); // swallow [ + + List args = parseExpressionList("]"); + args.add(0, eval); + + eval = new FunctionCallExpr(makeArray(args), ControlFunctionRegistry.getFunction("get")); + } else { + break; + } + } + + return eval; + } + + /** + * := + * | ( "," )* + * + */ + protected List parseExpressionList(String closingDelimiter) throws ParsingException { + List l = new LinkedList(); + + if (_token != null && + (_token.type != TokenType.Delimiter || !_token.text.equals(closingDelimiter))) { + + while (_token != null) { + Evaluable eval = parseExpression(); + + l.add(eval); + + if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(",")) { + next(true); // swallow comma, loop back for more + } else { + break; + } + } + } + + if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(closingDelimiter)) { + next(false); // swallow closing delimiter + } else { + throw makeException("Missing " + closingDelimiter); + } + + return l; + } + + protected Evaluable[] makeArray(List l) { + Evaluable[] a = new Evaluable[l.size()]; + l.toArray(a); + + return a; + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/Scanner.java b/src/main/java/com/metaweb/gridworks/gel/Scanner.java index 26630ad40..00ebdfd39 100644 --- a/src/main/java/com/metaweb/gridworks/gel/Scanner.java +++ b/src/main/java/com/metaweb/gridworks/gel/Scanner.java @@ -1,304 +1,304 @@ -package com.metaweb.gridworks.gel; - -public class Scanner { - static public enum TokenType { - Error, - Delimiter, - Operator, - Identifier, - Number, - String, - Regex - } - - static public class Token { - final public int start; - final public int end; - final public TokenType type; - final public String text; - - Token(int start, int end, TokenType type, String text) { - this.start = start; - this.end = end; - this.type = type; - this.text = text; - } - } - - static public class ErrorToken extends Token { - final public String detail; // error detail - - public ErrorToken(int start, int end, String text, String detail) { - super(start, end, TokenType.Error, text); - this.detail = detail; - } - } - - static public class NumberToken extends Token { - final public double value; - - public NumberToken(int start, int end, String text, double value) { - super(start, end, TokenType.Number, text); - this.value = value; - } - } - - static public class RegexToken extends Token { - final public boolean caseInsensitive; - - public RegexToken(int start, int end, String text, boolean caseInsensitive) { - super(start, end, TokenType.Regex, text); - this.caseInsensitive = caseInsensitive; - } - } - - protected String _text; // input text to tokenize - protected int _index; // index of the next character to process - protected int _limit; // process up to this index - - public Scanner(String s) { - this(s, 0, s.length()); - } - - public Scanner(String s, int from, int to) { - _text = s; - _index = from; - _limit = to; - } - - public int getIndex() { - return _index; - } - - /** - * The regexPossible flag is used by the parser to hint the scanner what to do - * when it encounters a slash. Since the divide operator / and the opening - * delimiter of a regex literal are the same, but divide operators and regex - * literals can't occur at the same place in an expression, this flag is a cheap - * way to distinguish the two without having to look ahead. - * - * @param regexPossible - * @return - */ - public Token next(boolean regexPossible) { - // skip whitespace - while (_index < _limit && Character.isWhitespace(_text.charAt(_index))) { - _index++; - } - if (_index == _limit) { - return null; - } - - char c = _text.charAt(_index); - int start = _index; - String detail = null; - - if (Character.isDigit(c)) { // number literal - double value = 0; - - while (_index < _limit && Character.isDigit(c = _text.charAt(_index))) { - value = value * 10 + (c - '0'); - _index++; - } - - if (_index < _limit && c == '.') { - _index++; - - double division = 1; - while (_index < _limit && Character.isDigit(c = _text.charAt(_index))) { - value = value * 10 + (c - '0'); - division *= 10; - _index++; - } - - value /= division; - } - - // TODO: support exponent e notation - - return new NumberToken( - start, - _index, - _text.substring(start, _index), - value - ); - } else if (c == '"' || c == '\'') { - /* - * String Literal - */ - - StringBuffer sb = new StringBuffer(); - char delimiter = c; - - _index++; // skip opening delimiter - - while (_index < _limit) { - c = _text.charAt(_index); - if (c == delimiter) { - _index++; // skip closing delimiter - - return new Token( - start, - _index, - TokenType.String, - sb.toString() - ); - } else if (c == '\\') { - _index++; // skip escaping marker - if (_index < _limit) { - char c2 = _text.charAt(_index); - if (c2 == 't') { - sb.append('\t'); - } else if (c2 == 'n') { - sb.append('\n'); - } else if (c2 == 'r') { - sb.append('\r'); - } else if (c2 == '\\') { - sb.append('\\'); - } else { - sb.append(c2); - } - } - } else { - sb.append(c); - } - _index++; - } - - detail = "String not properly closed"; - // fall through - - } else if (Character.isLetter(c) || c == '_') { // identifier - while (_index < _limit) { - char c1 = _text.charAt(_index); - if (c1 == '_' || Character.isLetterOrDigit(c1)) { - _index++; - } else { - break; - } - } - - return new Token( - start, - _index, - TokenType.Identifier, - _text.substring(start, _index) - ); - } else if (c == '/' && regexPossible) { - /* - * Regex literal - */ - StringBuffer sb = new StringBuffer(); - - _index++; // skip opening delimiter - - while (_index < _limit) { - c = _text.charAt(_index); - if (c == '/') { - _index++; // skip closing delimiter - - boolean caseInsensitive = false; - if (_index < _limit && _text.charAt(_index) == 'i') { - caseInsensitive = true; - _index++; - } - - return new RegexToken( - start, - _index, - sb.toString(), - caseInsensitive - ); - } else if (c == '\\') { - sb.append(c); - - _index++; // skip escaping marker - if (_index < _limit) { - sb.append(_text.charAt(_index)); - } - } else { - sb.append(c); - } - _index++; - } - - detail = "Regex not properly closed"; - // fall through - } else if ("+-*/.".indexOf(c) >= 0) { // operator - _index++; - - return new Token( - start, - _index, - TokenType.Operator, - _text.substring(start, _index) - ); - } else if ("()[],".indexOf(c) >= 0) { // delimiter - _index++; - - return new Token( - start, - _index, - TokenType.Delimiter, - _text.substring(start, _index) - ); - } else if (c == '!' && _index < _limit - 1 && _text.charAt(_index + 1) == '=') { - _index += 2; - return new Token( - start, - _index, - TokenType.Operator, - _text.substring(start, _index) - ); - } else if (c == '<') { - if (_index < _limit - 1 && - (_text.charAt(_index + 1) == '=' || - _text.charAt(_index + 1) == '>')) { - - _index += 2; - return new Token( - start, - _index, - TokenType.Operator, - _text.substring(start, _index) - ); - } else { - _index++; - return new Token( - start, - _index, - TokenType.Operator, - _text.substring(start, _index) - ); - } - } else if (">=".indexOf(c) >= 0) { // operator - if (_index < _limit - 1 && _text.charAt(_index + 1) == '=') { - _index += 2; - return new Token( - start, - _index, - TokenType.Operator, - _text.substring(start, _index) - ); - } else { - _index++; - return new Token( - start, - _index, - TokenType.Operator, - _text.substring(start, _index) - ); - } - } else { - _index++; - detail = "Unrecognized symbol"; - } - - return new ErrorToken( - start, - _index, - _text.substring(start, _index), - detail - ); - } -} +package com.metaweb.gridworks.gel; + +public class Scanner { + static public enum TokenType { + Error, + Delimiter, + Operator, + Identifier, + Number, + String, + Regex + } + + static public class Token { + final public int start; + final public int end; + final public TokenType type; + final public String text; + + Token(int start, int end, TokenType type, String text) { + this.start = start; + this.end = end; + this.type = type; + this.text = text; + } + } + + static public class ErrorToken extends Token { + final public String detail; // error detail + + public ErrorToken(int start, int end, String text, String detail) { + super(start, end, TokenType.Error, text); + this.detail = detail; + } + } + + static public class NumberToken extends Token { + final public double value; + + public NumberToken(int start, int end, String text, double value) { + super(start, end, TokenType.Number, text); + this.value = value; + } + } + + static public class RegexToken extends Token { + final public boolean caseInsensitive; + + public RegexToken(int start, int end, String text, boolean caseInsensitive) { + super(start, end, TokenType.Regex, text); + this.caseInsensitive = caseInsensitive; + } + } + + protected String _text; // input text to tokenize + protected int _index; // index of the next character to process + protected int _limit; // process up to this index + + public Scanner(String s) { + this(s, 0, s.length()); + } + + public Scanner(String s, int from, int to) { + _text = s; + _index = from; + _limit = to; + } + + public int getIndex() { + return _index; + } + + /** + * The regexPossible flag is used by the parser to hint the scanner what to do + * when it encounters a slash. Since the divide operator / and the opening + * delimiter of a regex literal are the same, but divide operators and regex + * literals can't occur at the same place in an expression, this flag is a cheap + * way to distinguish the two without having to look ahead. + * + * @param regexPossible + * @return + */ + public Token next(boolean regexPossible) { + // skip whitespace + while (_index < _limit && Character.isWhitespace(_text.charAt(_index))) { + _index++; + } + if (_index == _limit) { + return null; + } + + char c = _text.charAt(_index); + int start = _index; + String detail = null; + + if (Character.isDigit(c)) { // number literal + double value = 0; + + while (_index < _limit && Character.isDigit(c = _text.charAt(_index))) { + value = value * 10 + (c - '0'); + _index++; + } + + if (_index < _limit && c == '.') { + _index++; + + double division = 1; + while (_index < _limit && Character.isDigit(c = _text.charAt(_index))) { + value = value * 10 + (c - '0'); + division *= 10; + _index++; + } + + value /= division; + } + + // TODO: support exponent e notation + + return new NumberToken( + start, + _index, + _text.substring(start, _index), + value + ); + } else if (c == '"' || c == '\'') { + /* + * String Literal + */ + + StringBuffer sb = new StringBuffer(); + char delimiter = c; + + _index++; // skip opening delimiter + + while (_index < _limit) { + c = _text.charAt(_index); + if (c == delimiter) { + _index++; // skip closing delimiter + + return new Token( + start, + _index, + TokenType.String, + sb.toString() + ); + } else if (c == '\\') { + _index++; // skip escaping marker + if (_index < _limit) { + char c2 = _text.charAt(_index); + if (c2 == 't') { + sb.append('\t'); + } else if (c2 == 'n') { + sb.append('\n'); + } else if (c2 == 'r') { + sb.append('\r'); + } else if (c2 == '\\') { + sb.append('\\'); + } else { + sb.append(c2); + } + } + } else { + sb.append(c); + } + _index++; + } + + detail = "String not properly closed"; + // fall through + + } else if (Character.isLetter(c) || c == '_') { // identifier + while (_index < _limit) { + char c1 = _text.charAt(_index); + if (c1 == '_' || Character.isLetterOrDigit(c1)) { + _index++; + } else { + break; + } + } + + return new Token( + start, + _index, + TokenType.Identifier, + _text.substring(start, _index) + ); + } else if (c == '/' && regexPossible) { + /* + * Regex literal + */ + StringBuffer sb = new StringBuffer(); + + _index++; // skip opening delimiter + + while (_index < _limit) { + c = _text.charAt(_index); + if (c == '/') { + _index++; // skip closing delimiter + + boolean caseInsensitive = false; + if (_index < _limit && _text.charAt(_index) == 'i') { + caseInsensitive = true; + _index++; + } + + return new RegexToken( + start, + _index, + sb.toString(), + caseInsensitive + ); + } else if (c == '\\') { + sb.append(c); + + _index++; // skip escaping marker + if (_index < _limit) { + sb.append(_text.charAt(_index)); + } + } else { + sb.append(c); + } + _index++; + } + + detail = "Regex not properly closed"; + // fall through + } else if ("+-*/.".indexOf(c) >= 0) { // operator + _index++; + + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } else if ("()[],".indexOf(c) >= 0) { // delimiter + _index++; + + return new Token( + start, + _index, + TokenType.Delimiter, + _text.substring(start, _index) + ); + } else if (c == '!' && _index < _limit - 1 && _text.charAt(_index + 1) == '=') { + _index += 2; + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } else if (c == '<') { + if (_index < _limit - 1 && + (_text.charAt(_index + 1) == '=' || + _text.charAt(_index + 1) == '>')) { + + _index += 2; + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } else { + _index++; + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } + } else if (">=".indexOf(c) >= 0) { // operator + if (_index < _limit - 1 && _text.charAt(_index + 1) == '=') { + _index += 2; + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } else { + _index++; + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } + } else { + _index++; + detail = "Unrecognized symbol"; + } + + return new ErrorToken( + start, + _index, + _text.substring(start, _index), + detail + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/ast/ControlCallExpr.java b/src/main/java/com/metaweb/gridworks/gel/ast/ControlCallExpr.java index 08272854e..6a7268939 100644 --- a/src/main/java/com/metaweb/gridworks/gel/ast/ControlCallExpr.java +++ b/src/main/java/com/metaweb/gridworks/gel/ast/ControlCallExpr.java @@ -1,37 +1,37 @@ -package com.metaweb.gridworks.gel.ast; - -import java.util.Properties; - -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.gel.Control; - -/** - * An abstract syntax tree node encapsulating a control call, such as "if". - */ -public class ControlCallExpr implements Evaluable { - final protected Evaluable[] _args; - final protected Control _control; - - public ControlCallExpr(Evaluable[] args, Control c) { - _args = args; - _control = c; - } - - public Object evaluate(Properties bindings) { - return _control.call(bindings, _args); - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - - for (Evaluable ev : _args) { - if (sb.length() > 0) { - sb.append(", "); - } - sb.append(ev.toString()); - } - - return _control.getClass().getSimpleName() + "(" + sb.toString() + ")"; - } -} +package com.metaweb.gridworks.gel.ast; + +import java.util.Properties; + +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.gel.Control; + +/** + * An abstract syntax tree node encapsulating a control call, such as "if". + */ +public class ControlCallExpr implements Evaluable { + final protected Evaluable[] _args; + final protected Control _control; + + public ControlCallExpr(Evaluable[] args, Control c) { + _args = args; + _control = c; + } + + public Object evaluate(Properties bindings) { + return _control.call(bindings, _args); + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + + for (Evaluable ev : _args) { + if (sb.length() > 0) { + sb.append(", "); + } + sb.append(ev.toString()); + } + + return _control.getClass().getSimpleName() + "(" + sb.toString() + ")"; + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/ast/FieldAccessorExpr.java b/src/main/java/com/metaweb/gridworks/gel/ast/FieldAccessorExpr.java index bb5a045da..7b7024cbb 100644 --- a/src/main/java/com/metaweb/gridworks/gel/ast/FieldAccessorExpr.java +++ b/src/main/java/com/metaweb/gridworks/gel/ast/FieldAccessorExpr.java @@ -1,41 +1,41 @@ -package com.metaweb.gridworks.gel.ast; - -import java.util.Properties; - -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.expr.HasFields; - -/** - * An abstract syntax tree node encapsulating a field accessor, - * e.g., "cell.value" is accessing the field named "value" on the - * variable called "cell". - */ -public class FieldAccessorExpr implements Evaluable { - final protected Evaluable _inner; - final protected String _fieldName; - - public FieldAccessorExpr(Evaluable inner, String fieldName) { - _inner = inner; - _fieldName = fieldName; - } - - public Object evaluate(Properties bindings) { - Object o = _inner.evaluate(bindings); - if (ExpressionUtils.isError(o)) { - return o; // bubble the error up - } else if (o == null) { - return new EvalError("Cannot retrieve field from null"); - } else if (o instanceof HasFields) { - return ((HasFields) o).getField(_fieldName, bindings); - } else { - return new EvalError("Object does not have any field, including " + _fieldName); - } - } - - @Override - public String toString() { - return _inner.toString() + "." + _fieldName; - } -} +package com.metaweb.gridworks.gel.ast; + +import java.util.Properties; + +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.expr.HasFields; + +/** + * An abstract syntax tree node encapsulating a field accessor, + * e.g., "cell.value" is accessing the field named "value" on the + * variable called "cell". + */ +public class FieldAccessorExpr implements Evaluable { + final protected Evaluable _inner; + final protected String _fieldName; + + public FieldAccessorExpr(Evaluable inner, String fieldName) { + _inner = inner; + _fieldName = fieldName; + } + + public Object evaluate(Properties bindings) { + Object o = _inner.evaluate(bindings); + if (ExpressionUtils.isError(o)) { + return o; // bubble the error up + } else if (o == null) { + return new EvalError("Cannot retrieve field from null"); + } else if (o instanceof HasFields) { + return ((HasFields) o).getField(_fieldName, bindings); + } else { + return new EvalError("Object does not have any field, including " + _fieldName); + } + } + + @Override + public String toString() { + return _inner.toString() + "." + _fieldName; + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/ast/FunctionCallExpr.java b/src/main/java/com/metaweb/gridworks/gel/ast/FunctionCallExpr.java index 058d71bf2..269632bf8 100644 --- a/src/main/java/com/metaweb/gridworks/gel/ast/FunctionCallExpr.java +++ b/src/main/java/com/metaweb/gridworks/gel/ast/FunctionCallExpr.java @@ -1,48 +1,48 @@ -package com.metaweb.gridworks.gel.ast; - -import java.util.Properties; - -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.gel.Function; - -/** - * An abstract syntax tree node encapsulating a function call. The function's - * arguments are all evaluated down to values before the function is applied. - * If any argument is an error, the function is not applied, and the error is - * the result of the expression. - */ -public class FunctionCallExpr implements Evaluable { - final protected Evaluable[] _args; - final protected Function _function; - - public FunctionCallExpr(Evaluable[] args, Function f) { - _args = args; - _function = f; - } - - public Object evaluate(Properties bindings) { - Object[] args = new Object[_args.length]; - for (int i = 0; i < _args.length; i++) { - Object v = _args[i].evaluate(bindings); - if (ExpressionUtils.isError(v)) { - return v; // bubble up the error - } - args[i] = v; - } - return _function.call(bindings, args); - } - - public String toString() { - StringBuffer sb = new StringBuffer(); - - for (Evaluable ev : _args) { - if (sb.length() > 0) { - sb.append(", "); - } - sb.append(ev.toString()); - } - - return _function.getClass().getSimpleName() + "(" + sb.toString() + ")"; - } -} +package com.metaweb.gridworks.gel.ast; + +import java.util.Properties; + +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.gel.Function; + +/** + * An abstract syntax tree node encapsulating a function call. The function's + * arguments are all evaluated down to values before the function is applied. + * If any argument is an error, the function is not applied, and the error is + * the result of the expression. + */ +public class FunctionCallExpr implements Evaluable { + final protected Evaluable[] _args; + final protected Function _function; + + public FunctionCallExpr(Evaluable[] args, Function f) { + _args = args; + _function = f; + } + + public Object evaluate(Properties bindings) { + Object[] args = new Object[_args.length]; + for (int i = 0; i < _args.length; i++) { + Object v = _args[i].evaluate(bindings); + if (ExpressionUtils.isError(v)) { + return v; // bubble up the error + } + args[i] = v; + } + return _function.call(bindings, args); + } + + public String toString() { + StringBuffer sb = new StringBuffer(); + + for (Evaluable ev : _args) { + if (sb.length() > 0) { + sb.append(", "); + } + sb.append(ev.toString()); + } + + return _function.getClass().getSimpleName() + "(" + sb.toString() + ")"; + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/ast/LiteralExpr.java b/src/main/java/com/metaweb/gridworks/gel/ast/LiteralExpr.java index 34337ef23..9e5e666dc 100644 --- a/src/main/java/com/metaweb/gridworks/gel/ast/LiteralExpr.java +++ b/src/main/java/com/metaweb/gridworks/gel/ast/LiteralExpr.java @@ -1,26 +1,26 @@ -package com.metaweb.gridworks.gel.ast; - -import java.util.Properties; - -import org.json.JSONObject; - -import com.metaweb.gridworks.expr.Evaluable; - -/** - * An abstract syntax tree node encapsulating a literal value. - */ -public class LiteralExpr implements Evaluable { - final protected Object _value; - - public LiteralExpr(Object value) { - _value = value; - } - - public Object evaluate(Properties bindings) { - return _value; - } - - public String toString() { - return _value instanceof String ? JSONObject.quote((String) _value) : _value.toString(); - } -} +package com.metaweb.gridworks.gel.ast; + +import java.util.Properties; + +import org.json.JSONObject; + +import com.metaweb.gridworks.expr.Evaluable; + +/** + * An abstract syntax tree node encapsulating a literal value. + */ +public class LiteralExpr implements Evaluable { + final protected Object _value; + + public LiteralExpr(Object value) { + _value = value; + } + + public Object evaluate(Properties bindings) { + return _value; + } + + public String toString() { + return _value instanceof String ? JSONObject.quote((String) _value) : _value.toString(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/ast/OperatorCallExpr.java b/src/main/java/com/metaweb/gridworks/gel/ast/OperatorCallExpr.java index 0731b1e41..632a74fdc 100644 --- a/src/main/java/com/metaweb/gridworks/gel/ast/OperatorCallExpr.java +++ b/src/main/java/com/metaweb/gridworks/gel/ast/OperatorCallExpr.java @@ -1,89 +1,89 @@ -package com.metaweb.gridworks.gel.ast; - -import java.util.Properties; - -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; - -/** - * An abstract syntax tree node encapsulating an operator call, such as "+". - */ -public class OperatorCallExpr implements Evaluable { - final protected Evaluable[] _args; - final protected String _op; - - public OperatorCallExpr(Evaluable[] args, String op) { - _args = args; - _op = op; - } - - public Object evaluate(Properties bindings) { - Object[] args = new Object[_args.length]; - for (int i = 0; i < _args.length; i++) { - Object v = _args[i].evaluate(bindings); - if (ExpressionUtils.isError(v)) { - return v; - } - args[i] = v; - } - - if (args.length == 2) { - if (args[0] != null && args[1] != null) { - if (args[0] instanceof Number && args[1] instanceof Number) { - if ("+".equals(_op)) { - return ((Number) args[0]).doubleValue() + ((Number) args[1]).doubleValue(); - } else if ("-".equals(_op)) { - return ((Number) args[0]).doubleValue() - ((Number) args[1]).doubleValue(); - } else if ("*".equals(_op)) { - return ((Number) args[0]).doubleValue() * ((Number) args[1]).doubleValue(); - } else if ("/".equals(_op)) { - return ((Number) args[0]).doubleValue() / ((Number) args[1]).doubleValue(); - } else if (">".equals(_op)) { - return ((Number) args[0]).doubleValue() > ((Number) args[1]).doubleValue(); - } else if (">=".equals(_op)) { - return ((Number) args[0]).doubleValue() >= ((Number) args[1]).doubleValue(); - } else if ("<".equals(_op)) { - return ((Number) args[0]).doubleValue() < ((Number) args[1]).doubleValue(); - } else if ("<=".equals(_op)) { - return ((Number) args[0]).doubleValue() <= ((Number) args[1]).doubleValue(); - } - } - - if ("+".equals(_op)) { - return args[0].toString() + args[1].toString(); - } - } - - if ("==".equals(_op)) { - if (args[0] != null) { - return args[0].equals(args[1]); - } else { - return args[1] == null; - } - } else if ("!=".equals(_op)) { - if (args[0] != null) { - return !args[0].equals(args[1]); - } else { - return args[1] != null; - } - } - } - return null; - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - - for (Evaluable ev : _args) { - if (sb.length() > 0) { - sb.append(' '); - sb.append(_op); - sb.append(' '); - } - sb.append(ev.toString()); - } - - return sb.toString(); - } -} +package com.metaweb.gridworks.gel.ast; + +import java.util.Properties; + +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; + +/** + * An abstract syntax tree node encapsulating an operator call, such as "+". + */ +public class OperatorCallExpr implements Evaluable { + final protected Evaluable[] _args; + final protected String _op; + + public OperatorCallExpr(Evaluable[] args, String op) { + _args = args; + _op = op; + } + + public Object evaluate(Properties bindings) { + Object[] args = new Object[_args.length]; + for (int i = 0; i < _args.length; i++) { + Object v = _args[i].evaluate(bindings); + if (ExpressionUtils.isError(v)) { + return v; + } + args[i] = v; + } + + if (args.length == 2) { + if (args[0] != null && args[1] != null) { + if (args[0] instanceof Number && args[1] instanceof Number) { + if ("+".equals(_op)) { + return ((Number) args[0]).doubleValue() + ((Number) args[1]).doubleValue(); + } else if ("-".equals(_op)) { + return ((Number) args[0]).doubleValue() - ((Number) args[1]).doubleValue(); + } else if ("*".equals(_op)) { + return ((Number) args[0]).doubleValue() * ((Number) args[1]).doubleValue(); + } else if ("/".equals(_op)) { + return ((Number) args[0]).doubleValue() / ((Number) args[1]).doubleValue(); + } else if (">".equals(_op)) { + return ((Number) args[0]).doubleValue() > ((Number) args[1]).doubleValue(); + } else if (">=".equals(_op)) { + return ((Number) args[0]).doubleValue() >= ((Number) args[1]).doubleValue(); + } else if ("<".equals(_op)) { + return ((Number) args[0]).doubleValue() < ((Number) args[1]).doubleValue(); + } else if ("<=".equals(_op)) { + return ((Number) args[0]).doubleValue() <= ((Number) args[1]).doubleValue(); + } + } + + if ("+".equals(_op)) { + return args[0].toString() + args[1].toString(); + } + } + + if ("==".equals(_op)) { + if (args[0] != null) { + return args[0].equals(args[1]); + } else { + return args[1] == null; + } + } else if ("!=".equals(_op)) { + if (args[0] != null) { + return !args[0].equals(args[1]); + } else { + return args[1] != null; + } + } + } + return null; + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + + for (Evaluable ev : _args) { + if (sb.length() > 0) { + sb.append(' '); + sb.append(_op); + sb.append(' '); + } + sb.append(ev.toString()); + } + + return sb.toString(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/ast/VariableExpr.java b/src/main/java/com/metaweb/gridworks/gel/ast/VariableExpr.java index 9fc574d59..e1546e4b9 100644 --- a/src/main/java/com/metaweb/gridworks/gel/ast/VariableExpr.java +++ b/src/main/java/com/metaweb/gridworks/gel/ast/VariableExpr.java @@ -1,28 +1,28 @@ -package com.metaweb.gridworks.gel.ast; - -import java.util.Properties; - -import com.metaweb.gridworks.expr.Evaluable; - -/** - * An abstract syntax tree node encapsulating the retrieval of a variable's content. - */ -public class VariableExpr implements Evaluable { - final protected String _name; - - public VariableExpr(String name) { - _name = name; - } - - public Object evaluate(Properties bindings) { - return bindings.get(_name); - } - - public String toString() { - return _name; - } - - public String getName() { - return _name; - } -} +package com.metaweb.gridworks.gel.ast; + +import java.util.Properties; + +import com.metaweb.gridworks.expr.Evaluable; + +/** + * An abstract syntax tree node encapsulating the retrieval of a variable's content. + */ +public class VariableExpr implements Evaluable { + final protected String _name; + + public VariableExpr(String name) { + _name = name; + } + + public Object evaluate(Properties bindings) { + return bindings.get(_name); + } + + public String toString() { + return _name; + } + + public String getName() { + return _name; + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/controls/IsBlank.java b/src/main/java/com/metaweb/gridworks/gel/controls/IsBlank.java index 3e7fad2f7..a3b2424a0 100644 --- a/src/main/java/com/metaweb/gridworks/gel/controls/IsBlank.java +++ b/src/main/java/com/metaweb/gridworks/gel/controls/IsBlank.java @@ -1,15 +1,15 @@ -package com.metaweb.gridworks.gel.controls; - -import com.metaweb.gridworks.expr.ExpressionUtils; - -public class IsBlank extends IsTest { - @Override - protected String getDescription() { - return "Returns whether o is null or an empty string"; - } - - @Override - protected boolean test(Object o) { - return !ExpressionUtils.isNonBlankData(o); - } -} +package com.metaweb.gridworks.gel.controls; + +import com.metaweb.gridworks.expr.ExpressionUtils; + +public class IsBlank extends IsTest { + @Override + protected String getDescription() { + return "Returns whether o is null or an empty string"; + } + + @Override + protected boolean test(Object o) { + return !ExpressionUtils.isNonBlankData(o); + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/controls/IsError.java b/src/main/java/com/metaweb/gridworks/gel/controls/IsError.java index 52ba9923f..a40647c9d 100644 --- a/src/main/java/com/metaweb/gridworks/gel/controls/IsError.java +++ b/src/main/java/com/metaweb/gridworks/gel/controls/IsError.java @@ -1,15 +1,15 @@ -package com.metaweb.gridworks.gel.controls; - -import com.metaweb.gridworks.expr.ExpressionUtils; - -public class IsError extends IsTest { - @Override - protected String getDescription() { - return "Returns whether o is an error"; - } - - @Override - protected boolean test(Object o) { - return ExpressionUtils.isError(o); - } -} +package com.metaweb.gridworks.gel.controls; + +import com.metaweb.gridworks.expr.ExpressionUtils; + +public class IsError extends IsTest { + @Override + protected String getDescription() { + return "Returns whether o is an error"; + } + + @Override + protected boolean test(Object o) { + return ExpressionUtils.isError(o); + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/controls/IsNonBlank.java b/src/main/java/com/metaweb/gridworks/gel/controls/IsNonBlank.java index e6fa15076..7a03db1e8 100644 --- a/src/main/java/com/metaweb/gridworks/gel/controls/IsNonBlank.java +++ b/src/main/java/com/metaweb/gridworks/gel/controls/IsNonBlank.java @@ -1,15 +1,15 @@ -package com.metaweb.gridworks.gel.controls; - -import com.metaweb.gridworks.expr.ExpressionUtils; - -public class IsNonBlank extends IsTest { - @Override - protected String getDescription() { - return "Returns whether o is not null and not an empty string"; - } - - @Override - protected boolean test(Object o) { - return ExpressionUtils.isNonBlankData(o); - } -} +package com.metaweb.gridworks.gel.controls; + +import com.metaweb.gridworks.expr.ExpressionUtils; + +public class IsNonBlank extends IsTest { + @Override + protected String getDescription() { + return "Returns whether o is not null and not an empty string"; + } + + @Override + protected boolean test(Object o) { + return ExpressionUtils.isNonBlankData(o); + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/controls/IsNotNull.java b/src/main/java/com/metaweb/gridworks/gel/controls/IsNotNull.java index eb5fd39df..e4e66f686 100644 --- a/src/main/java/com/metaweb/gridworks/gel/controls/IsNotNull.java +++ b/src/main/java/com/metaweb/gridworks/gel/controls/IsNotNull.java @@ -1,13 +1,13 @@ -package com.metaweb.gridworks.gel.controls; - -public class IsNotNull extends IsTest { - @Override - protected String getDescription() { - return "Returns whether o is not null"; - } - - @Override - protected boolean test(Object o) { - return o != null; - } -} +package com.metaweb.gridworks.gel.controls; + +public class IsNotNull extends IsTest { + @Override + protected String getDescription() { + return "Returns whether o is not null"; + } + + @Override + protected boolean test(Object o) { + return o != null; + } +} diff --git a/src/main/java/com/metaweb/gridworks/gel/controls/IsNull.java b/src/main/java/com/metaweb/gridworks/gel/controls/IsNull.java index e6dc9c8f8..0848448e8 100644 --- a/src/main/java/com/metaweb/gridworks/gel/controls/IsNull.java +++ b/src/main/java/com/metaweb/gridworks/gel/controls/IsNull.java @@ -1,13 +1,13 @@ -package com.metaweb.gridworks.gel.controls; - -public class IsNull extends IsTest { - @Override - protected String getDescription() { - return "Returns whether o is null"; - } - - @Override - protected boolean test(Object o) { - return o == null; - } -} +package com.metaweb.gridworks.gel.controls; + +public class IsNull extends IsTest { + @Override + protected String getDescription() { + return "Returns whether o is null"; + } + + @Override + protected boolean test(Object o) { + return o == null; + } +} diff --git a/src/main/java/com/metaweb/gridworks/history/Change.java b/src/main/java/com/metaweb/gridworks/history/Change.java index 02fdfe97a..1e98ab363 100644 --- a/src/main/java/com/metaweb/gridworks/history/Change.java +++ b/src/main/java/com/metaweb/gridworks/history/Change.java @@ -1,21 +1,21 @@ -package com.metaweb.gridworks.history; - -import java.io.IOException; -import java.io.Writer; -import java.util.Properties; - -import com.metaweb.gridworks.model.Project; - -/** - * Interface for a concrete change to a project's data. A change should consist - * of new values already computed. When apply() is called, the change should not - * spend any more time computing anything. It should simply save existing values - * and swap in new values. Similarly, when revert() is called, the change - * should only swap old values back in. - */ -public interface Change { - public void apply(Project project); - public void revert(Project project); - - public void save(Writer writer, Properties options) throws IOException; -} +package com.metaweb.gridworks.history; + +import java.io.IOException; +import java.io.Writer; +import java.util.Properties; + +import com.metaweb.gridworks.model.Project; + +/** + * Interface for a concrete change to a project's data. A change should consist + * of new values already computed. When apply() is called, the change should not + * spend any more time computing anything. It should simply save existing values + * and swap in new values. Similarly, when revert() is called, the change + * should only swap old values back in. + */ +public interface Change { + public void apply(Project project); + public void revert(Project project); + + public void save(Writer writer, Properties options) throws IOException; +} diff --git a/src/main/java/com/metaweb/gridworks/history/History.java b/src/main/java/com/metaweb/gridworks/history/History.java index e82333ec8..3eba08dbd 100644 --- a/src/main/java/com/metaweb/gridworks/history/History.java +++ b/src/main/java/com/metaweb/gridworks/history/History.java @@ -1,269 +1,269 @@ -package com.metaweb.gridworks.history; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.LineNumberReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Gridworks; -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.ProjectManager; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.util.Pool; - -/** - * Track done and undone changes. Done changes can be undone; undone changes can be redone. - * Each change is actually not tracked directly but through a history entry. The history - * entry stores only the metadata, while the change object stores the actual data. Thus - * the history entries are much smaller and can be kept in memory, while the change objects - * are only loaded into memory on demand. - */ -public class History implements Jsonizable { - static public Change readOneChange(InputStream in, Pool pool) throws Exception { - LineNumberReader reader = new LineNumberReader(new InputStreamReader(in)); - try { - return readOneChange(reader, pool); - } finally { - reader.close(); - } - } - - static public Change readOneChange(LineNumberReader reader, Pool pool) throws Exception { - /* String version = */ reader.readLine(); - - String className = reader.readLine(); - Class klass = getChangeClass(className); - - Method load = klass.getMethod("load", LineNumberReader.class, Pool.class); - - return (Change) load.invoke(null, reader, pool); - } - - static public void writeOneChange(OutputStream out, Change change, Pool pool) throws Exception { - Writer writer = new OutputStreamWriter(out); - try { - History.writeOneChange(writer, change, pool); - } finally { - writer.flush(); - } - } - - static public void writeOneChange(Writer writer, Change change, Pool pool) throws Exception { - writer.write(Gridworks.getVersion()); writer.write('\n'); - writer.write(change.getClass().getName()); writer.write('\n'); - - Properties options = new Properties(); - options.setProperty("mode", "save"); - options.put("pool", pool); - - change.save(writer, options); - } - - @SuppressWarnings("unchecked") - static public Class getChangeClass(String className) throws ClassNotFoundException { - return (Class) Class.forName(className); - } - - protected long _projectID; - protected List _pastEntries; // done changes, can be undone - protected List _futureEntries; // undone changes, can be redone - - public History(Project project) { - _projectID = project.id; - _pastEntries = new ArrayList(); - _futureEntries = new ArrayList(); - } - - public void addEntry(HistoryEntry entry) { - entry.apply(ProjectManager.singleton.getProject(_projectID)); - _pastEntries.add(entry); - - setModified(); - - // Any new change will clear all future entries. - List futureEntries = _futureEntries; - _futureEntries = new ArrayList(); - - for (HistoryEntry entry2 : futureEntries) { - try { - // remove residual data on disk - entry2.delete(); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - - protected void setModified() { - ProjectManager.singleton.getProjectMetadata(_projectID).updateModified(); - } - - public List getLastPastEntries(int count) { - if (count <= 0) { - return new LinkedList(_pastEntries); - } else { - return _pastEntries.subList(Math.max(_pastEntries.size() - count, 0), _pastEntries.size()); - } - } - - public void undoRedo(long lastDoneEntryID) { - if (lastDoneEntryID == 0) { - // undo all the way back to the start of the project - undo(_pastEntries.size()); - } else { - for (int i = 0; i < _pastEntries.size(); i++) { - if (_pastEntries.get(i).id == lastDoneEntryID) { - undo(_pastEntries.size() - i - 1); - return; - } - } - - for (int i = 0; i < _futureEntries.size(); i++) { - if (_futureEntries.get(i).id == lastDoneEntryID) { - redo(i + 1); - return; - } - } - } - } - - public long getPrecedingEntryID(long entryID) { - if (entryID == 0) { - return -1; - } else { - for (int i = 0; i < _pastEntries.size(); i++) { - if (_pastEntries.get(i).id == entryID) { - return i == 0 ? 0 : _pastEntries.get(i - 1).id; - } - } - - for (int i = 0; i < _futureEntries.size(); i++) { - if (_futureEntries.get(i).id == entryID) { - if (i > 0) { - return _futureEntries.get(i - 1).id; - } else if (_pastEntries.size() > 0) { - return _pastEntries.get(_pastEntries.size() - 1).id; - } else { - return 0; - } - } - } - } - return -1; - } - - protected HistoryEntry getEntry(long entryID) { - for (int i = 0; i < _pastEntries.size(); i++) { - if (_pastEntries.get(i).id == entryID) { - return _pastEntries.get(i); - } - } - - for (int i = 0; i < _futureEntries.size(); i++) { - if (_futureEntries.get(i).id == entryID) { - return _futureEntries.get(i); - } - } - return null; - } - - protected void undo(int times) { - Project project = ProjectManager.singleton.getProject(_projectID); - - while (times > 0 && _pastEntries.size() > 0) { - HistoryEntry entry = _pastEntries.get(_pastEntries.size() - 1); - - entry.revert(project); - - setModified(); - times--; - - _pastEntries.remove(_pastEntries.size() - 1); - _futureEntries.add(0, entry); - } - } - - protected void redo(int times) { - Project project = ProjectManager.singleton.getProject(_projectID); - - while (times > 0 && _futureEntries.size() > 0) { - HistoryEntry entry = _futureEntries.get(0); - - entry.apply(project); - - setModified(); - times--; - - _pastEntries.add(entry); - _futureEntries.remove(0); - } - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - - writer.key("past"); writer.array(); - for (HistoryEntry entry : _pastEntries) { - entry.write(writer, options); - } - writer.endArray(); - - writer.key("future"); writer.array(); - for (HistoryEntry entry : _futureEntries) { - entry.write(writer, options); - } - writer.endArray(); - - writer.endObject(); - } - - public void save(Writer writer, Properties options) throws IOException { - writer.write("pastEntryCount="); writer.write(Integer.toString(_pastEntries.size())); writer.write('\n'); - for (HistoryEntry entry : _pastEntries) { - entry.save(writer, options); writer.write('\n'); - } - - writer.write("futureEntryCount="); writer.write(Integer.toString(_futureEntries.size())); writer.write('\n'); - for (HistoryEntry entry : _futureEntries) { - entry.save(writer, options); writer.write('\n'); - } - - writer.write("/e/\n"); - } - - public void load(Project project, LineNumberReader reader) throws Exception { - String line; - while ((line = reader.readLine()) != null && !"/e/".equals(line)) { - int equal = line.indexOf('='); - CharSequence field = line.subSequence(0, equal); - String value = line.substring(equal + 1); - - if ("pastEntryCount".equals(field)) { - int count = Integer.parseInt(value); - - for (int i = 0; i < count; i++) { - _pastEntries.add(HistoryEntry.load(project, reader.readLine())); - } - } else if ("futureEntryCount".equals(field)) { - int count = Integer.parseInt(value); - - for (int i = 0; i < count; i++) { - _futureEntries.add(HistoryEntry.load(project, reader.readLine())); - } - } - } - } -} +package com.metaweb.gridworks.history; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Gridworks; +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.util.Pool; + +/** + * Track done and undone changes. Done changes can be undone; undone changes can be redone. + * Each change is actually not tracked directly but through a history entry. The history + * entry stores only the metadata, while the change object stores the actual data. Thus + * the history entries are much smaller and can be kept in memory, while the change objects + * are only loaded into memory on demand. + */ +public class History implements Jsonizable { + static public Change readOneChange(InputStream in, Pool pool) throws Exception { + LineNumberReader reader = new LineNumberReader(new InputStreamReader(in)); + try { + return readOneChange(reader, pool); + } finally { + reader.close(); + } + } + + static public Change readOneChange(LineNumberReader reader, Pool pool) throws Exception { + /* String version = */ reader.readLine(); + + String className = reader.readLine(); + Class klass = getChangeClass(className); + + Method load = klass.getMethod("load", LineNumberReader.class, Pool.class); + + return (Change) load.invoke(null, reader, pool); + } + + static public void writeOneChange(OutputStream out, Change change, Pool pool) throws Exception { + Writer writer = new OutputStreamWriter(out); + try { + History.writeOneChange(writer, change, pool); + } finally { + writer.flush(); + } + } + + static public void writeOneChange(Writer writer, Change change, Pool pool) throws Exception { + writer.write(Gridworks.getVersion()); writer.write('\n'); + writer.write(change.getClass().getName()); writer.write('\n'); + + Properties options = new Properties(); + options.setProperty("mode", "save"); + options.put("pool", pool); + + change.save(writer, options); + } + + @SuppressWarnings("unchecked") + static public Class getChangeClass(String className) throws ClassNotFoundException { + return (Class) Class.forName(className); + } + + protected long _projectID; + protected List _pastEntries; // done changes, can be undone + protected List _futureEntries; // undone changes, can be redone + + public History(Project project) { + _projectID = project.id; + _pastEntries = new ArrayList(); + _futureEntries = new ArrayList(); + } + + public void addEntry(HistoryEntry entry) { + entry.apply(ProjectManager.singleton.getProject(_projectID)); + _pastEntries.add(entry); + + setModified(); + + // Any new change will clear all future entries. + List futureEntries = _futureEntries; + _futureEntries = new ArrayList(); + + for (HistoryEntry entry2 : futureEntries) { + try { + // remove residual data on disk + entry2.delete(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + protected void setModified() { + ProjectManager.singleton.getProjectMetadata(_projectID).updateModified(); + } + + public List getLastPastEntries(int count) { + if (count <= 0) { + return new LinkedList(_pastEntries); + } else { + return _pastEntries.subList(Math.max(_pastEntries.size() - count, 0), _pastEntries.size()); + } + } + + public void undoRedo(long lastDoneEntryID) { + if (lastDoneEntryID == 0) { + // undo all the way back to the start of the project + undo(_pastEntries.size()); + } else { + for (int i = 0; i < _pastEntries.size(); i++) { + if (_pastEntries.get(i).id == lastDoneEntryID) { + undo(_pastEntries.size() - i - 1); + return; + } + } + + for (int i = 0; i < _futureEntries.size(); i++) { + if (_futureEntries.get(i).id == lastDoneEntryID) { + redo(i + 1); + return; + } + } + } + } + + public long getPrecedingEntryID(long entryID) { + if (entryID == 0) { + return -1; + } else { + for (int i = 0; i < _pastEntries.size(); i++) { + if (_pastEntries.get(i).id == entryID) { + return i == 0 ? 0 : _pastEntries.get(i - 1).id; + } + } + + for (int i = 0; i < _futureEntries.size(); i++) { + if (_futureEntries.get(i).id == entryID) { + if (i > 0) { + return _futureEntries.get(i - 1).id; + } else if (_pastEntries.size() > 0) { + return _pastEntries.get(_pastEntries.size() - 1).id; + } else { + return 0; + } + } + } + } + return -1; + } + + protected HistoryEntry getEntry(long entryID) { + for (int i = 0; i < _pastEntries.size(); i++) { + if (_pastEntries.get(i).id == entryID) { + return _pastEntries.get(i); + } + } + + for (int i = 0; i < _futureEntries.size(); i++) { + if (_futureEntries.get(i).id == entryID) { + return _futureEntries.get(i); + } + } + return null; + } + + protected void undo(int times) { + Project project = ProjectManager.singleton.getProject(_projectID); + + while (times > 0 && _pastEntries.size() > 0) { + HistoryEntry entry = _pastEntries.get(_pastEntries.size() - 1); + + entry.revert(project); + + setModified(); + times--; + + _pastEntries.remove(_pastEntries.size() - 1); + _futureEntries.add(0, entry); + } + } + + protected void redo(int times) { + Project project = ProjectManager.singleton.getProject(_projectID); + + while (times > 0 && _futureEntries.size() > 0) { + HistoryEntry entry = _futureEntries.get(0); + + entry.apply(project); + + setModified(); + times--; + + _pastEntries.add(entry); + _futureEntries.remove(0); + } + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + + writer.key("past"); writer.array(); + for (HistoryEntry entry : _pastEntries) { + entry.write(writer, options); + } + writer.endArray(); + + writer.key("future"); writer.array(); + for (HistoryEntry entry : _futureEntries) { + entry.write(writer, options); + } + writer.endArray(); + + writer.endObject(); + } + + public void save(Writer writer, Properties options) throws IOException { + writer.write("pastEntryCount="); writer.write(Integer.toString(_pastEntries.size())); writer.write('\n'); + for (HistoryEntry entry : _pastEntries) { + entry.save(writer, options); writer.write('\n'); + } + + writer.write("futureEntryCount="); writer.write(Integer.toString(_futureEntries.size())); writer.write('\n'); + for (HistoryEntry entry : _futureEntries) { + entry.save(writer, options); writer.write('\n'); + } + + writer.write("/e/\n"); + } + + public void load(Project project, LineNumberReader reader) throws Exception { + String line; + while ((line = reader.readLine()) != null && !"/e/".equals(line)) { + int equal = line.indexOf('='); + CharSequence field = line.subSequence(0, equal); + String value = line.substring(equal + 1); + + if ("pastEntryCount".equals(field)) { + int count = Integer.parseInt(value); + + for (int i = 0; i < count; i++) { + _pastEntries.add(HistoryEntry.load(project, reader.readLine())); + } + } else if ("futureEntryCount".equals(field)) { + int count = Integer.parseInt(value); + + for (int i = 0; i < count; i++) { + _futureEntries.add(HistoryEntry.load(project, reader.readLine())); + } + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/history/HistoryEntry.java b/src/main/java/com/metaweb/gridworks/history/HistoryEntry.java index b959220b9..4cffde6bd 100644 --- a/src/main/java/com/metaweb/gridworks/history/HistoryEntry.java +++ b/src/main/java/com/metaweb/gridworks/history/HistoryEntry.java @@ -1,209 +1,209 @@ -package com.metaweb.gridworks.history; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStreamReader; -import java.io.Writer; -import java.util.Date; -import java.util.Properties; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.ProjectManager; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.operations.OperationRegistry; -import com.metaweb.gridworks.util.ParsingUtilities; -import com.metaweb.gridworks.util.Pool; - -/** - * This is the metadata of a Change. It's small, so we can load it in order to - * obtain information about a change without actually loading the change. - */ -public class HistoryEntry implements Jsonizable { - final public long id; - final public long projectID; - final public String description; - final public Date time; - - // the abstract operation, if any, that results in the change - final public AbstractOperation operation; - - // the actual change, loaded on demand - transient protected Change _change; - - private final static String OPERATION = "operation"; - - static public long allocateID() { - return Math.round(Math.random() * 1000000) + System.currentTimeMillis(); - } - - public HistoryEntry(long id, Project project, String description, AbstractOperation operation, Change change) { - this.id = id; - this.projectID = project.id; - this.description = description; - this.operation = operation; - this.time = new Date(); - - _change = change; - } - - protected HistoryEntry(long id, long projectID, String description, AbstractOperation operation, Date time) { - this.id = id; - this.projectID = projectID; - this.description = description; - this.operation = operation; - this.time = time; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("id"); writer.value(id); - writer.key("description"); writer.value(description); - writer.key("time"); writer.value(ParsingUtilities.dateToString(time)); - if ("save".equals(options.getProperty("mode")) && operation != null) { - writer.key(OPERATION); operation.write(writer, options); - } - writer.endObject(); - } - - public void apply(Project project) { - if (_change == null) { - loadChange(); - } - - synchronized (project) { - _change.apply(project); - - // When a change is applied, it can hang on to old data (in order to be able - // to revert later). Hence, we need to save the change out. - - try { - saveChange(); - } catch (Exception e) { - e.printStackTrace(); - - _change.revert(project); - - throw new RuntimeException("Failed to apply change", e); - } - } - } - - public void revert(Project project) { - if (_change == null) { - loadChange(); - } - _change.revert(project); - } - - public void delete() { - File file = getChangeFile(); - if (file.exists()) { - file.delete(); - } - } - - public void save(Writer writer, Properties options) { - JSONWriter jsonWriter = new JSONWriter(writer); - try { - write(jsonWriter, options); - } catch (JSONException e) { - e.printStackTrace(); - } - } - - static public HistoryEntry load(Project project, String s) throws Exception { - JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(s); - - AbstractOperation operation = null; - if (obj.has(OPERATION) && !obj.isNull(OPERATION)) { - operation = OperationRegistry.reconstruct(project, obj.getJSONObject(OPERATION)); - } - - return new HistoryEntry( - obj.getLong("id"), - project.id, - obj.getString("description"), - operation, - ParsingUtilities.stringToDate(obj.getString("time")) - ); - } - - - protected void loadChange() { - File changeFile = getChangeFile(); - - try { - loadChange(changeFile); - } catch (Exception e) { - throw new RuntimeException("Failed to load change file " + changeFile.getAbsolutePath(), e); - } - } - - protected void loadChange(File file) throws Exception { - ZipFile zipFile = new ZipFile(file); - try { - Pool pool = new Pool(); - ZipEntry poolEntry = zipFile.getEntry("pool.txt"); - if (poolEntry != null) { - pool.load(new InputStreamReader( - zipFile.getInputStream(poolEntry))); - } // else, it's a legacy project file - - _change = History.readOneChange( - zipFile.getInputStream(zipFile.getEntry("change.txt")), pool); - } finally { - zipFile.close(); - } - } - - protected void saveChange() throws Exception { - File changeFile = getChangeFile(); - if (!(changeFile.exists())) { - saveChange(changeFile); - } - } - - protected void saveChange(File file) throws Exception { - ZipOutputStream out = new ZipOutputStream(new FileOutputStream(file)); - try { - Pool pool = new Pool(); - - out.putNextEntry(new ZipEntry("change.txt")); - try { - History.writeOneChange(out, _change, pool); - } finally { - out.closeEntry(); - } - - out.putNextEntry(new ZipEntry("pool.txt")); - try { - pool.save(out); - } finally { - out.closeEntry(); - } - } finally { - out.close(); - } - } - - protected File getChangeFile() { - return new File(getHistoryDir(), id + ".change.zip"); - } - - protected File getHistoryDir() { - File dir = new File(ProjectManager.singleton.getProjectDir(projectID), "history"); - dir.mkdirs(); - - return dir; - } -} +package com.metaweb.gridworks.history; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStreamReader; +import java.io.Writer; +import java.util.Date; +import java.util.Properties; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.operations.OperationRegistry; +import com.metaweb.gridworks.util.ParsingUtilities; +import com.metaweb.gridworks.util.Pool; + +/** + * This is the metadata of a Change. It's small, so we can load it in order to + * obtain information about a change without actually loading the change. + */ +public class HistoryEntry implements Jsonizable { + final public long id; + final public long projectID; + final public String description; + final public Date time; + + // the abstract operation, if any, that results in the change + final public AbstractOperation operation; + + // the actual change, loaded on demand + transient protected Change _change; + + private final static String OPERATION = "operation"; + + static public long allocateID() { + return Math.round(Math.random() * 1000000) + System.currentTimeMillis(); + } + + public HistoryEntry(long id, Project project, String description, AbstractOperation operation, Change change) { + this.id = id; + this.projectID = project.id; + this.description = description; + this.operation = operation; + this.time = new Date(); + + _change = change; + } + + protected HistoryEntry(long id, long projectID, String description, AbstractOperation operation, Date time) { + this.id = id; + this.projectID = projectID; + this.description = description; + this.operation = operation; + this.time = time; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("id"); writer.value(id); + writer.key("description"); writer.value(description); + writer.key("time"); writer.value(ParsingUtilities.dateToString(time)); + if ("save".equals(options.getProperty("mode")) && operation != null) { + writer.key(OPERATION); operation.write(writer, options); + } + writer.endObject(); + } + + public void apply(Project project) { + if (_change == null) { + loadChange(); + } + + synchronized (project) { + _change.apply(project); + + // When a change is applied, it can hang on to old data (in order to be able + // to revert later). Hence, we need to save the change out. + + try { + saveChange(); + } catch (Exception e) { + e.printStackTrace(); + + _change.revert(project); + + throw new RuntimeException("Failed to apply change", e); + } + } + } + + public void revert(Project project) { + if (_change == null) { + loadChange(); + } + _change.revert(project); + } + + public void delete() { + File file = getChangeFile(); + if (file.exists()) { + file.delete(); + } + } + + public void save(Writer writer, Properties options) { + JSONWriter jsonWriter = new JSONWriter(writer); + try { + write(jsonWriter, options); + } catch (JSONException e) { + e.printStackTrace(); + } + } + + static public HistoryEntry load(Project project, String s) throws Exception { + JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(s); + + AbstractOperation operation = null; + if (obj.has(OPERATION) && !obj.isNull(OPERATION)) { + operation = OperationRegistry.reconstruct(project, obj.getJSONObject(OPERATION)); + } + + return new HistoryEntry( + obj.getLong("id"), + project.id, + obj.getString("description"), + operation, + ParsingUtilities.stringToDate(obj.getString("time")) + ); + } + + + protected void loadChange() { + File changeFile = getChangeFile(); + + try { + loadChange(changeFile); + } catch (Exception e) { + throw new RuntimeException("Failed to load change file " + changeFile.getAbsolutePath(), e); + } + } + + protected void loadChange(File file) throws Exception { + ZipFile zipFile = new ZipFile(file); + try { + Pool pool = new Pool(); + ZipEntry poolEntry = zipFile.getEntry("pool.txt"); + if (poolEntry != null) { + pool.load(new InputStreamReader( + zipFile.getInputStream(poolEntry))); + } // else, it's a legacy project file + + _change = History.readOneChange( + zipFile.getInputStream(zipFile.getEntry("change.txt")), pool); + } finally { + zipFile.close(); + } + } + + protected void saveChange() throws Exception { + File changeFile = getChangeFile(); + if (!(changeFile.exists())) { + saveChange(changeFile); + } + } + + protected void saveChange(File file) throws Exception { + ZipOutputStream out = new ZipOutputStream(new FileOutputStream(file)); + try { + Pool pool = new Pool(); + + out.putNextEntry(new ZipEntry("change.txt")); + try { + History.writeOneChange(out, _change, pool); + } finally { + out.closeEntry(); + } + + out.putNextEntry(new ZipEntry("pool.txt")); + try { + pool.save(out); + } finally { + out.closeEntry(); + } + } finally { + out.close(); + } + } + + protected File getChangeFile() { + return new File(getHistoryDir(), id + ".change.zip"); + } + + protected File getHistoryDir() { + File dir = new File(ProjectManager.singleton.getProjectDir(projectID), "history"); + dir.mkdirs(); + + return dir; + } +} diff --git a/src/main/java/com/metaweb/gridworks/history/HistoryProcess.java b/src/main/java/com/metaweb/gridworks/history/HistoryProcess.java index 2fd5284f8..626ef1b97 100644 --- a/src/main/java/com/metaweb/gridworks/history/HistoryProcess.java +++ b/src/main/java/com/metaweb/gridworks/history/HistoryProcess.java @@ -1,73 +1,73 @@ -package com.metaweb.gridworks.history; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.process.Process; -import com.metaweb.gridworks.process.ProcessManager; - -/** - * The process for undoing or redoing. This involves calling apply() and revert() - * on changes. - */ -public class HistoryProcess extends Process { - final protected Project _project; - final protected long _lastDoneID; - final protected String _description; - - protected boolean _done = false; - - private final static String WARN = "Not a long-running process"; - - public HistoryProcess(Project project, long lastDoneID) { - _project = project; - _lastDoneID = lastDoneID; - - if (_lastDoneID == 0) { - _description = "Undo all"; - } else { - HistoryEntry entry = _project.history.getEntry(_lastDoneID); - _description = "Undo/redo until after " + entry.description; - } - } - - public void cancel() { - throw new RuntimeException(WARN); - } - - public boolean isImmediate() { - return true; - } - - public HistoryEntry performImmediate() { - _project.history.undoRedo(_lastDoneID); - _done = true; - - return null; - } - - public void startPerforming(ProcessManager manager) { - throw new RuntimeException(WARN); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("description"); writer.value(_description); - writer.key("immediate"); writer.value(true); - writer.key("status"); writer.value(_done ? "done" : "pending"); - writer.endObject(); - } - - public boolean isDone() { - throw new RuntimeException(WARN); - } - - public boolean isRunning() { - throw new RuntimeException(WARN); - } -} +package com.metaweb.gridworks.history; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.process.Process; +import com.metaweb.gridworks.process.ProcessManager; + +/** + * The process for undoing or redoing. This involves calling apply() and revert() + * on changes. + */ +public class HistoryProcess extends Process { + final protected Project _project; + final protected long _lastDoneID; + final protected String _description; + + protected boolean _done = false; + + private final static String WARN = "Not a long-running process"; + + public HistoryProcess(Project project, long lastDoneID) { + _project = project; + _lastDoneID = lastDoneID; + + if (_lastDoneID == 0) { + _description = "Undo all"; + } else { + HistoryEntry entry = _project.history.getEntry(_lastDoneID); + _description = "Undo/redo until after " + entry.description; + } + } + + public void cancel() { + throw new RuntimeException(WARN); + } + + public boolean isImmediate() { + return true; + } + + public HistoryEntry performImmediate() { + _project.history.undoRedo(_lastDoneID); + _done = true; + + return null; + } + + public void startPerforming(ProcessManager manager) { + throw new RuntimeException(WARN); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value(_description); + writer.key("immediate"); writer.value(true); + writer.key("status"); writer.value(_done ? "done" : "pending"); + writer.endObject(); + } + + public boolean isDone() { + throw new RuntimeException(WARN); + } + + public boolean isRunning() { + throw new RuntimeException(WARN); + } +} diff --git a/src/main/java/com/metaweb/gridworks/importers/ExcelImporter.java b/src/main/java/com/metaweb/gridworks/importers/ExcelImporter.java index cbcbd4cf4..5005a4e13 100644 --- a/src/main/java/com/metaweb/gridworks/importers/ExcelImporter.java +++ b/src/main/java/com/metaweb/gridworks/importers/ExcelImporter.java @@ -1,251 +1,251 @@ -package com.metaweb.gridworks.importers; - -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.commons.lang.NotImplementedException; -import org.apache.poi.common.usermodel.Hyperlink; -import org.apache.poi.hssf.usermodel.HSSFDateUtil; -import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.apache.poi.poifs.filesystem.POIFSFileSystem; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; - -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.Recon.Judgment; - -public class ExcelImporter implements Importer { - final protected boolean _xmlBased; - - public ExcelImporter(boolean xmlBased) { - _xmlBased = xmlBased; - } - - public boolean takesReader() { - return false; - } - - public void read(Reader reader, Project project, Properties options) throws Exception { - throw new NotImplementedException(); - } - - public void read(InputStream inputStream, Project project, Properties options) throws Exception { - int ignoreLines = ImporterUtilities.getIntegerOption("ignore", options, -1); - int limit = ImporterUtilities.getIntegerOption("limit",options,-1); - int skip = ImporterUtilities.getIntegerOption("skip",options,0); - - Workbook wb = null; - try { - wb = _xmlBased ? - new XSSFWorkbook(inputStream) : - new HSSFWorkbook(new POIFSFileSystem(inputStream)); - } catch (IOException e) { - throw new IOException( - "Attempted to parse file as Excel file but failed. " + - "Try to use Excel to re-save the file as a different Excel version or as TSV and upload again.", - e - ); - } - - Sheet sheet = wb.getSheetAt(0); - - int firstRow = sheet.getFirstRowNum(); - int lastRow = sheet.getLastRowNum(); - int r = firstRow; - - List nonBlankIndices = null; - List nonBlankHeaderStrings = null; - - /* - * Find the header row - */ - for (; r <= lastRow; r++) { - org.apache.poi.ss.usermodel.Row row = sheet.getRow(r); - if (row == null) { - continue; - } else if (ignoreLines > 0) { - ignoreLines--; - continue; - } - - short firstCell = row.getFirstCellNum(); - short lastCell = row.getLastCellNum(); - if (firstCell >= 0 && firstCell <= lastCell) { - nonBlankIndices = new ArrayList(lastCell - firstCell + 1); - nonBlankHeaderStrings = new ArrayList(lastCell - firstCell + 1); - - for (int c = firstCell; c <= lastCell; c++) { - org.apache.poi.ss.usermodel.Cell cell = row.getCell(c); - if (cell != null) { - String text = cell.getStringCellValue().trim(); - if (text.length() > 0) { - nonBlankIndices.add((int) c); - nonBlankHeaderStrings.add(text); - } - } - } - - if (nonBlankIndices.size() > 0) { - r++; - break; - } - } - } - - if (nonBlankIndices == null || nonBlankIndices.size() == 0) { - return; - } - - /* - * Create columns - */ - Map nameToIndex = new HashMap(); - for (int c = 0; c < nonBlankIndices.size(); c++) { - String cell = nonBlankHeaderStrings.get(c); - if (nameToIndex.containsKey(cell)) { - int index = nameToIndex.get(cell); - nameToIndex.put(cell, index + 1); - - cell = cell.contains(" ") ? (cell + " " + index) : (cell + index); - } else { - nameToIndex.put(cell, 2); - } - - Column column = new Column(c, cell); - project.columnModel.columns.add(column); - } - - /* - * Now process the data rows - */ - int rowsWithData = 0; - Map reconMap = new HashMap(); - - for (; r <= lastRow; r++) { - org.apache.poi.ss.usermodel.Row row = sheet.getRow(r); - if (row == null) { - continue; - } - - short firstCell = row.getFirstCellNum(); - short lastCell = row.getLastCellNum(); - if (firstCell >= 0 && firstCell <= lastCell) { - Row newRow = new Row(nonBlankIndices.size()); - boolean hasData = false; - - for (int c = 0; c < nonBlankIndices.size(); c++) { - if (c < firstCell || c > lastCell) { - continue; - } - - org.apache.poi.ss.usermodel.Cell cell = row.getCell(c); - if (cell == null) { - continue; - } - - int cellType = cell.getCellType(); - if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_ERROR || - cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_BLANK) { - continue; - } - if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_FORMULA) { - cellType = cell.getCachedFormulaResultType(); - } - - Serializable value = null; - if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_BOOLEAN) { - value = cell.getBooleanCellValue(); - } else if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_NUMERIC) { - double d = cell.getNumericCellValue(); - - if (HSSFDateUtil.isCellDateFormatted(cell)) { - value = HSSFDateUtil.getJavaDate(d); - } else { - value = d; - } - } else { - String text = cell.getStringCellValue().trim(); - if (text.length() > 0) { - value = text; - } - } - - if (value != null) { - Recon recon = null; - - Hyperlink hyperlink = cell.getHyperlink(); - if (hyperlink != null) { - String url = hyperlink.getAddress(); - - if (url.startsWith("http://") || - url.startsWith("https://")) { - - final String sig = "freebase.com/view"; - - int i = url.indexOf(sig); - if (i > 0) { - String id = url.substring(i + sig.length()); - - int q = id.indexOf('?'); - if (q > 0) { - id = id.substring(0, q); - } - int h = id.indexOf('#'); - if (h > 0) { - id = id.substring(0, h); - } - - if (reconMap.containsKey(id)) { - recon = reconMap.get(id); - recon.judgmentBatchSize++; - } else { - recon = new Recon(0); - recon.service = "import"; - recon.match = new ReconCandidate(id, "", value.toString(), new String[0], 100); - recon.matchRank = 0; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - recon.judgmentBatchSize = 1; - recon.addCandidate(recon.match); - - reconMap.put(id, recon); - } - - } - } - } - - newRow.setCell(c, new Cell(value, recon)); - hasData = true; - } - } - - if (hasData) { - rowsWithData++; - - if (skip <= 0 || rowsWithData > skip) { - project.rows.add(newRow); - project.columnModel.setMaxCellIndex(newRow.cells.size()); - - if (limit > 0 && project.rows.size() >= limit) { - break; - } - } - } - } - } - } -} +package com.metaweb.gridworks.importers; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.commons.lang.NotImplementedException; +import org.apache.poi.common.usermodel.Hyperlink; +import org.apache.poi.hssf.usermodel.HSSFDateUtil; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.Recon.Judgment; + +public class ExcelImporter implements Importer { + final protected boolean _xmlBased; + + public ExcelImporter(boolean xmlBased) { + _xmlBased = xmlBased; + } + + public boolean takesReader() { + return false; + } + + public void read(Reader reader, Project project, Properties options) throws Exception { + throw new NotImplementedException(); + } + + public void read(InputStream inputStream, Project project, Properties options) throws Exception { + int ignoreLines = ImporterUtilities.getIntegerOption("ignore", options, -1); + int limit = ImporterUtilities.getIntegerOption("limit",options,-1); + int skip = ImporterUtilities.getIntegerOption("skip",options,0); + + Workbook wb = null; + try { + wb = _xmlBased ? + new XSSFWorkbook(inputStream) : + new HSSFWorkbook(new POIFSFileSystem(inputStream)); + } catch (IOException e) { + throw new IOException( + "Attempted to parse file as Excel file but failed. " + + "Try to use Excel to re-save the file as a different Excel version or as TSV and upload again.", + e + ); + } + + Sheet sheet = wb.getSheetAt(0); + + int firstRow = sheet.getFirstRowNum(); + int lastRow = sheet.getLastRowNum(); + int r = firstRow; + + List nonBlankIndices = null; + List nonBlankHeaderStrings = null; + + /* + * Find the header row + */ + for (; r <= lastRow; r++) { + org.apache.poi.ss.usermodel.Row row = sheet.getRow(r); + if (row == null) { + continue; + } else if (ignoreLines > 0) { + ignoreLines--; + continue; + } + + short firstCell = row.getFirstCellNum(); + short lastCell = row.getLastCellNum(); + if (firstCell >= 0 && firstCell <= lastCell) { + nonBlankIndices = new ArrayList(lastCell - firstCell + 1); + nonBlankHeaderStrings = new ArrayList(lastCell - firstCell + 1); + + for (int c = firstCell; c <= lastCell; c++) { + org.apache.poi.ss.usermodel.Cell cell = row.getCell(c); + if (cell != null) { + String text = cell.getStringCellValue().trim(); + if (text.length() > 0) { + nonBlankIndices.add((int) c); + nonBlankHeaderStrings.add(text); + } + } + } + + if (nonBlankIndices.size() > 0) { + r++; + break; + } + } + } + + if (nonBlankIndices == null || nonBlankIndices.size() == 0) { + return; + } + + /* + * Create columns + */ + Map nameToIndex = new HashMap(); + for (int c = 0; c < nonBlankIndices.size(); c++) { + String cell = nonBlankHeaderStrings.get(c); + if (nameToIndex.containsKey(cell)) { + int index = nameToIndex.get(cell); + nameToIndex.put(cell, index + 1); + + cell = cell.contains(" ") ? (cell + " " + index) : (cell + index); + } else { + nameToIndex.put(cell, 2); + } + + Column column = new Column(c, cell); + project.columnModel.columns.add(column); + } + + /* + * Now process the data rows + */ + int rowsWithData = 0; + Map reconMap = new HashMap(); + + for (; r <= lastRow; r++) { + org.apache.poi.ss.usermodel.Row row = sheet.getRow(r); + if (row == null) { + continue; + } + + short firstCell = row.getFirstCellNum(); + short lastCell = row.getLastCellNum(); + if (firstCell >= 0 && firstCell <= lastCell) { + Row newRow = new Row(nonBlankIndices.size()); + boolean hasData = false; + + for (int c = 0; c < nonBlankIndices.size(); c++) { + if (c < firstCell || c > lastCell) { + continue; + } + + org.apache.poi.ss.usermodel.Cell cell = row.getCell(c); + if (cell == null) { + continue; + } + + int cellType = cell.getCellType(); + if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_ERROR || + cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_BLANK) { + continue; + } + if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_FORMULA) { + cellType = cell.getCachedFormulaResultType(); + } + + Serializable value = null; + if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_BOOLEAN) { + value = cell.getBooleanCellValue(); + } else if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_NUMERIC) { + double d = cell.getNumericCellValue(); + + if (HSSFDateUtil.isCellDateFormatted(cell)) { + value = HSSFDateUtil.getJavaDate(d); + } else { + value = d; + } + } else { + String text = cell.getStringCellValue().trim(); + if (text.length() > 0) { + value = text; + } + } + + if (value != null) { + Recon recon = null; + + Hyperlink hyperlink = cell.getHyperlink(); + if (hyperlink != null) { + String url = hyperlink.getAddress(); + + if (url.startsWith("http://") || + url.startsWith("https://")) { + + final String sig = "freebase.com/view"; + + int i = url.indexOf(sig); + if (i > 0) { + String id = url.substring(i + sig.length()); + + int q = id.indexOf('?'); + if (q > 0) { + id = id.substring(0, q); + } + int h = id.indexOf('#'); + if (h > 0) { + id = id.substring(0, h); + } + + if (reconMap.containsKey(id)) { + recon = reconMap.get(id); + recon.judgmentBatchSize++; + } else { + recon = new Recon(0); + recon.service = "import"; + recon.match = new ReconCandidate(id, "", value.toString(), new String[0], 100); + recon.matchRank = 0; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.judgmentBatchSize = 1; + recon.addCandidate(recon.match); + + reconMap.put(id, recon); + } + + } + } + } + + newRow.setCell(c, new Cell(value, recon)); + hasData = true; + } + } + + if (hasData) { + rowsWithData++; + + if (skip <= 0 || rowsWithData > skip) { + project.rows.add(newRow); + project.columnModel.setMaxCellIndex(newRow.cells.size()); + + if (limit > 0 && project.rows.size() >= limit) { + break; + } + } + } + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/importers/Importer.java b/src/main/java/com/metaweb/gridworks/importers/Importer.java index d769f171b..d3294762b 100644 --- a/src/main/java/com/metaweb/gridworks/importers/Importer.java +++ b/src/main/java/com/metaweb/gridworks/importers/Importer.java @@ -1,14 +1,14 @@ -package com.metaweb.gridworks.importers; - -import java.io.InputStream; -import java.io.Reader; -import java.util.Properties; - -import com.metaweb.gridworks.model.Project; - -public interface Importer { - public boolean takesReader(); - - public void read(Reader reader, Project project, Properties options) throws Exception; - public void read(InputStream inputStream, Project project, Properties options) throws Exception; -} +package com.metaweb.gridworks.importers; + +import java.io.InputStream; +import java.io.Reader; +import java.util.Properties; + +import com.metaweb.gridworks.model.Project; + +public interface Importer { + public boolean takesReader(); + + public void read(Reader reader, Project project, Properties options) throws Exception; + public void read(InputStream inputStream, Project project, Properties options) throws Exception; +} diff --git a/src/main/java/com/metaweb/gridworks/importers/ImporterUtilities.java b/src/main/java/com/metaweb/gridworks/importers/ImporterUtilities.java index 672b25a31..57f3d9bef 100644 --- a/src/main/java/com/metaweb/gridworks/importers/ImporterUtilities.java +++ b/src/main/java/com/metaweb/gridworks/importers/ImporterUtilities.java @@ -1,110 +1,110 @@ -package com.metaweb.gridworks.importers; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - -public class ImporterUtilities { - - static public Serializable parseCellValue(String text) { - if (text.length() > 0) { - if (text.length() > 1 && text.startsWith("\"") && text.endsWith("\"")) { - return text.substring(1, text.length() - 1); - } - - try { - return Long.parseLong(text); - } catch (NumberFormatException e) { - } - - try { - double d = Double.parseDouble(text); - if (!Double.isInfinite(d) && !Double.isNaN(d)) { - return d; - } - } catch (NumberFormatException e) { - } - } - return text; - } - - static public int getIntegerOption(String name, Properties options, int def) { - int value = def; - if (options.containsKey(name)) { - String s = options.getProperty(name); - try { - value = Integer.parseInt(s); - } catch (Exception e) { - } - } - return value; - } - - static public boolean getBooleanOption(String name, Properties options, boolean def) { - boolean value = def; - if (options.containsKey(name)) { - String s = options.getProperty(name); - try { - value = Boolean.parseBoolean(s); - } catch (Exception e) { - } - } - return value; - } - - static public void appendColumnName(List columnNames, int index, String name) { - name = name.trim(); - - while (columnNames.size() <= index) { - columnNames.add(""); - } - - if (!name.isEmpty()) { - String oldName = columnNames.get(index); - if (!oldName.isEmpty()) { - name = oldName + " " + name; - } - - columnNames.set(index, name); - } - } - - static public void ensureColumnsInRowExist(List columnNames, Row row) { - int count = row.cells.size(); - while (count > columnNames.size()) { - columnNames.add(""); - } - } - - static public void setupColumns(Project project, List columnNames) { - Map nameToIndex = new HashMap(); - for (int c = 0; c < columnNames.size(); c++) { - String cell = columnNames.get(c).trim(); - if (cell.isEmpty()) { - cell = "Column"; - } else if (cell.startsWith("\"") && cell.endsWith("\"")) { - cell = cell.substring(1, cell.length() - 1).trim(); - } - - if (nameToIndex.containsKey(cell)) { - int index = nameToIndex.get(cell); - nameToIndex.put(cell, index + 1); - - cell = cell.contains(" ") ? (cell + " " + index) : (cell + index); - } else { - nameToIndex.put(cell, 2); - } - - Column column = new Column(c, cell); - - project.columnModel.columns.add(column); - } - } - -} +package com.metaweb.gridworks.importers; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +public class ImporterUtilities { + + static public Serializable parseCellValue(String text) { + if (text.length() > 0) { + if (text.length() > 1 && text.startsWith("\"") && text.endsWith("\"")) { + return text.substring(1, text.length() - 1); + } + + try { + return Long.parseLong(text); + } catch (NumberFormatException e) { + } + + try { + double d = Double.parseDouble(text); + if (!Double.isInfinite(d) && !Double.isNaN(d)) { + return d; + } + } catch (NumberFormatException e) { + } + } + return text; + } + + static public int getIntegerOption(String name, Properties options, int def) { + int value = def; + if (options.containsKey(name)) { + String s = options.getProperty(name); + try { + value = Integer.parseInt(s); + } catch (Exception e) { + } + } + return value; + } + + static public boolean getBooleanOption(String name, Properties options, boolean def) { + boolean value = def; + if (options.containsKey(name)) { + String s = options.getProperty(name); + try { + value = Boolean.parseBoolean(s); + } catch (Exception e) { + } + } + return value; + } + + static public void appendColumnName(List columnNames, int index, String name) { + name = name.trim(); + + while (columnNames.size() <= index) { + columnNames.add(""); + } + + if (!name.isEmpty()) { + String oldName = columnNames.get(index); + if (!oldName.isEmpty()) { + name = oldName + " " + name; + } + + columnNames.set(index, name); + } + } + + static public void ensureColumnsInRowExist(List columnNames, Row row) { + int count = row.cells.size(); + while (count > columnNames.size()) { + columnNames.add(""); + } + } + + static public void setupColumns(Project project, List columnNames) { + Map nameToIndex = new HashMap(); + for (int c = 0; c < columnNames.size(); c++) { + String cell = columnNames.get(c).trim(); + if (cell.isEmpty()) { + cell = "Column"; + } else if (cell.startsWith("\"") && cell.endsWith("\"")) { + cell = cell.substring(1, cell.length() - 1).trim(); + } + + if (nameToIndex.containsKey(cell)) { + int index = nameToIndex.get(cell); + nameToIndex.put(cell, index + 1); + + cell = cell.contains(" ") ? (cell + " " + index) : (cell + index); + } else { + nameToIndex.put(cell, 2); + } + + Column column = new Column(c, cell); + + project.columnModel.columns.add(column); + } + } + +} diff --git a/src/main/java/com/metaweb/gridworks/importers/MarcImporter.java b/src/main/java/com/metaweb/gridworks/importers/MarcImporter.java index f2c2f0c49..a1b0b65b2 100644 --- a/src/main/java/com/metaweb/gridworks/importers/MarcImporter.java +++ b/src/main/java/com/metaweb/gridworks/importers/MarcImporter.java @@ -1,79 +1,79 @@ -package com.metaweb.gridworks.importers; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.Reader; -import java.util.Properties; - -import org.apache.commons.lang.NotImplementedException; -import org.marc4j.MarcPermissiveStreamReader; -import org.marc4j.MarcWriter; -import org.marc4j.MarcXmlWriter; -import org.marc4j.marc.Record; - -import com.metaweb.gridworks.model.Project; - -public class MarcImporter implements Importer { - - public boolean takesReader() { - return false; - } - - public void read(Reader reader, Project project, Properties options) - throws Exception { - - throw new NotImplementedException(); - } - - public void read( - InputStream inputStream, - Project project, - Properties options - ) throws Exception { - int limit = ImporterUtilities.getIntegerOption("limit",options,-1); - int skip = ImporterUtilities.getIntegerOption("skip",options,0); - - File tempFile = File.createTempFile("gridworks-import-", ".marc.xml"); - try { - OutputStream os = new FileOutputStream(tempFile); - try { - MarcPermissiveStreamReader reader = new MarcPermissiveStreamReader( - inputStream, - true, - true - ); - MarcWriter writer = new MarcXmlWriter(os, true); - - int count = 0; - while (reader.hasNext()) { - Record record = reader.next(); - if (skip <= 0) { - if (limit == -1 || count < limit) { - writer.write(record); - count++; - } else { - break; - } - } else { - skip--; - } - } - writer.close(); - } finally { - os.close(); - } - - InputStream is = new FileInputStream(tempFile); - try { - new XmlImporter().read(is, project, options); - } finally { - is.close(); - } - } finally { - tempFile.delete(); - } - } -} +package com.metaweb.gridworks.importers; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; +import java.util.Properties; + +import org.apache.commons.lang.NotImplementedException; +import org.marc4j.MarcPermissiveStreamReader; +import org.marc4j.MarcWriter; +import org.marc4j.MarcXmlWriter; +import org.marc4j.marc.Record; + +import com.metaweb.gridworks.model.Project; + +public class MarcImporter implements Importer { + + public boolean takesReader() { + return false; + } + + public void read(Reader reader, Project project, Properties options) + throws Exception { + + throw new NotImplementedException(); + } + + public void read( + InputStream inputStream, + Project project, + Properties options + ) throws Exception { + int limit = ImporterUtilities.getIntegerOption("limit",options,-1); + int skip = ImporterUtilities.getIntegerOption("skip",options,0); + + File tempFile = File.createTempFile("gridworks-import-", ".marc.xml"); + try { + OutputStream os = new FileOutputStream(tempFile); + try { + MarcPermissiveStreamReader reader = new MarcPermissiveStreamReader( + inputStream, + true, + true + ); + MarcWriter writer = new MarcXmlWriter(os, true); + + int count = 0; + while (reader.hasNext()) { + Record record = reader.next(); + if (skip <= 0) { + if (limit == -1 || count < limit) { + writer.write(record); + count++; + } else { + break; + } + } else { + skip--; + } + } + writer.close(); + } finally { + os.close(); + } + + InputStream is = new FileInputStream(tempFile); + try { + new XmlImporter().read(is, project, options); + } finally { + is.close(); + } + } finally { + tempFile.delete(); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/importers/TsvCsvImporter.java b/src/main/java/com/metaweb/gridworks/importers/TsvCsvImporter.java index 94a6e0943..8ef6b47eb 100644 --- a/src/main/java/com/metaweb/gridworks/importers/TsvCsvImporter.java +++ b/src/main/java/com/metaweb/gridworks/importers/TsvCsvImporter.java @@ -1,103 +1,103 @@ -package com.metaweb.gridworks.importers; - -import java.io.InputStream; -import java.io.LineNumberReader; -import java.io.Reader; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.apache.commons.lang.NotImplementedException; -import org.apache.commons.lang.StringUtils; - -import com.metaweb.gridworks.importers.parsers.CSVRowParser; -import com.metaweb.gridworks.importers.parsers.NonSplitRowParser; -import com.metaweb.gridworks.importers.parsers.RowParser; -import com.metaweb.gridworks.importers.parsers.SeparatorRowParser; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; - -public class TsvCsvImporter implements Importer { - public void read(Reader reader, Project project, Properties options) throws Exception { - boolean splitIntoColumns = ImporterUtilities.getBooleanOption("split-into-columns", options, true); - - String sep = options.getProperty("separator"); // auto-detect if not present - int ignoreLines = ImporterUtilities.getIntegerOption("ignore", options, -1); - int headerLines = ImporterUtilities.getIntegerOption("header-lines", options, 1); - - int limit = ImporterUtilities.getIntegerOption("limit",options,-1); - int skip = ImporterUtilities.getIntegerOption("skip",options,0); - boolean guessValueType = ImporterUtilities.getBooleanOption("guess-value-type", options, true); - - List columnNames = new ArrayList(); - - LineNumberReader lnReader = new LineNumberReader(reader); - RowParser parser = (sep != null && sep.length() > 0 && splitIntoColumns) ? - new SeparatorRowParser(sep) : null; - - String line = null; - int rowsWithData = 0; - - while ((line = lnReader.readLine()) != null) { - if (ignoreLines > 0) { - ignoreLines--; - continue; - } else if (StringUtils.isBlank(line)) { - continue; - } - - if (parser == null) { - if (splitIntoColumns) { - int tab = line.indexOf('\t'); - if (tab >= 0) { - sep = "\t"; - parser = new SeparatorRowParser(sep); - } else { - sep = ","; - parser = new CSVRowParser(); - } - } else { - parser = new NonSplitRowParser(); - } - } - - if (headerLines > 0) { - headerLines--; - - List cells = parser.split(line); - for (int c = 0; c < cells.size(); c++) { - String cell = cells.get(c).trim(); - - ImporterUtilities.appendColumnName(columnNames, c, cell); - } - } else { - Row row = new Row(columnNames.size()); - - if (parser.parseRow(row, line, guessValueType)) { - rowsWithData++; - - if (skip <= 0 || rowsWithData > skip) { - project.rows.add(row); - project.columnModel.setMaxCellIndex(row.cells.size()); - - ImporterUtilities.ensureColumnsInRowExist(columnNames, row); - - if (limit > 0 && project.rows.size() >= limit) { - break; - } - } - } - } - } - - ImporterUtilities.setupColumns(project, columnNames); - } - - public void read(InputStream inputStream, Project project, Properties options) throws Exception { - throw new NotImplementedException(); - } - - public boolean takesReader() { - return true; - } -} +package com.metaweb.gridworks.importers; + +import java.io.InputStream; +import java.io.LineNumberReader; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang.StringUtils; + +import com.metaweb.gridworks.importers.parsers.CSVRowParser; +import com.metaweb.gridworks.importers.parsers.NonSplitRowParser; +import com.metaweb.gridworks.importers.parsers.RowParser; +import com.metaweb.gridworks.importers.parsers.SeparatorRowParser; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; + +public class TsvCsvImporter implements Importer { + public void read(Reader reader, Project project, Properties options) throws Exception { + boolean splitIntoColumns = ImporterUtilities.getBooleanOption("split-into-columns", options, true); + + String sep = options.getProperty("separator"); // auto-detect if not present + int ignoreLines = ImporterUtilities.getIntegerOption("ignore", options, -1); + int headerLines = ImporterUtilities.getIntegerOption("header-lines", options, 1); + + int limit = ImporterUtilities.getIntegerOption("limit",options,-1); + int skip = ImporterUtilities.getIntegerOption("skip",options,0); + boolean guessValueType = ImporterUtilities.getBooleanOption("guess-value-type", options, true); + + List columnNames = new ArrayList(); + + LineNumberReader lnReader = new LineNumberReader(reader); + RowParser parser = (sep != null && sep.length() > 0 && splitIntoColumns) ? + new SeparatorRowParser(sep) : null; + + String line = null; + int rowsWithData = 0; + + while ((line = lnReader.readLine()) != null) { + if (ignoreLines > 0) { + ignoreLines--; + continue; + } else if (StringUtils.isBlank(line)) { + continue; + } + + if (parser == null) { + if (splitIntoColumns) { + int tab = line.indexOf('\t'); + if (tab >= 0) { + sep = "\t"; + parser = new SeparatorRowParser(sep); + } else { + sep = ","; + parser = new CSVRowParser(); + } + } else { + parser = new NonSplitRowParser(); + } + } + + if (headerLines > 0) { + headerLines--; + + List cells = parser.split(line); + for (int c = 0; c < cells.size(); c++) { + String cell = cells.get(c).trim(); + + ImporterUtilities.appendColumnName(columnNames, c, cell); + } + } else { + Row row = new Row(columnNames.size()); + + if (parser.parseRow(row, line, guessValueType)) { + rowsWithData++; + + if (skip <= 0 || rowsWithData > skip) { + project.rows.add(row); + project.columnModel.setMaxCellIndex(row.cells.size()); + + ImporterUtilities.ensureColumnsInRowExist(columnNames, row); + + if (limit > 0 && project.rows.size() >= limit) { + break; + } + } + } + } + } + + ImporterUtilities.setupColumns(project, columnNames); + } + + public void read(InputStream inputStream, Project project, Properties options) throws Exception { + throw new NotImplementedException(); + } + + public boolean takesReader() { + return true; + } +} diff --git a/src/main/java/com/metaweb/gridworks/importers/XmlImporter.java b/src/main/java/com/metaweb/gridworks/importers/XmlImporter.java index 658ee460d..af0a49404 100644 --- a/src/main/java/com/metaweb/gridworks/importers/XmlImporter.java +++ b/src/main/java/com/metaweb/gridworks/importers/XmlImporter.java @@ -1,64 +1,64 @@ -package com.metaweb.gridworks.importers; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.io.PushbackInputStream; -import java.io.Reader; -import java.util.Properties; - -import org.apache.commons.lang.NotImplementedException; - -import com.metaweb.gridworks.importers.XmlImportUtilities.ImportColumnGroup; -import com.metaweb.gridworks.model.Project; - -public class XmlImporter implements Importer { - - public static final int BUFFER_SIZE = 64 * 1024; - - public boolean takesReader() { - return false; - } - - public void read(Reader reader, Project project, Properties options) - throws Exception { - - throw new NotImplementedException(); - } - - public void read( - InputStream inputStream, - Project project, - Properties options - ) throws Exception { - PushbackInputStream pis = new PushbackInputStream(inputStream,BUFFER_SIZE); - - String[] recordPath = null; - { - byte[] buffer = new byte[BUFFER_SIZE]; - int bytes_read = 0; - while (bytes_read < BUFFER_SIZE) { - int c = pis.read(buffer, bytes_read, BUFFER_SIZE - bytes_read); - if (c == -1) break; - bytes_read +=c ; - } - pis.unread(buffer, 0, bytes_read); - - if (options.containsKey("importer-record-tag")) { - recordPath = XmlImportUtilities.detectPathFromTag( - new ByteArrayInputStream(buffer, 0, bytes_read), - options.getProperty("importer-record-tag")); - } else { - recordPath = XmlImportUtilities.detectRecordElement( - new ByteArrayInputStream(buffer, 0, bytes_read)); - } - } - - ImportColumnGroup rootColumnGroup = new ImportColumnGroup(); - - XmlImportUtilities.importXml(pis, project, recordPath, rootColumnGroup); - XmlImportUtilities.createColumnsFromImport(project, rootColumnGroup); - - project.columnModel.update(); - } - -} +package com.metaweb.gridworks.importers; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.PushbackInputStream; +import java.io.Reader; +import java.util.Properties; + +import org.apache.commons.lang.NotImplementedException; + +import com.metaweb.gridworks.importers.XmlImportUtilities.ImportColumnGroup; +import com.metaweb.gridworks.model.Project; + +public class XmlImporter implements Importer { + + public static final int BUFFER_SIZE = 64 * 1024; + + public boolean takesReader() { + return false; + } + + public void read(Reader reader, Project project, Properties options) + throws Exception { + + throw new NotImplementedException(); + } + + public void read( + InputStream inputStream, + Project project, + Properties options + ) throws Exception { + PushbackInputStream pis = new PushbackInputStream(inputStream,BUFFER_SIZE); + + String[] recordPath = null; + { + byte[] buffer = new byte[BUFFER_SIZE]; + int bytes_read = 0; + while (bytes_read < BUFFER_SIZE) { + int c = pis.read(buffer, bytes_read, BUFFER_SIZE - bytes_read); + if (c == -1) break; + bytes_read +=c ; + } + pis.unread(buffer, 0, bytes_read); + + if (options.containsKey("importer-record-tag")) { + recordPath = XmlImportUtilities.detectPathFromTag( + new ByteArrayInputStream(buffer, 0, bytes_read), + options.getProperty("importer-record-tag")); + } else { + recordPath = XmlImportUtilities.detectRecordElement( + new ByteArrayInputStream(buffer, 0, bytes_read)); + } + } + + ImportColumnGroup rootColumnGroup = new ImportColumnGroup(); + + XmlImportUtilities.importXml(pis, project, recordPath, rootColumnGroup); + XmlImportUtilities.createColumnsFromImport(project, rootColumnGroup); + + project.columnModel.update(); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/model/AbstractOperation.java b/src/main/java/com/metaweb/gridworks/model/AbstractOperation.java index a4cca1bb3..486f7831f 100644 --- a/src/main/java/com/metaweb/gridworks/model/AbstractOperation.java +++ b/src/main/java/com/metaweb/gridworks/model/AbstractOperation.java @@ -1,33 +1,33 @@ -package com.metaweb.gridworks.model; - -import java.util.Properties; - -import org.apache.commons.lang.NotImplementedException; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.process.Process; -import com.metaweb.gridworks.process.QuickHistoryEntryProcess; - -/* - * An abstract operation can be applied to different but similar - * projects. - */ -abstract public class AbstractOperation implements Jsonizable { - public Process createProcess(Project project, Properties options) throws Exception { - return new QuickHistoryEntryProcess(project, getBriefDescription(null)) { - @Override - protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception { - return AbstractOperation.this.createHistoryEntry(_project, historyEntryID); - } - }; - } - - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - throw new NotImplementedException(); - } - - protected String getBriefDescription(Project project) { - throw new NotImplementedException(); - } -} +package com.metaweb.gridworks.model; + +import java.util.Properties; + +import org.apache.commons.lang.NotImplementedException; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.process.Process; +import com.metaweb.gridworks.process.QuickHistoryEntryProcess; + +/* + * An abstract operation can be applied to different but similar + * projects. + */ +abstract public class AbstractOperation implements Jsonizable { + public Process createProcess(Project project, Properties options) throws Exception { + return new QuickHistoryEntryProcess(project, getBriefDescription(null)) { + @Override + protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception { + return AbstractOperation.this.createHistoryEntry(_project, historyEntryID); + } + }; + } + + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + throw new NotImplementedException(); + } + + protected String getBriefDescription(Project project) { + throw new NotImplementedException(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/Cell.java b/src/main/java/com/metaweb/gridworks/model/Cell.java index 5c321fc21..14140cc93 100644 --- a/src/main/java/com/metaweb/gridworks/model/Cell.java +++ b/src/main/java/com/metaweb/gridworks/model/Cell.java @@ -1,151 +1,151 @@ -package com.metaweb.gridworks.model; - -import java.io.Serializable; -import java.io.Writer; -import java.util.Calendar; -import java.util.Date; -import java.util.Properties; - -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonToken; -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.expr.EvalError; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.expr.HasFields; -import com.metaweb.gridworks.util.ParsingUtilities; -import com.metaweb.gridworks.util.Pool; - -public class Cell implements HasFields, Jsonizable { - final public Serializable value; - final public Recon recon; - - public Cell(Serializable value, Recon recon) { - this.value = value; - this.recon = recon; - } - - public Object getField(String name, Properties bindings) { - if ("value".equals(name)) { - return value; - } else if ("recon".equals(name)) { - return recon; - } - return null; - } - - public boolean fieldAlsoHasFields(String name) { - return "recon".equals(name); - } - - public void write(JSONWriter writer, Properties options) throws JSONException { - writer.object(); - if (ExpressionUtils.isError(value)) { - writer.key("e"); - writer.value(((EvalError) value).message); - } else { - writer.key("v"); - if (value != null) { - if (value instanceof Calendar) { - writer.value(ParsingUtilities.dateToString(((Calendar) value).getTime())); - writer.key("t"); writer.value("date"); - } else if (value instanceof Date) { - writer.value(ParsingUtilities.dateToString((Date) value)); - writer.key("t"); writer.value("date"); - } else { - writer.value(value); - } - } else { - writer.value(null); - } - } - - if (recon != null) { - writer.key("r"); - writer.value(Long.toString(recon.id)); - - Pool pool = (Pool) options.get("pool"); - pool.pool(recon); - } - writer.endObject(); - } - - public void save(Writer writer, Properties options) { - JSONWriter jsonWriter = new JSONWriter(writer); - try { - write(jsonWriter, options); - } catch (JSONException e) { - e.printStackTrace(); - } - } - - static public Cell loadStreaming(String s, Pool pool) throws Exception { - JsonFactory jsonFactory = new JsonFactory(); - JsonParser jp = jsonFactory.createJsonParser(s); - - if (jp.nextToken() != JsonToken.START_OBJECT) { - return null; - } - - return loadStreaming(jp, pool); - } - - static public Cell loadStreaming(JsonParser jp, Pool pool) throws Exception { - JsonToken t = jp.getCurrentToken(); - if (t == JsonToken.VALUE_NULL || t != JsonToken.START_OBJECT) { - return null; - } - - Serializable value = null; - String type = null; - Recon recon = null; - - while (jp.nextToken() != JsonToken.END_OBJECT) { - String fieldName = jp.getCurrentName(); - jp.nextToken(); - - if ("r".equals(fieldName)) { - if (jp.getCurrentToken() == JsonToken.VALUE_STRING) { - String reconID = jp.getText(); - - recon = pool.getRecon(reconID); - } else { - // legacy - recon = Recon.loadStreaming(jp, pool); - } - } else if ("e".equals(fieldName)) { - value = new EvalError(jp.getText()); - } else if ("v".equals(fieldName)) { - JsonToken token = jp.getCurrentToken(); - - if (token == JsonToken.VALUE_STRING) { - value = jp.getText(); - } else if (token == JsonToken.VALUE_NUMBER_INT) { - value = jp.getLongValue(); - } else if (token == JsonToken.VALUE_NUMBER_FLOAT) { - value = jp.getDoubleValue(); - } else if (token == JsonToken.VALUE_TRUE) { - value = true; - } else if (token == JsonToken.VALUE_FALSE) { - value = false; - } - } else if ("t".equals(fieldName)) { - type = jp.getText(); - } - } - - if (value != null) { - if (type != null) { - if ("date".equals(type)) { - value = ParsingUtilities.stringToDate((String) value); - } - } - return new Cell(value, recon); - } else { - return null; - } - } -} +package com.metaweb.gridworks.model; + +import java.io.Serializable; +import java.io.Writer; +import java.util.Calendar; +import java.util.Date; +import java.util.Properties; + +import org.codehaus.jackson.JsonFactory; +import org.codehaus.jackson.JsonParser; +import org.codehaus.jackson.JsonToken; +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.expr.EvalError; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.expr.HasFields; +import com.metaweb.gridworks.util.ParsingUtilities; +import com.metaweb.gridworks.util.Pool; + +public class Cell implements HasFields, Jsonizable { + final public Serializable value; + final public Recon recon; + + public Cell(Serializable value, Recon recon) { + this.value = value; + this.recon = recon; + } + + public Object getField(String name, Properties bindings) { + if ("value".equals(name)) { + return value; + } else if ("recon".equals(name)) { + return recon; + } + return null; + } + + public boolean fieldAlsoHasFields(String name) { + return "recon".equals(name); + } + + public void write(JSONWriter writer, Properties options) throws JSONException { + writer.object(); + if (ExpressionUtils.isError(value)) { + writer.key("e"); + writer.value(((EvalError) value).message); + } else { + writer.key("v"); + if (value != null) { + if (value instanceof Calendar) { + writer.value(ParsingUtilities.dateToString(((Calendar) value).getTime())); + writer.key("t"); writer.value("date"); + } else if (value instanceof Date) { + writer.value(ParsingUtilities.dateToString((Date) value)); + writer.key("t"); writer.value("date"); + } else { + writer.value(value); + } + } else { + writer.value(null); + } + } + + if (recon != null) { + writer.key("r"); + writer.value(Long.toString(recon.id)); + + Pool pool = (Pool) options.get("pool"); + pool.pool(recon); + } + writer.endObject(); + } + + public void save(Writer writer, Properties options) { + JSONWriter jsonWriter = new JSONWriter(writer); + try { + write(jsonWriter, options); + } catch (JSONException e) { + e.printStackTrace(); + } + } + + static public Cell loadStreaming(String s, Pool pool) throws Exception { + JsonFactory jsonFactory = new JsonFactory(); + JsonParser jp = jsonFactory.createJsonParser(s); + + if (jp.nextToken() != JsonToken.START_OBJECT) { + return null; + } + + return loadStreaming(jp, pool); + } + + static public Cell loadStreaming(JsonParser jp, Pool pool) throws Exception { + JsonToken t = jp.getCurrentToken(); + if (t == JsonToken.VALUE_NULL || t != JsonToken.START_OBJECT) { + return null; + } + + Serializable value = null; + String type = null; + Recon recon = null; + + while (jp.nextToken() != JsonToken.END_OBJECT) { + String fieldName = jp.getCurrentName(); + jp.nextToken(); + + if ("r".equals(fieldName)) { + if (jp.getCurrentToken() == JsonToken.VALUE_STRING) { + String reconID = jp.getText(); + + recon = pool.getRecon(reconID); + } else { + // legacy + recon = Recon.loadStreaming(jp, pool); + } + } else if ("e".equals(fieldName)) { + value = new EvalError(jp.getText()); + } else if ("v".equals(fieldName)) { + JsonToken token = jp.getCurrentToken(); + + if (token == JsonToken.VALUE_STRING) { + value = jp.getText(); + } else if (token == JsonToken.VALUE_NUMBER_INT) { + value = jp.getLongValue(); + } else if (token == JsonToken.VALUE_NUMBER_FLOAT) { + value = jp.getDoubleValue(); + } else if (token == JsonToken.VALUE_TRUE) { + value = true; + } else if (token == JsonToken.VALUE_FALSE) { + value = false; + } + } else if ("t".equals(fieldName)) { + type = jp.getText(); + } + } + + if (value != null) { + if (type != null) { + if ("date".equals(type)) { + value = ParsingUtilities.stringToDate((String) value); + } + } + return new Cell(value, recon); + } else { + return null; + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/Column.java b/src/main/java/com/metaweb/gridworks/model/Column.java index 61a4b9139..90a6dd81d 100644 --- a/src/main/java/com/metaweb/gridworks/model/Column.java +++ b/src/main/java/com/metaweb/gridworks/model/Column.java @@ -1,123 +1,123 @@ -package com.metaweb.gridworks.model; - -import java.io.Writer; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.model.recon.ReconConfig; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class Column implements Jsonizable { - final private int _cellIndex; - final private String _originalName; - private String _name; - private ReconConfig _reconConfig; - private ReconStats _reconStats; - - transient protected Map _precomputes; - - public Column(int cellIndex, String originalName) { - _cellIndex = cellIndex; - _originalName = _name = originalName; - } - - public int getCellIndex() { - return _cellIndex; - } - - public String getOriginalHeaderLabel() { - return _originalName; - } - - public void setName(String name) { - this._name = name; - } - - public String getName() { - return _name; - } - - public void setReconConfig(ReconConfig config) { - this._reconConfig = config; - } - - public ReconConfig getReconConfig() { - return _reconConfig; - } - - public void setReconStats(ReconStats stats) { - this._reconStats = stats; - } - - public ReconStats getReconStats() { - return _reconStats; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("cellIndex"); writer.value(_cellIndex); - writer.key("originalName"); writer.value(_originalName); - writer.key("name"); writer.value(_name); - if (_reconConfig != null) { - writer.key("reconConfig"); - _reconConfig.write(writer, options); - } - if (_reconStats != null) { - writer.key("reconStats"); - _reconStats.write(writer, options); - } - writer.endObject(); - } - - public void clearPrecomputes() { - if (_precomputes != null) { - _precomputes.clear(); - } - } - - public Object getPrecompute(String key) { - if (_precomputes != null) { - return _precomputes.get(key); - } - return null; - } - - public void setPrecompute(String key, Object value) { - if (_precomputes == null) { - _precomputes = new HashMap(); - } - _precomputes.put(key, value); - } - - public void save(Writer writer) { - JSONWriter jsonWriter = new JSONWriter(writer); - try { - write(jsonWriter, new Properties()); - } catch (JSONException e) { - e.printStackTrace(); - } - } - - static public Column load(String s) throws Exception { - JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(s); - Column column = new Column(obj.getInt("cellIndex"), obj.getString("originalName")); - - column._name = obj.getString("name"); - if (obj.has("reconConfig")) { - column._reconConfig = ReconConfig.reconstruct(obj.getJSONObject("reconConfig")); - } - if (obj.has("reconStats")) { - column._reconStats = ReconStats.load(obj.getJSONObject("reconStats")); - } - - return column; - } -} +package com.metaweb.gridworks.model; + +import java.io.Writer; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.model.recon.ReconConfig; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class Column implements Jsonizable { + final private int _cellIndex; + final private String _originalName; + private String _name; + private ReconConfig _reconConfig; + private ReconStats _reconStats; + + transient protected Map _precomputes; + + public Column(int cellIndex, String originalName) { + _cellIndex = cellIndex; + _originalName = _name = originalName; + } + + public int getCellIndex() { + return _cellIndex; + } + + public String getOriginalHeaderLabel() { + return _originalName; + } + + public void setName(String name) { + this._name = name; + } + + public String getName() { + return _name; + } + + public void setReconConfig(ReconConfig config) { + this._reconConfig = config; + } + + public ReconConfig getReconConfig() { + return _reconConfig; + } + + public void setReconStats(ReconStats stats) { + this._reconStats = stats; + } + + public ReconStats getReconStats() { + return _reconStats; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("cellIndex"); writer.value(_cellIndex); + writer.key("originalName"); writer.value(_originalName); + writer.key("name"); writer.value(_name); + if (_reconConfig != null) { + writer.key("reconConfig"); + _reconConfig.write(writer, options); + } + if (_reconStats != null) { + writer.key("reconStats"); + _reconStats.write(writer, options); + } + writer.endObject(); + } + + public void clearPrecomputes() { + if (_precomputes != null) { + _precomputes.clear(); + } + } + + public Object getPrecompute(String key) { + if (_precomputes != null) { + return _precomputes.get(key); + } + return null; + } + + public void setPrecompute(String key, Object value) { + if (_precomputes == null) { + _precomputes = new HashMap(); + } + _precomputes.put(key, value); + } + + public void save(Writer writer) { + JSONWriter jsonWriter = new JSONWriter(writer); + try { + write(jsonWriter, new Properties()); + } catch (JSONException e) { + e.printStackTrace(); + } + } + + static public Column load(String s) throws Exception { + JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(s); + Column column = new Column(obj.getInt("cellIndex"), obj.getString("originalName")); + + column._name = obj.getString("name"); + if (obj.has("reconConfig")) { + column._reconConfig = ReconConfig.reconstruct(obj.getJSONObject("reconConfig")); + } + if (obj.has("reconStats")) { + column._reconStats = ReconStats.load(obj.getJSONObject("reconStats")); + } + + return column; + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/ColumnGroup.java b/src/main/java/com/metaweb/gridworks/model/ColumnGroup.java index 7b1a17c68..9ee775a1f 100644 --- a/src/main/java/com/metaweb/gridworks/model/ColumnGroup.java +++ b/src/main/java/com/metaweb/gridworks/model/ColumnGroup.java @@ -1,76 +1,76 @@ -package com.metaweb.gridworks.model; - -import java.io.Writer; -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class ColumnGroup implements Jsonizable { - final public int startColumnIndex; - final public int columnSpan; - final public int keyColumnIndex; // could be -1 if there is no key cell - - transient public ColumnGroup parentGroup; - transient public List subgroups; - - public ColumnGroup(int startColumnIndex, int columnSpan, int keyColumnIndex) { - this.startColumnIndex = startColumnIndex; - this.columnSpan = columnSpan; - this.keyColumnIndex = keyColumnIndex; - internalInitialize(); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - - writer.key("startColumnIndex"); writer.value(startColumnIndex); - writer.key("columnSpan"); writer.value(columnSpan); - writer.key("keyColumnIndex"); writer.value(keyColumnIndex); - - if (!"save".equals(options.get("mode")) && (subgroups != null) && (subgroups.size() > 0)) { - writer.key("subgroups"); writer.array(); - for (ColumnGroup g : subgroups) { - g.write(writer, options); - } - writer.endArray(); - } - writer.endObject(); - } - - public boolean contains(ColumnGroup g) { - return (g.startColumnIndex >= startColumnIndex && - g.startColumnIndex < startColumnIndex + columnSpan); - } - - public void save(Writer writer) { - JSONWriter jsonWriter = new JSONWriter(writer); - try { - write(jsonWriter, new Properties()); - } catch (JSONException e) { - e.printStackTrace(); - } - } - - static public ColumnGroup load(String s) throws Exception { - JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(s); - - return new ColumnGroup( - obj.getInt("startColumnIndex"), - obj.getInt("columnSpan"), - obj.getInt("keyColumnIndex") - ); - } - - protected void internalInitialize() { - subgroups = new LinkedList(); - } -} +package com.metaweb.gridworks.model; + +import java.io.Writer; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class ColumnGroup implements Jsonizable { + final public int startColumnIndex; + final public int columnSpan; + final public int keyColumnIndex; // could be -1 if there is no key cell + + transient public ColumnGroup parentGroup; + transient public List subgroups; + + public ColumnGroup(int startColumnIndex, int columnSpan, int keyColumnIndex) { + this.startColumnIndex = startColumnIndex; + this.columnSpan = columnSpan; + this.keyColumnIndex = keyColumnIndex; + internalInitialize(); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + + writer.key("startColumnIndex"); writer.value(startColumnIndex); + writer.key("columnSpan"); writer.value(columnSpan); + writer.key("keyColumnIndex"); writer.value(keyColumnIndex); + + if (!"save".equals(options.get("mode")) && (subgroups != null) && (subgroups.size() > 0)) { + writer.key("subgroups"); writer.array(); + for (ColumnGroup g : subgroups) { + g.write(writer, options); + } + writer.endArray(); + } + writer.endObject(); + } + + public boolean contains(ColumnGroup g) { + return (g.startColumnIndex >= startColumnIndex && + g.startColumnIndex < startColumnIndex + columnSpan); + } + + public void save(Writer writer) { + JSONWriter jsonWriter = new JSONWriter(writer); + try { + write(jsonWriter, new Properties()); + } catch (JSONException e) { + e.printStackTrace(); + } + } + + static public ColumnGroup load(String s) throws Exception { + JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(s); + + return new ColumnGroup( + obj.getInt("startColumnIndex"), + obj.getInt("columnSpan"), + obj.getInt("keyColumnIndex") + ); + } + + protected void internalInitialize() { + subgroups = new LinkedList(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/ColumnModel.java b/src/main/java/com/metaweb/gridworks/model/ColumnModel.java index d4a0697c9..0ffb5acdc 100644 --- a/src/main/java/com/metaweb/gridworks/model/ColumnModel.java +++ b/src/main/java/com/metaweb/gridworks/model/ColumnModel.java @@ -1,222 +1,222 @@ -package com.metaweb.gridworks.model; - -import java.io.IOException; -import java.io.LineNumberReader; -import java.io.Writer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; - -public class ColumnModel implements Jsonizable { - final public List columns = new LinkedList(); - final public List columnGroups = new LinkedList(); - - private int _maxCellIndex; - private int _keyColumnIndex; - - transient protected Map _nameToColumn; - transient protected Map _cellIndexToColumn; - transient protected List _rootColumnGroups; - transient protected List _columnNames; - transient boolean _hasDependentRows; - - public ColumnModel() { - internalInitialize(); - } - - public void setMaxCellIndex(int maxCellIndex) { - this._maxCellIndex = Math.max(this._maxCellIndex, maxCellIndex); - } - - public int getMaxCellIndex() { - return _maxCellIndex; - } - - public int allocateNewCellIndex() { - return ++_maxCellIndex; - } - - public void setKeyColumnIndex(int keyColumnIndex) { - // TODO: check validity of new cell index, e.g., it's not in any group - this._keyColumnIndex = keyColumnIndex; - } - - public int getKeyColumnIndex() { - return _keyColumnIndex; - } - - public void addColumnGroup(int startColumnIndex, int span, int keyColumnIndex) { - for (ColumnGroup g : columnGroups) { - if (g.startColumnIndex == startColumnIndex && g.columnSpan == span) { - if (g.keyColumnIndex == keyColumnIndex) { - return; - } else { - columnGroups.remove(g); - break; - } - } - } - - ColumnGroup cg = new ColumnGroup(startColumnIndex, span, keyColumnIndex); - - columnGroups.add(cg); - - } - - public void update() { - internalInitialize(); - } - - public Column getColumnByName(String name) { - return _nameToColumn.get(name); - } - - public int getColumnIndexByName(String name) { - for (int i = 0; i < _columnNames.size(); i++) { - String s = _columnNames.get(i); - if (name.equals(s)) { - return i; - } - } - return -1; - } - - public Column getColumnByCellIndex(int cellIndex) { - return _cellIndexToColumn.get(cellIndex); - } - - public List getColumnNames() { - return _columnNames; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - - writer.key("hasDependentRows"); writer.value(_hasDependentRows); - - writer.key("columns"); - writer.array(); - for (Column column : columns) { - column.write(writer, options); - } - writer.endArray(); - - if (columns.size() > 0) { - writer.key("keyCellIndex"); writer.value(getKeyColumnIndex()); - writer.key("keyColumnName"); writer.value(columns.get(_keyColumnIndex).getName()); - } - - writer.key("columnGroups"); - writer.array(); - for (ColumnGroup g : _rootColumnGroups) { - g.write(writer, options); - } - writer.endArray(); - - writer.endObject(); - } - - public void save(Writer writer, Properties options) throws IOException { - writer.write("maxCellIndex="); writer.write(Integer.toString(_maxCellIndex)); writer.write('\n'); - writer.write("keyColumnIndex="); writer.write(Integer.toString(_keyColumnIndex)); writer.write('\n'); - - writer.write("columnCount="); writer.write(Integer.toString(columns.size())); writer.write('\n'); - for (Column column : columns) { - column.save(writer); writer.write('\n'); - } - - writer.write("columnGroupCount="); writer.write(Integer.toString(columnGroups.size())); writer.write('\n'); - for (ColumnGroup group : columnGroups) { - group.save(writer); writer.write('\n'); - } - - writer.write("/e/\n"); - } - - public void load(LineNumberReader reader) throws Exception { - String line; - while ((line = reader.readLine()) != null && !"/e/".equals(line)) { - int equal = line.indexOf('='); - CharSequence field = line.subSequence(0, equal); - String value = line.substring(equal + 1); - - if ("maxCellIndex".equals(field)) { - _maxCellIndex = Integer.parseInt(value); - } else if ("keyColumnIndex".equals(field)) { - _keyColumnIndex = Integer.parseInt(value); - } else if ("columnCount".equals(field)) { - int count = Integer.parseInt(value); - - for (int i = 0; i < count; i++) { - columns.add(Column.load(reader.readLine())); - } - } else if ("columnGroupCount".equals(field)) { - int count = Integer.parseInt(value); - - for (int i = 0; i < count; i++) { - columnGroups.add(ColumnGroup.load(reader.readLine())); - } - } - } - - internalInitialize(); - } - - protected void internalInitialize() { - generateMaps(); - - // Turn the flat list of column groups into a tree - - _rootColumnGroups = new LinkedList(columnGroups); - Collections.sort(_rootColumnGroups, new Comparator() { - public int compare(ColumnGroup o1, ColumnGroup o2) { - int firstDiff = o1.startColumnIndex - o2.startColumnIndex; - return firstDiff != 0 ? - firstDiff : // whichever group that starts first goes first - (o2.columnSpan - o1.columnSpan); // otherwise, the larger group goes first - } - }); - - for (int i = _rootColumnGroups.size() - 1; i >= 0; i--) { - ColumnGroup g = _rootColumnGroups.get(i); - - for (int j = i + 1; j < _rootColumnGroups.size(); j++) { - ColumnGroup g2 = _rootColumnGroups.get(j); - if (g2.parentGroup == null && g.contains(g2)) { - g2.parentGroup = g; - g.subgroups.add(g2); - } - } - } - - for (int i = _rootColumnGroups.size() - 1; i >= 0; i--) { - if (_rootColumnGroups.get(i).parentGroup != null) { - _rootColumnGroups.remove(i); - } - } - } - - protected void generateMaps() { - _nameToColumn = new HashMap(); - _cellIndexToColumn = new HashMap(); - _columnNames = new ArrayList(); - - for (Column column : columns) { - _nameToColumn.put(column.getName(), column); - _cellIndexToColumn.put(column.getCellIndex(), column); - _columnNames.add(column.getName()); - } - } -} +package com.metaweb.gridworks.model; + +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; + +public class ColumnModel implements Jsonizable { + final public List columns = new LinkedList(); + final public List columnGroups = new LinkedList(); + + private int _maxCellIndex; + private int _keyColumnIndex; + + transient protected Map _nameToColumn; + transient protected Map _cellIndexToColumn; + transient protected List _rootColumnGroups; + transient protected List _columnNames; + transient boolean _hasDependentRows; + + public ColumnModel() { + internalInitialize(); + } + + public void setMaxCellIndex(int maxCellIndex) { + this._maxCellIndex = Math.max(this._maxCellIndex, maxCellIndex); + } + + public int getMaxCellIndex() { + return _maxCellIndex; + } + + public int allocateNewCellIndex() { + return ++_maxCellIndex; + } + + public void setKeyColumnIndex(int keyColumnIndex) { + // TODO: check validity of new cell index, e.g., it's not in any group + this._keyColumnIndex = keyColumnIndex; + } + + public int getKeyColumnIndex() { + return _keyColumnIndex; + } + + public void addColumnGroup(int startColumnIndex, int span, int keyColumnIndex) { + for (ColumnGroup g : columnGroups) { + if (g.startColumnIndex == startColumnIndex && g.columnSpan == span) { + if (g.keyColumnIndex == keyColumnIndex) { + return; + } else { + columnGroups.remove(g); + break; + } + } + } + + ColumnGroup cg = new ColumnGroup(startColumnIndex, span, keyColumnIndex); + + columnGroups.add(cg); + + } + + public void update() { + internalInitialize(); + } + + public Column getColumnByName(String name) { + return _nameToColumn.get(name); + } + + public int getColumnIndexByName(String name) { + for (int i = 0; i < _columnNames.size(); i++) { + String s = _columnNames.get(i); + if (name.equals(s)) { + return i; + } + } + return -1; + } + + public Column getColumnByCellIndex(int cellIndex) { + return _cellIndexToColumn.get(cellIndex); + } + + public List getColumnNames() { + return _columnNames; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + + writer.key("hasDependentRows"); writer.value(_hasDependentRows); + + writer.key("columns"); + writer.array(); + for (Column column : columns) { + column.write(writer, options); + } + writer.endArray(); + + if (columns.size() > 0) { + writer.key("keyCellIndex"); writer.value(getKeyColumnIndex()); + writer.key("keyColumnName"); writer.value(columns.get(_keyColumnIndex).getName()); + } + + writer.key("columnGroups"); + writer.array(); + for (ColumnGroup g : _rootColumnGroups) { + g.write(writer, options); + } + writer.endArray(); + + writer.endObject(); + } + + public void save(Writer writer, Properties options) throws IOException { + writer.write("maxCellIndex="); writer.write(Integer.toString(_maxCellIndex)); writer.write('\n'); + writer.write("keyColumnIndex="); writer.write(Integer.toString(_keyColumnIndex)); writer.write('\n'); + + writer.write("columnCount="); writer.write(Integer.toString(columns.size())); writer.write('\n'); + for (Column column : columns) { + column.save(writer); writer.write('\n'); + } + + writer.write("columnGroupCount="); writer.write(Integer.toString(columnGroups.size())); writer.write('\n'); + for (ColumnGroup group : columnGroups) { + group.save(writer); writer.write('\n'); + } + + writer.write("/e/\n"); + } + + public void load(LineNumberReader reader) throws Exception { + String line; + while ((line = reader.readLine()) != null && !"/e/".equals(line)) { + int equal = line.indexOf('='); + CharSequence field = line.subSequence(0, equal); + String value = line.substring(equal + 1); + + if ("maxCellIndex".equals(field)) { + _maxCellIndex = Integer.parseInt(value); + } else if ("keyColumnIndex".equals(field)) { + _keyColumnIndex = Integer.parseInt(value); + } else if ("columnCount".equals(field)) { + int count = Integer.parseInt(value); + + for (int i = 0; i < count; i++) { + columns.add(Column.load(reader.readLine())); + } + } else if ("columnGroupCount".equals(field)) { + int count = Integer.parseInt(value); + + for (int i = 0; i < count; i++) { + columnGroups.add(ColumnGroup.load(reader.readLine())); + } + } + } + + internalInitialize(); + } + + protected void internalInitialize() { + generateMaps(); + + // Turn the flat list of column groups into a tree + + _rootColumnGroups = new LinkedList(columnGroups); + Collections.sort(_rootColumnGroups, new Comparator() { + public int compare(ColumnGroup o1, ColumnGroup o2) { + int firstDiff = o1.startColumnIndex - o2.startColumnIndex; + return firstDiff != 0 ? + firstDiff : // whichever group that starts first goes first + (o2.columnSpan - o1.columnSpan); // otherwise, the larger group goes first + } + }); + + for (int i = _rootColumnGroups.size() - 1; i >= 0; i--) { + ColumnGroup g = _rootColumnGroups.get(i); + + for (int j = i + 1; j < _rootColumnGroups.size(); j++) { + ColumnGroup g2 = _rootColumnGroups.get(j); + if (g2.parentGroup == null && g.contains(g2)) { + g2.parentGroup = g; + g.subgroups.add(g2); + } + } + } + + for (int i = _rootColumnGroups.size() - 1; i >= 0; i--) { + if (_rootColumnGroups.get(i).parentGroup != null) { + _rootColumnGroups.remove(i); + } + } + } + + protected void generateMaps() { + _nameToColumn = new HashMap(); + _cellIndexToColumn = new HashMap(); + _columnNames = new ArrayList(); + + for (Column column : columns) { + _nameToColumn.put(column.getName(), column); + _cellIndexToColumn.put(column.getCellIndex(), column); + _columnNames.add(column.getName()); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/Project.java b/src/main/java/com/metaweb/gridworks/model/Project.java index f20f9eff1..86d5e97bb 100644 --- a/src/main/java/com/metaweb/gridworks/model/Project.java +++ b/src/main/java/com/metaweb/gridworks/model/Project.java @@ -1,378 +1,378 @@ -package com.metaweb.gridworks.model; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.LineNumberReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.List; -import java.util.Properties; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.metaweb.gridworks.Gridworks; -import com.metaweb.gridworks.ProjectManager; -import com.metaweb.gridworks.ProjectMetadata; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.history.History; -import com.metaweb.gridworks.process.ProcessManager; -import com.metaweb.gridworks.protograph.Protograph; -import com.metaweb.gridworks.util.Pool; - -public class Project { - final public long id; - - final public ColumnModel columnModel = new ColumnModel(); - final public List rows = new ArrayList(); - final public History history; - - public Protograph protograph; - - transient public ProcessManager processManager = new ProcessManager(); - transient public Date lastSave = new Date(); - - final static Logger logger = LoggerFactory.getLogger("project"); - - static public long generateID() { - return System.currentTimeMillis() + Math.round(Math.random() * 1000000000000L); - } - - public Project() { - id = generateID(); - history = new History(this); - } - - protected Project(long id) { - this.id = id; - this.history = new History(this); - } - - public ProjectMetadata getMetadata() { - return ProjectManager.singleton.getProjectMetadata(id); - } - - public void save() { - synchronized (this) { - File dir = ProjectManager.singleton.getProjectDir(id); - - File tempFile = new File(dir, "data.temp.zip"); - try { - saveToFile(tempFile); - } catch (Exception e) { - e.printStackTrace(); - - logger.warn("Failed to save project {}", id); - return; - } - - File file = new File(dir, "data.zip"); - File oldFile = new File(dir, "data.old.zip"); - - if (file.exists()) { - file.renameTo(oldFile); - } - - tempFile.renameTo(file); - if (oldFile.exists()) { - oldFile.delete(); - } - - lastSave = new Date(); - - logger.info("Saved project '{}'",id); - } - } - - protected void saveToFile(File file) throws Exception { - ZipOutputStream out = new ZipOutputStream(new FileOutputStream(file)); - try { - Pool pool = new Pool(); - - out.putNextEntry(new ZipEntry("data.txt")); - try { - saveToOutputStream(out, pool); - } finally { - out.closeEntry(); - } - - out.putNextEntry(new ZipEntry("pool.txt")); - try { - pool.save(out); - } finally { - out.closeEntry(); - } - } finally { - out.close(); - } - } - - protected void saveToOutputStream(OutputStream out, Pool pool) throws IOException { - Writer writer = new OutputStreamWriter(out); - try { - Properties options = new Properties(); - options.setProperty("mode", "save"); - options.put("pool", pool); - - saveToWriter(writer, options); - } finally { - writer.flush(); - } - } - - protected void saveToWriter(Writer writer, Properties options) throws IOException { - writer.write(Gridworks.getVersion()); writer.write('\n'); - - writer.write("columnModel=\n"); columnModel.save(writer, options); - writer.write("history=\n"); history.save(writer, options); - if (protograph != null) { - writer.write("protograph="); protograph.save(writer, options); writer.write('\n'); - } - - writer.write("rowCount="); writer.write(Integer.toString(rows.size())); writer.write('\n'); - for (Row row : rows) { - row.save(writer, options); writer.write('\n'); - } - } - - static public Project load(File dir, long id) { - try { - File file = new File(dir, "data.zip"); - if (file.exists()) { - return loadFromFile(file, id); - } - } catch (Exception e) { - e.printStackTrace(); - } - - try { - File file = new File(dir, "data.temp.zip"); - if (file.exists()) { - return loadFromFile(file, id); - } - } catch (Exception e) { - e.printStackTrace(); - } - - try { - File file = new File(dir, "data.old.zip"); - if (file.exists()) { - return loadFromFile(file, id); - } - } catch (Exception e) { - e.printStackTrace(); - } - - return null; - } - - static protected Project loadFromFile( - File file, - long id - ) throws Exception { - ZipFile zipFile = new ZipFile(file); - try { - Pool pool = new Pool(); - ZipEntry poolEntry = zipFile.getEntry("pool.txt"); - if (poolEntry != null) { - pool.load(new InputStreamReader( - zipFile.getInputStream(poolEntry))); - } // else, it's a legacy project file - - return loadFromReader( - new LineNumberReader( - new InputStreamReader( - zipFile.getInputStream( - zipFile.getEntry("data.txt")))), - id, - pool - ); - } finally { - zipFile.close(); - } - } - - static protected Project loadFromReader( - LineNumberReader reader, - long id, - Pool pool - ) throws Exception { - long start = System.currentTimeMillis(); - - /* String version = */ reader.readLine(); - - Project project = new Project(id); - int maxCellCount = 0; - - String line; - while ((line = reader.readLine()) != null) { - int equal = line.indexOf('='); - CharSequence field = line.subSequence(0, equal); - String value = line.substring(equal + 1); - - if ("columnModel".equals(field)) { - project.columnModel.load(reader); - } else if ("history".equals(field)) { - project.history.load(project, reader); - } else if ("protograph".equals(field)) { - project.protograph = Protograph.load(project, value); - } else if ("rowCount".equals(field)) { - int count = Integer.parseInt(value); - - for (int i = 0; i < count; i++) { - line = reader.readLine(); - if (line != null) { - Row row = Row.load(line, pool); - project.rows.add(row); - maxCellCount = Math.max(maxCellCount, row.cells.size()); - } - } - } - } - - project.columnModel.setMaxCellIndex(maxCellCount - 1); - - logger.info( - "Loaded project {} from disk in {} sec(s)",id,Long.toString((System.currentTimeMillis() - start) / 1000) - ); - - project.recomputeRowContextDependencies(); - - return project; - } - - - static protected class Group { - int[] cellIndices; - int keyCellIndex; - } - - public void recomputeRowContextDependencies() { - List keyedGroups = new ArrayList(); - - addRootKeyedGroup(keyedGroups); - - for (ColumnGroup group : columnModel.columnGroups) { - if (group.keyColumnIndex >= 0) { - Group keyedGroup = new Group(); - keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex(); - keyedGroup.cellIndices = new int[group.columnSpan - 1]; - - int c = 0; - for (int i = 0; i < group.columnSpan; i++) { - int columnIndex = group.startColumnIndex + i; - if (columnIndex != group.keyColumnIndex) { - int cellIndex = columnModel.columns.get(columnIndex).getCellIndex(); - keyedGroup.cellIndices[c++] = cellIndex; - } - } - - keyedGroups.add(keyedGroup); - } - } - - Collections.sort(keyedGroups, new Comparator() { - public int compare(Group o1, Group o2) { - return o2.cellIndices.length - o1.cellIndices.length; // larger groups first - } - }); - - int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()]; - for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) { - lastNonBlankRowsByGroup[i] = -1; - } - - int rowCount = rows.size(); - int groupCount = keyedGroups.size(); - - int recordIndex = 0; - for (int r = 0; r < rowCount; r++) { - Row row = rows.get(r); - row.contextRows = null; - row.contextRowSlots = null; - row.contextCellSlots = null; - - for (int g = 0; g < groupCount; g++) { - Group group = keyedGroups.get(g); - - if (!ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) { - int contextRowIndex = lastNonBlankRowsByGroup[g]; - if (contextRowIndex >= 0) { - for (int dependentCellIndex : group.cellIndices) { - if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) { - setRowDependency( - row, - dependentCellIndex, - contextRowIndex, - group.keyCellIndex - ); - } - } - } - } else { - lastNonBlankRowsByGroup[g] = r; - } - } - - if (row.contextRowSlots != null && row.contextRowSlots.length > 0) { - row.recordIndex = -1; - row.contextRows = new ArrayList(); - for (int index : row.contextRowSlots) { - if (index >= 0) { - row.contextRows.add(index); - } - } - Collections.sort(row.contextRows); - - columnModel._hasDependentRows = true; - } else { - row.recordIndex = recordIndex++; - } - } - } - - protected void addRootKeyedGroup(List keyedGroups) { - int count = columnModel.getMaxCellIndex() + 1; - if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) { - Group rootKeyedGroup = new Group(); - - rootKeyedGroup.cellIndices = new int[count - 1]; - rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex(); - - for (int i = 0; i < count; i++) { - if (i < rootKeyedGroup.keyCellIndex) { - rootKeyedGroup.cellIndices[i] = i; - } else if (i > rootKeyedGroup.keyCellIndex) { - rootKeyedGroup.cellIndices[i - 1] = i; - } - } - keyedGroups.add(rootKeyedGroup); - } - } - - protected void setRowDependency(Row row, int cellIndex, int contextRowIndex, int contextCellIndex) { - int count = columnModel.getMaxCellIndex() + 1; - if (row.contextRowSlots == null || row.contextCellSlots == null) { - row.contextRowSlots = new int[count]; - row.contextCellSlots = new int[count]; - - for (int i = 0; i < count; i++) { - row.contextRowSlots[i] = -1; - row.contextCellSlots[i] = -1; - } - } - - row.contextRowSlots[cellIndex] = contextRowIndex; - row.contextCellSlots[cellIndex] = contextCellIndex; - } -} +package com.metaweb.gridworks.model; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.Properties; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.metaweb.gridworks.Gridworks; +import com.metaweb.gridworks.ProjectManager; +import com.metaweb.gridworks.ProjectMetadata; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.history.History; +import com.metaweb.gridworks.process.ProcessManager; +import com.metaweb.gridworks.protograph.Protograph; +import com.metaweb.gridworks.util.Pool; + +public class Project { + final public long id; + + final public ColumnModel columnModel = new ColumnModel(); + final public List rows = new ArrayList(); + final public History history; + + public Protograph protograph; + + transient public ProcessManager processManager = new ProcessManager(); + transient public Date lastSave = new Date(); + + final static Logger logger = LoggerFactory.getLogger("project"); + + static public long generateID() { + return System.currentTimeMillis() + Math.round(Math.random() * 1000000000000L); + } + + public Project() { + id = generateID(); + history = new History(this); + } + + protected Project(long id) { + this.id = id; + this.history = new History(this); + } + + public ProjectMetadata getMetadata() { + return ProjectManager.singleton.getProjectMetadata(id); + } + + public void save() { + synchronized (this) { + File dir = ProjectManager.singleton.getProjectDir(id); + + File tempFile = new File(dir, "data.temp.zip"); + try { + saveToFile(tempFile); + } catch (Exception e) { + e.printStackTrace(); + + logger.warn("Failed to save project {}", id); + return; + } + + File file = new File(dir, "data.zip"); + File oldFile = new File(dir, "data.old.zip"); + + if (file.exists()) { + file.renameTo(oldFile); + } + + tempFile.renameTo(file); + if (oldFile.exists()) { + oldFile.delete(); + } + + lastSave = new Date(); + + logger.info("Saved project '{}'",id); + } + } + + protected void saveToFile(File file) throws Exception { + ZipOutputStream out = new ZipOutputStream(new FileOutputStream(file)); + try { + Pool pool = new Pool(); + + out.putNextEntry(new ZipEntry("data.txt")); + try { + saveToOutputStream(out, pool); + } finally { + out.closeEntry(); + } + + out.putNextEntry(new ZipEntry("pool.txt")); + try { + pool.save(out); + } finally { + out.closeEntry(); + } + } finally { + out.close(); + } + } + + protected void saveToOutputStream(OutputStream out, Pool pool) throws IOException { + Writer writer = new OutputStreamWriter(out); + try { + Properties options = new Properties(); + options.setProperty("mode", "save"); + options.put("pool", pool); + + saveToWriter(writer, options); + } finally { + writer.flush(); + } + } + + protected void saveToWriter(Writer writer, Properties options) throws IOException { + writer.write(Gridworks.getVersion()); writer.write('\n'); + + writer.write("columnModel=\n"); columnModel.save(writer, options); + writer.write("history=\n"); history.save(writer, options); + if (protograph != null) { + writer.write("protograph="); protograph.save(writer, options); writer.write('\n'); + } + + writer.write("rowCount="); writer.write(Integer.toString(rows.size())); writer.write('\n'); + for (Row row : rows) { + row.save(writer, options); writer.write('\n'); + } + } + + static public Project load(File dir, long id) { + try { + File file = new File(dir, "data.zip"); + if (file.exists()) { + return loadFromFile(file, id); + } + } catch (Exception e) { + e.printStackTrace(); + } + + try { + File file = new File(dir, "data.temp.zip"); + if (file.exists()) { + return loadFromFile(file, id); + } + } catch (Exception e) { + e.printStackTrace(); + } + + try { + File file = new File(dir, "data.old.zip"); + if (file.exists()) { + return loadFromFile(file, id); + } + } catch (Exception e) { + e.printStackTrace(); + } + + return null; + } + + static protected Project loadFromFile( + File file, + long id + ) throws Exception { + ZipFile zipFile = new ZipFile(file); + try { + Pool pool = new Pool(); + ZipEntry poolEntry = zipFile.getEntry("pool.txt"); + if (poolEntry != null) { + pool.load(new InputStreamReader( + zipFile.getInputStream(poolEntry))); + } // else, it's a legacy project file + + return loadFromReader( + new LineNumberReader( + new InputStreamReader( + zipFile.getInputStream( + zipFile.getEntry("data.txt")))), + id, + pool + ); + } finally { + zipFile.close(); + } + } + + static protected Project loadFromReader( + LineNumberReader reader, + long id, + Pool pool + ) throws Exception { + long start = System.currentTimeMillis(); + + /* String version = */ reader.readLine(); + + Project project = new Project(id); + int maxCellCount = 0; + + String line; + while ((line = reader.readLine()) != null) { + int equal = line.indexOf('='); + CharSequence field = line.subSequence(0, equal); + String value = line.substring(equal + 1); + + if ("columnModel".equals(field)) { + project.columnModel.load(reader); + } else if ("history".equals(field)) { + project.history.load(project, reader); + } else if ("protograph".equals(field)) { + project.protograph = Protograph.load(project, value); + } else if ("rowCount".equals(field)) { + int count = Integer.parseInt(value); + + for (int i = 0; i < count; i++) { + line = reader.readLine(); + if (line != null) { + Row row = Row.load(line, pool); + project.rows.add(row); + maxCellCount = Math.max(maxCellCount, row.cells.size()); + } + } + } + } + + project.columnModel.setMaxCellIndex(maxCellCount - 1); + + logger.info( + "Loaded project {} from disk in {} sec(s)",id,Long.toString((System.currentTimeMillis() - start) / 1000) + ); + + project.recomputeRowContextDependencies(); + + return project; + } + + + static protected class Group { + int[] cellIndices; + int keyCellIndex; + } + + public void recomputeRowContextDependencies() { + List keyedGroups = new ArrayList(); + + addRootKeyedGroup(keyedGroups); + + for (ColumnGroup group : columnModel.columnGroups) { + if (group.keyColumnIndex >= 0) { + Group keyedGroup = new Group(); + keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex(); + keyedGroup.cellIndices = new int[group.columnSpan - 1]; + + int c = 0; + for (int i = 0; i < group.columnSpan; i++) { + int columnIndex = group.startColumnIndex + i; + if (columnIndex != group.keyColumnIndex) { + int cellIndex = columnModel.columns.get(columnIndex).getCellIndex(); + keyedGroup.cellIndices[c++] = cellIndex; + } + } + + keyedGroups.add(keyedGroup); + } + } + + Collections.sort(keyedGroups, new Comparator() { + public int compare(Group o1, Group o2) { + return o2.cellIndices.length - o1.cellIndices.length; // larger groups first + } + }); + + int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()]; + for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) { + lastNonBlankRowsByGroup[i] = -1; + } + + int rowCount = rows.size(); + int groupCount = keyedGroups.size(); + + int recordIndex = 0; + for (int r = 0; r < rowCount; r++) { + Row row = rows.get(r); + row.contextRows = null; + row.contextRowSlots = null; + row.contextCellSlots = null; + + for (int g = 0; g < groupCount; g++) { + Group group = keyedGroups.get(g); + + if (!ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) { + int contextRowIndex = lastNonBlankRowsByGroup[g]; + if (contextRowIndex >= 0) { + for (int dependentCellIndex : group.cellIndices) { + if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) { + setRowDependency( + row, + dependentCellIndex, + contextRowIndex, + group.keyCellIndex + ); + } + } + } + } else { + lastNonBlankRowsByGroup[g] = r; + } + } + + if (row.contextRowSlots != null && row.contextRowSlots.length > 0) { + row.recordIndex = -1; + row.contextRows = new ArrayList(); + for (int index : row.contextRowSlots) { + if (index >= 0) { + row.contextRows.add(index); + } + } + Collections.sort(row.contextRows); + + columnModel._hasDependentRows = true; + } else { + row.recordIndex = recordIndex++; + } + } + } + + protected void addRootKeyedGroup(List keyedGroups) { + int count = columnModel.getMaxCellIndex() + 1; + if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) { + Group rootKeyedGroup = new Group(); + + rootKeyedGroup.cellIndices = new int[count - 1]; + rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex(); + + for (int i = 0; i < count; i++) { + if (i < rootKeyedGroup.keyCellIndex) { + rootKeyedGroup.cellIndices[i] = i; + } else if (i > rootKeyedGroup.keyCellIndex) { + rootKeyedGroup.cellIndices[i - 1] = i; + } + } + keyedGroups.add(rootKeyedGroup); + } + } + + protected void setRowDependency(Row row, int cellIndex, int contextRowIndex, int contextCellIndex) { + int count = columnModel.getMaxCellIndex() + 1; + if (row.contextRowSlots == null || row.contextCellSlots == null) { + row.contextRowSlots = new int[count]; + row.contextCellSlots = new int[count]; + + for (int i = 0; i < count; i++) { + row.contextRowSlots[i] = -1; + row.contextCellSlots[i] = -1; + } + } + + row.contextRowSlots[cellIndex] = contextRowIndex; + row.contextCellSlots[cellIndex] = contextCellIndex; + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/Recon.java b/src/main/java/com/metaweb/gridworks/model/Recon.java index 6e9dd7a8a..c565e33fa 100644 --- a/src/main/java/com/metaweb/gridworks/model/Recon.java +++ b/src/main/java/com/metaweb/gridworks/model/Recon.java @@ -1,330 +1,330 @@ -package com.metaweb.gridworks.model; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonToken; -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.expr.HasFields; -import com.metaweb.gridworks.util.Pool; - -public class Recon implements HasFields, Jsonizable { - - static public enum Judgment { - None, - Matched, - New - } - - static public String judgmentToString(Judgment judgment) { - if (judgment == Judgment.Matched) { - return "matched"; - } else if (judgment == Judgment.New) { - return "new"; - } else { - return "none"; - } - } - - static public Judgment stringToJudgment(String s) { - if ("matched".equals(s)) { - return Judgment.Matched; - } else if ("new".equals(s)) { - return Judgment.New; - } else { - return Judgment.None; - } - } - - static final public int Feature_typeMatch = 0; - static final public int Feature_nameMatch = 1; - static final public int Feature_nameLevenshtein = 2; - static final public int Feature_nameWordDistance = 3; - static final public int Feature_max = 4; - - static final protected Map s_featureMap = new HashMap(); - static { - s_featureMap.put("typeMatch", Feature_typeMatch); - s_featureMap.put("nameMatch", Feature_nameMatch); - s_featureMap.put("nameLevenshtein", Feature_nameLevenshtein); - s_featureMap.put("nameWordDistance", Feature_nameWordDistance); - } - - final public long id; - public Object[] features = new Object[Feature_max]; - public String service = "unknown"; - public List candidates; - - public Judgment judgment = Judgment.None; - public String judgmentAction = "unknown"; - public long judgmentHistoryEntry; - public int judgmentBatchSize = 0; - - public ReconCandidate match = null; - public int matchRank = -1; - - public Recon(long judgmentHistoryEntry) { - id = System.currentTimeMillis() * 1000000 + Math.round(Math.random() * 1000000); - this.judgmentHistoryEntry = judgmentHistoryEntry; - } - - protected Recon(long id, long judgmentHistoryEntry) { - this.id = id; - this.judgmentHistoryEntry = judgmentHistoryEntry; - } - - public Recon dup(long judgmentHistoryEntry) { - Recon r = new Recon(judgmentHistoryEntry); - - System.arraycopy(features, 0, r.features, 0, features.length); - - if (candidates != null) { - r.candidates = new ArrayList(candidates); - } - - r.service = service; - - r.judgment = judgment; - - r.judgmentAction = judgmentAction; - r.judgmentBatchSize = judgmentBatchSize; - - r.match = match; - r.matchRank = matchRank; - - return r; - } - - public void addCandidate(ReconCandidate candidate) { - if (candidates == null) { - candidates = new ArrayList(3); - } - candidates.add(candidate); - } - - public ReconCandidate getBestCandidate() { - if (candidates != null && candidates.size() > 0) { - return candidates.get(0); - } - return null; - } - - public Object getFeature(int feature) { - return feature < features.length ? features[feature] : null; - } - - public void setFeature(int feature, Object v) { - if (feature >= features.length) { - if (feature >= Feature_max) { - return; - } - - // We deserialized this object from an older version of the class - // that had fewer features, so we can just try to extend it - - Object[] newFeatures = new Object[Feature_max]; - - System.arraycopy(features, 0, newFeatures, 0, features.length); - - features = newFeatures; - } - - features[feature] = v; - } - - public Object getField(String name, Properties bindings) { - if ("best".equals(name)) { - return candidates != null && candidates.size() > 0 ? candidates.get(0) : null; - } else if ("candidates".equals(name)) { - return candidates; - } else if ("judgment".equals(name) || "judgement".equals(name)) { - return judgmentToString(); - } else if ("judgmentAction".equals(name) || "judgementAction".equals(name)) { - return judgmentAction; - } else if ("judgmentHistoryEntry".equals(name) || "judgementHistoryEntry".equals(name)) { - return judgmentHistoryEntry; - } else if ("judgmentBatchSize".equals(name) || "judgementBatchSize".equals(name)) { - return judgmentBatchSize; - } else if ("matched".equals(name)) { - return judgment == Judgment.Matched; - } else if ("new".equals(name)) { - return judgment == Judgment.New; - } else if ("match".equals(name)) { - return match; - } else if ("matchRank".equals(name)) { - return matchRank; - } else if ("features".equals(name)) { - return new Features(); - } - return null; - } - - public boolean fieldAlsoHasFields(String name) { - return "match".equals(name) || "best".equals(name); - } - - protected String judgmentToString() { - return judgmentToString(judgment); - } - - public class Features implements HasFields { - public Object getField(String name, Properties bindings) { - int index = s_featureMap.get(name); - return index < features.length ? features[index] : null; - } - - public boolean fieldAlsoHasFields(String name) { - return false; - } - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - boolean saveMode = "save".equals(options.getProperty("mode")); - - writer.object(); - writer.key("id"); writer.value(id); - if (saveMode) { - writer.key("judgmentHistoryEntry"); writer.value(judgmentHistoryEntry); - } - - writer.key("j"); writer.value(judgmentToString()); - if (match != null) { - writer.key("m"); - writer.value(match.topicID); - } - if (match == null || saveMode) { - writer.key("c"); writer.array(); - if (candidates != null) { - for (ReconCandidate c : candidates) { - writer.value(c.topicID); - } - } - writer.endArray(); - } - - if (saveMode) { - writer.key("f"); - writer.array(); - for (Object o : features) { - writer.value(o); - } - writer.endArray(); - - writer.key("service"); writer.value(service); - writer.key("judgmentAction"); writer.value(judgmentAction); - writer.key("judgmentBatchSize"); writer.value(judgmentBatchSize); - - if (match != null) { - writer.key("matchRank"); writer.value(matchRank); - } - } - - writer.endObject(); - } - - static public Recon loadStreaming(String s, Pool pool) throws Exception { - JsonFactory jsonFactory = new JsonFactory(); - JsonParser jp = jsonFactory.createJsonParser(s); - - if (jp.nextToken() != JsonToken.START_OBJECT) { - return null; - } - return loadStreaming(jp, pool); - } - - static public Recon loadStreaming(JsonParser jp, Pool pool) throws Exception { - JsonToken t = jp.getCurrentToken(); - if (t == JsonToken.VALUE_NULL || t != JsonToken.START_OBJECT) { - return null; - } - - Recon recon = null; - long id = -1; - long judgmentHistoryEntry = -1; - - while (jp.nextToken() != JsonToken.END_OBJECT) { - String fieldName = jp.getCurrentName(); - jp.nextToken(); - - if ("id".equals(fieldName)) { - id = jp.getLongValue(); - } else if ("judgmentHistoryEntry".equals(fieldName)) { - judgmentHistoryEntry = jp.getLongValue(); - } else { - if (recon == null) { - recon = new Recon(id, judgmentHistoryEntry); - } - - if ("j".equals(fieldName)) { - recon.judgment = stringToJudgment(jp.getText()); - } else if ("m".equals(fieldName)) { - if (jp.getCurrentToken() == JsonToken.VALUE_STRING) { - String candidateID = jp.getText(); - - recon.match = pool.getReconCandidate(candidateID); - } else { - // legacy - recon.match = ReconCandidate.loadStreaming(jp); - } - } else if ("f".equals(fieldName)) { - if (jp.getCurrentToken() != JsonToken.START_ARRAY) { - return null; - } - - int feature = 0; - while (jp.nextToken() != JsonToken.END_ARRAY) { - if (feature < recon.features.length) { - JsonToken token = jp.getCurrentToken(); - if (token == JsonToken.VALUE_STRING) { - recon.features[feature++] = jp.getText(); - } else if (token == JsonToken.VALUE_NUMBER_INT) { - recon.features[feature++] = jp.getLongValue(); - } else if (token == JsonToken.VALUE_NUMBER_FLOAT) { - recon.features[feature++] = jp.getDoubleValue(); - } else if (token == JsonToken.VALUE_FALSE) { - recon.features[feature++] = false; - } else if (token == JsonToken.VALUE_TRUE) { - recon.features[feature++] = true; - } - } - } - } else if ("c".equals(fieldName)) { - if (jp.getCurrentToken() != JsonToken.START_ARRAY) { - return null; - } - - while (jp.nextToken() != JsonToken.END_ARRAY) { - if (jp.getCurrentToken() == JsonToken.VALUE_STRING) { - String candidateID = jp.getText(); - - recon.addCandidate(pool.getReconCandidate(candidateID)); - } else { - // legacy - recon.addCandidate(ReconCandidate.loadStreaming(jp)); - } - } - } else if ("service".equals(fieldName)) { - recon.service = jp.getText(); - } else if ("judgmentAction".equals(fieldName)) { - recon.judgmentAction = jp.getText(); - } else if ("judgmentBatchSize".equals(fieldName)) { - recon.judgmentBatchSize = jp.getIntValue(); - } else if ("matchRank".equals(fieldName)) { - recon.matchRank = jp.getIntValue(); - } - } - } - - return recon; - } -} +package com.metaweb.gridworks.model; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.codehaus.jackson.JsonFactory; +import org.codehaus.jackson.JsonParser; +import org.codehaus.jackson.JsonToken; +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.expr.HasFields; +import com.metaweb.gridworks.util.Pool; + +public class Recon implements HasFields, Jsonizable { + + static public enum Judgment { + None, + Matched, + New + } + + static public String judgmentToString(Judgment judgment) { + if (judgment == Judgment.Matched) { + return "matched"; + } else if (judgment == Judgment.New) { + return "new"; + } else { + return "none"; + } + } + + static public Judgment stringToJudgment(String s) { + if ("matched".equals(s)) { + return Judgment.Matched; + } else if ("new".equals(s)) { + return Judgment.New; + } else { + return Judgment.None; + } + } + + static final public int Feature_typeMatch = 0; + static final public int Feature_nameMatch = 1; + static final public int Feature_nameLevenshtein = 2; + static final public int Feature_nameWordDistance = 3; + static final public int Feature_max = 4; + + static final protected Map s_featureMap = new HashMap(); + static { + s_featureMap.put("typeMatch", Feature_typeMatch); + s_featureMap.put("nameMatch", Feature_nameMatch); + s_featureMap.put("nameLevenshtein", Feature_nameLevenshtein); + s_featureMap.put("nameWordDistance", Feature_nameWordDistance); + } + + final public long id; + public Object[] features = new Object[Feature_max]; + public String service = "unknown"; + public List candidates; + + public Judgment judgment = Judgment.None; + public String judgmentAction = "unknown"; + public long judgmentHistoryEntry; + public int judgmentBatchSize = 0; + + public ReconCandidate match = null; + public int matchRank = -1; + + public Recon(long judgmentHistoryEntry) { + id = System.currentTimeMillis() * 1000000 + Math.round(Math.random() * 1000000); + this.judgmentHistoryEntry = judgmentHistoryEntry; + } + + protected Recon(long id, long judgmentHistoryEntry) { + this.id = id; + this.judgmentHistoryEntry = judgmentHistoryEntry; + } + + public Recon dup(long judgmentHistoryEntry) { + Recon r = new Recon(judgmentHistoryEntry); + + System.arraycopy(features, 0, r.features, 0, features.length); + + if (candidates != null) { + r.candidates = new ArrayList(candidates); + } + + r.service = service; + + r.judgment = judgment; + + r.judgmentAction = judgmentAction; + r.judgmentBatchSize = judgmentBatchSize; + + r.match = match; + r.matchRank = matchRank; + + return r; + } + + public void addCandidate(ReconCandidate candidate) { + if (candidates == null) { + candidates = new ArrayList(3); + } + candidates.add(candidate); + } + + public ReconCandidate getBestCandidate() { + if (candidates != null && candidates.size() > 0) { + return candidates.get(0); + } + return null; + } + + public Object getFeature(int feature) { + return feature < features.length ? features[feature] : null; + } + + public void setFeature(int feature, Object v) { + if (feature >= features.length) { + if (feature >= Feature_max) { + return; + } + + // We deserialized this object from an older version of the class + // that had fewer features, so we can just try to extend it + + Object[] newFeatures = new Object[Feature_max]; + + System.arraycopy(features, 0, newFeatures, 0, features.length); + + features = newFeatures; + } + + features[feature] = v; + } + + public Object getField(String name, Properties bindings) { + if ("best".equals(name)) { + return candidates != null && candidates.size() > 0 ? candidates.get(0) : null; + } else if ("candidates".equals(name)) { + return candidates; + } else if ("judgment".equals(name) || "judgement".equals(name)) { + return judgmentToString(); + } else if ("judgmentAction".equals(name) || "judgementAction".equals(name)) { + return judgmentAction; + } else if ("judgmentHistoryEntry".equals(name) || "judgementHistoryEntry".equals(name)) { + return judgmentHistoryEntry; + } else if ("judgmentBatchSize".equals(name) || "judgementBatchSize".equals(name)) { + return judgmentBatchSize; + } else if ("matched".equals(name)) { + return judgment == Judgment.Matched; + } else if ("new".equals(name)) { + return judgment == Judgment.New; + } else if ("match".equals(name)) { + return match; + } else if ("matchRank".equals(name)) { + return matchRank; + } else if ("features".equals(name)) { + return new Features(); + } + return null; + } + + public boolean fieldAlsoHasFields(String name) { + return "match".equals(name) || "best".equals(name); + } + + protected String judgmentToString() { + return judgmentToString(judgment); + } + + public class Features implements HasFields { + public Object getField(String name, Properties bindings) { + int index = s_featureMap.get(name); + return index < features.length ? features[index] : null; + } + + public boolean fieldAlsoHasFields(String name) { + return false; + } + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + boolean saveMode = "save".equals(options.getProperty("mode")); + + writer.object(); + writer.key("id"); writer.value(id); + if (saveMode) { + writer.key("judgmentHistoryEntry"); writer.value(judgmentHistoryEntry); + } + + writer.key("j"); writer.value(judgmentToString()); + if (match != null) { + writer.key("m"); + writer.value(match.topicID); + } + if (match == null || saveMode) { + writer.key("c"); writer.array(); + if (candidates != null) { + for (ReconCandidate c : candidates) { + writer.value(c.topicID); + } + } + writer.endArray(); + } + + if (saveMode) { + writer.key("f"); + writer.array(); + for (Object o : features) { + writer.value(o); + } + writer.endArray(); + + writer.key("service"); writer.value(service); + writer.key("judgmentAction"); writer.value(judgmentAction); + writer.key("judgmentBatchSize"); writer.value(judgmentBatchSize); + + if (match != null) { + writer.key("matchRank"); writer.value(matchRank); + } + } + + writer.endObject(); + } + + static public Recon loadStreaming(String s, Pool pool) throws Exception { + JsonFactory jsonFactory = new JsonFactory(); + JsonParser jp = jsonFactory.createJsonParser(s); + + if (jp.nextToken() != JsonToken.START_OBJECT) { + return null; + } + return loadStreaming(jp, pool); + } + + static public Recon loadStreaming(JsonParser jp, Pool pool) throws Exception { + JsonToken t = jp.getCurrentToken(); + if (t == JsonToken.VALUE_NULL || t != JsonToken.START_OBJECT) { + return null; + } + + Recon recon = null; + long id = -1; + long judgmentHistoryEntry = -1; + + while (jp.nextToken() != JsonToken.END_OBJECT) { + String fieldName = jp.getCurrentName(); + jp.nextToken(); + + if ("id".equals(fieldName)) { + id = jp.getLongValue(); + } else if ("judgmentHistoryEntry".equals(fieldName)) { + judgmentHistoryEntry = jp.getLongValue(); + } else { + if (recon == null) { + recon = new Recon(id, judgmentHistoryEntry); + } + + if ("j".equals(fieldName)) { + recon.judgment = stringToJudgment(jp.getText()); + } else if ("m".equals(fieldName)) { + if (jp.getCurrentToken() == JsonToken.VALUE_STRING) { + String candidateID = jp.getText(); + + recon.match = pool.getReconCandidate(candidateID); + } else { + // legacy + recon.match = ReconCandidate.loadStreaming(jp); + } + } else if ("f".equals(fieldName)) { + if (jp.getCurrentToken() != JsonToken.START_ARRAY) { + return null; + } + + int feature = 0; + while (jp.nextToken() != JsonToken.END_ARRAY) { + if (feature < recon.features.length) { + JsonToken token = jp.getCurrentToken(); + if (token == JsonToken.VALUE_STRING) { + recon.features[feature++] = jp.getText(); + } else if (token == JsonToken.VALUE_NUMBER_INT) { + recon.features[feature++] = jp.getLongValue(); + } else if (token == JsonToken.VALUE_NUMBER_FLOAT) { + recon.features[feature++] = jp.getDoubleValue(); + } else if (token == JsonToken.VALUE_FALSE) { + recon.features[feature++] = false; + } else if (token == JsonToken.VALUE_TRUE) { + recon.features[feature++] = true; + } + } + } + } else if ("c".equals(fieldName)) { + if (jp.getCurrentToken() != JsonToken.START_ARRAY) { + return null; + } + + while (jp.nextToken() != JsonToken.END_ARRAY) { + if (jp.getCurrentToken() == JsonToken.VALUE_STRING) { + String candidateID = jp.getText(); + + recon.addCandidate(pool.getReconCandidate(candidateID)); + } else { + // legacy + recon.addCandidate(ReconCandidate.loadStreaming(jp)); + } + } + } else if ("service".equals(fieldName)) { + recon.service = jp.getText(); + } else if ("judgmentAction".equals(fieldName)) { + recon.judgmentAction = jp.getText(); + } else if ("judgmentBatchSize".equals(fieldName)) { + recon.judgmentBatchSize = jp.getIntValue(); + } else if ("matchRank".equals(fieldName)) { + recon.matchRank = jp.getIntValue(); + } + } + } + + return recon; + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/ReconCandidate.java b/src/main/java/com/metaweb/gridworks/model/ReconCandidate.java index 83051dcda..256fd9838 100644 --- a/src/main/java/com/metaweb/gridworks/model/ReconCandidate.java +++ b/src/main/java/com/metaweb/gridworks/model/ReconCandidate.java @@ -1,133 +1,133 @@ -package com.metaweb.gridworks.model; - -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonToken; -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.expr.HasFields; - -public class ReconCandidate implements HasFields, Jsonizable { - final public String topicID; - final public String topicGUID; - final public String topicName; - final public String[] typeIDs; - final public double score; - - public ReconCandidate(String topicID, String topicGUID, String topicName, String[] typeIDs, double score) { - this.topicID = topicID; - this.topicGUID = topicGUID; - this.topicName = topicName; - this.typeIDs = typeIDs; - this.score = score; - } - - public Object getField(String name, Properties bindings) { - if ("id".equals(name)) { - return topicID; - } else if ("guid".equals(name)) { - return topicGUID; - } else if ("name".equals(name)) { - return topicName; - } else if ("type".equals(name)) { - return typeIDs; - } else if ("score".equals(name)) { - return score; - } - return null; - } - - public boolean fieldAlsoHasFields(String name) { - return false; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("id"); writer.value(topicID); - writer.key("guid"); writer.value(topicGUID); - writer.key("name"); writer.value(topicName); - writer.key("score"); writer.value(score); - - /* if (!options.containsKey("reconCandidateOmitTypes")) */ { - writer.key("types"); writer.array(); - for (String typeID : typeIDs) { - writer.value(typeID); - } - writer.endArray(); - } - - writer.endObject(); - } - - static public ReconCandidate loadStreaming(String s) throws Exception { - JsonFactory jsonFactory = new JsonFactory(); - JsonParser jp = jsonFactory.createJsonParser(s); - - if (jp.nextToken() != JsonToken.START_OBJECT) { - return null; - } - return loadStreaming(jp); - } - - static public ReconCandidate loadStreaming(JsonParser jp) throws Exception { - JsonToken t = jp.getCurrentToken(); - if (t == JsonToken.VALUE_NULL || t != JsonToken.START_OBJECT) { - return null; - } - - String id = null; - String guid = null; - String name = null; - List types = null; - double score = 0; - - while (jp.nextToken() != JsonToken.END_OBJECT) { - String fieldName = jp.getCurrentName(); - jp.nextToken(); - - if ("id".equals(fieldName)) { - id = jp.getText(); - } else if ("guid".equals(fieldName)) { - guid = jp.getText(); - } else if ("name".equals(fieldName)) { - name = jp.getText(); - } else if ("score".equals(fieldName)) { - score = jp.getDoubleValue(); - } else if ("types".equals(fieldName)) { - if (jp.getCurrentToken() != JsonToken.START_ARRAY) { - return null; - } - - types = new ArrayList(); - - while (jp.nextToken() != JsonToken.END_ARRAY) { - types.add(jp.getText()); - } - } - } - - String[] typesA; - if (types != null) { - typesA = new String[types.size()]; - types.toArray(typesA); - } else { - typesA = new String[0]; - } - - return new ReconCandidate( - id, - guid, - name, - typesA, - score - ); - } +package com.metaweb.gridworks.model; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.codehaus.jackson.JsonFactory; +import org.codehaus.jackson.JsonParser; +import org.codehaus.jackson.JsonToken; +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.expr.HasFields; + +public class ReconCandidate implements HasFields, Jsonizable { + final public String topicID; + final public String topicGUID; + final public String topicName; + final public String[] typeIDs; + final public double score; + + public ReconCandidate(String topicID, String topicGUID, String topicName, String[] typeIDs, double score) { + this.topicID = topicID; + this.topicGUID = topicGUID; + this.topicName = topicName; + this.typeIDs = typeIDs; + this.score = score; + } + + public Object getField(String name, Properties bindings) { + if ("id".equals(name)) { + return topicID; + } else if ("guid".equals(name)) { + return topicGUID; + } else if ("name".equals(name)) { + return topicName; + } else if ("type".equals(name)) { + return typeIDs; + } else if ("score".equals(name)) { + return score; + } + return null; + } + + public boolean fieldAlsoHasFields(String name) { + return false; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("id"); writer.value(topicID); + writer.key("guid"); writer.value(topicGUID); + writer.key("name"); writer.value(topicName); + writer.key("score"); writer.value(score); + + /* if (!options.containsKey("reconCandidateOmitTypes")) */ { + writer.key("types"); writer.array(); + for (String typeID : typeIDs) { + writer.value(typeID); + } + writer.endArray(); + } + + writer.endObject(); + } + + static public ReconCandidate loadStreaming(String s) throws Exception { + JsonFactory jsonFactory = new JsonFactory(); + JsonParser jp = jsonFactory.createJsonParser(s); + + if (jp.nextToken() != JsonToken.START_OBJECT) { + return null; + } + return loadStreaming(jp); + } + + static public ReconCandidate loadStreaming(JsonParser jp) throws Exception { + JsonToken t = jp.getCurrentToken(); + if (t == JsonToken.VALUE_NULL || t != JsonToken.START_OBJECT) { + return null; + } + + String id = null; + String guid = null; + String name = null; + List types = null; + double score = 0; + + while (jp.nextToken() != JsonToken.END_OBJECT) { + String fieldName = jp.getCurrentName(); + jp.nextToken(); + + if ("id".equals(fieldName)) { + id = jp.getText(); + } else if ("guid".equals(fieldName)) { + guid = jp.getText(); + } else if ("name".equals(fieldName)) { + name = jp.getText(); + } else if ("score".equals(fieldName)) { + score = jp.getDoubleValue(); + } else if ("types".equals(fieldName)) { + if (jp.getCurrentToken() != JsonToken.START_ARRAY) { + return null; + } + + types = new ArrayList(); + + while (jp.nextToken() != JsonToken.END_ARRAY) { + types.add(jp.getText()); + } + } + } + + String[] typesA; + if (types != null) { + typesA = new String[types.size()]; + types.toArray(typesA); + } else { + typesA = new String[0]; + } + + return new ReconCandidate( + id, + guid, + name, + typesA, + score + ); + } } \ No newline at end of file diff --git a/src/main/java/com/metaweb/gridworks/model/Row.java b/src/main/java/com/metaweb/gridworks/model/Row.java index f12d01983..e9352d290 100644 --- a/src/main/java/com/metaweb/gridworks/model/Row.java +++ b/src/main/java/com/metaweb/gridworks/model/Row.java @@ -1,202 +1,202 @@ -package com.metaweb.gridworks.model; - -import java.io.Writer; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import java.util.Map.Entry; - -import org.codehaus.jackson.JsonFactory; -import org.codehaus.jackson.JsonParser; -import org.codehaus.jackson.JsonToken; -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.expr.CellTuple; -import com.metaweb.gridworks.expr.HasFields; -import com.metaweb.gridworks.util.Pool; - -public class Row implements HasFields, Jsonizable { - public boolean flagged; - public boolean starred; - final public List cells; - - transient public int recordIndex = -1; // -1 for rows that are not main record rows - transient public List contextRows; - transient public int[] contextRowSlots; - transient public int[] contextCellSlots; - - private static final String FLAGGED = "flagged"; - private static final String STARRED = "starred"; - - public Row(int cellCount) { - cells = new ArrayList(cellCount); - } - - protected Row(List cells, boolean flagged, boolean starred) { - this.cells = cells; - this.flagged = flagged; - this.starred = starred; - } - - public Row dup() { - Row row = new Row(cells.size()); - row.flagged = flagged; - row.starred = starred; - row.cells.addAll(cells); - return row; - } - - public Object getField(String name, Properties bindings) { - if (FLAGGED.equals(name)) { - return flagged; - } else if (STARRED.equals(name)) { - return starred; - } - return null; - } - - public boolean fieldAlsoHasFields(String name) { - return "cells".equals(name) || "record".equals(name); - } - - public boolean isEmpty() { - for (Cell cell : cells) { - if (cell != null && cell.value != null && !isValueBlank(cell.value)) { - return false; - } - } - return true; - } - - public Cell getCell(int cellIndex) { - if (cellIndex >= 0 && cellIndex < cells.size()) { - return cells.get(cellIndex); - } else { - return null; - } - } - - public Object getCellValue(int cellIndex) { - if (cellIndex >= 0 && cellIndex < cells.size()) { - Cell cell = cells.get(cellIndex); - if (cell != null) { - return cell.value; - } - } - return null; - } - - public boolean isCellBlank(int cellIndex) { - return isValueBlank(getCellValue(cellIndex)); - } - - protected boolean isValueBlank(Object value) { - return value == null || !(value instanceof String) || ((String) value).trim().length() == 0; - } - - public void setCell(int cellIndex, Cell cell) { - if (cellIndex < cells.size()) { - cells.set(cellIndex, cell); - } else { - while (cellIndex > cells.size()) { - cells.add(null); - } - cells.add(cell); - } - } - - public CellTuple getCellTuple(Project project) { - return new CellTuple(project, this); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key(FLAGGED); writer.value(flagged); - writer.key(STARRED); writer.value(starred); - - writer.key("cells"); writer.array(); - for (Cell cell : cells) { - if (cell != null) { - cell.write(writer, options); - } else { - writer.value(null); - } - } - writer.endArray(); - - if (!"save".equals(options.getProperty("mode"))) { - if (recordIndex >= 0) { - writer.key("j"); writer.value(recordIndex); - } - - if (options.containsKey("rowIndex")) { - writer.key("i"); writer.value(options.get("rowIndex")); - } - if (options.containsKey("extra")) { - Properties extra = (Properties) options.get("extra"); - if (extra != null) { - for (Entry e : extra.entrySet()) { - writer.key((String) e.getKey()); - writer.value(e.getValue()); - } - } - } - } - - writer.endObject(); - } - - public void save(Writer writer, Properties options) { - JSONWriter jsonWriter = new JSONWriter(writer); - try { - write(jsonWriter, options); - } catch (JSONException e) { - e.printStackTrace(); - } - } - - static public Row load(String s, Pool pool) throws Exception { - return s.length() == 0 ? null : - loadStreaming(s, pool); - } - - static public Row loadStreaming(String s, Pool pool) throws Exception { - JsonFactory jsonFactory = new JsonFactory(); - JsonParser jp = jsonFactory.createJsonParser(s); - - if (jp.nextToken() != JsonToken.START_OBJECT) { - return null; - } - - List cells = new ArrayList(); - boolean starred = false; - boolean flagged = false; - - while (jp.nextToken() != JsonToken.END_OBJECT) { - String fieldName = jp.getCurrentName(); - jp.nextToken(); - - if (STARRED.equals(fieldName)) { - starred = jp.getBooleanValue(); - } else if (FLAGGED.equals(fieldName)) { - flagged = jp.getBooleanValue(); - } else if ("cells".equals(fieldName)) { - if (jp.getCurrentToken() != JsonToken.START_ARRAY) { - return null; - } - - while (jp.nextToken() != JsonToken.END_ARRAY) { - Cell cell = Cell.loadStreaming(jp, pool); - - cells.add(cell); - } - } - } - - return (cells.size() > 0) ? new Row(cells, flagged, starred) : null; - } -} +package com.metaweb.gridworks.model; + +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Map.Entry; + +import org.codehaus.jackson.JsonFactory; +import org.codehaus.jackson.JsonParser; +import org.codehaus.jackson.JsonToken; +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.expr.CellTuple; +import com.metaweb.gridworks.expr.HasFields; +import com.metaweb.gridworks.util.Pool; + +public class Row implements HasFields, Jsonizable { + public boolean flagged; + public boolean starred; + final public List cells; + + transient public int recordIndex = -1; // -1 for rows that are not main record rows + transient public List contextRows; + transient public int[] contextRowSlots; + transient public int[] contextCellSlots; + + private static final String FLAGGED = "flagged"; + private static final String STARRED = "starred"; + + public Row(int cellCount) { + cells = new ArrayList(cellCount); + } + + protected Row(List cells, boolean flagged, boolean starred) { + this.cells = cells; + this.flagged = flagged; + this.starred = starred; + } + + public Row dup() { + Row row = new Row(cells.size()); + row.flagged = flagged; + row.starred = starred; + row.cells.addAll(cells); + return row; + } + + public Object getField(String name, Properties bindings) { + if (FLAGGED.equals(name)) { + return flagged; + } else if (STARRED.equals(name)) { + return starred; + } + return null; + } + + public boolean fieldAlsoHasFields(String name) { + return "cells".equals(name) || "record".equals(name); + } + + public boolean isEmpty() { + for (Cell cell : cells) { + if (cell != null && cell.value != null && !isValueBlank(cell.value)) { + return false; + } + } + return true; + } + + public Cell getCell(int cellIndex) { + if (cellIndex >= 0 && cellIndex < cells.size()) { + return cells.get(cellIndex); + } else { + return null; + } + } + + public Object getCellValue(int cellIndex) { + if (cellIndex >= 0 && cellIndex < cells.size()) { + Cell cell = cells.get(cellIndex); + if (cell != null) { + return cell.value; + } + } + return null; + } + + public boolean isCellBlank(int cellIndex) { + return isValueBlank(getCellValue(cellIndex)); + } + + protected boolean isValueBlank(Object value) { + return value == null || !(value instanceof String) || ((String) value).trim().length() == 0; + } + + public void setCell(int cellIndex, Cell cell) { + if (cellIndex < cells.size()) { + cells.set(cellIndex, cell); + } else { + while (cellIndex > cells.size()) { + cells.add(null); + } + cells.add(cell); + } + } + + public CellTuple getCellTuple(Project project) { + return new CellTuple(project, this); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key(FLAGGED); writer.value(flagged); + writer.key(STARRED); writer.value(starred); + + writer.key("cells"); writer.array(); + for (Cell cell : cells) { + if (cell != null) { + cell.write(writer, options); + } else { + writer.value(null); + } + } + writer.endArray(); + + if (!"save".equals(options.getProperty("mode"))) { + if (recordIndex >= 0) { + writer.key("j"); writer.value(recordIndex); + } + + if (options.containsKey("rowIndex")) { + writer.key("i"); writer.value(options.get("rowIndex")); + } + if (options.containsKey("extra")) { + Properties extra = (Properties) options.get("extra"); + if (extra != null) { + for (Entry e : extra.entrySet()) { + writer.key((String) e.getKey()); + writer.value(e.getValue()); + } + } + } + } + + writer.endObject(); + } + + public void save(Writer writer, Properties options) { + JSONWriter jsonWriter = new JSONWriter(writer); + try { + write(jsonWriter, options); + } catch (JSONException e) { + e.printStackTrace(); + } + } + + static public Row load(String s, Pool pool) throws Exception { + return s.length() == 0 ? null : + loadStreaming(s, pool); + } + + static public Row loadStreaming(String s, Pool pool) throws Exception { + JsonFactory jsonFactory = new JsonFactory(); + JsonParser jp = jsonFactory.createJsonParser(s); + + if (jp.nextToken() != JsonToken.START_OBJECT) { + return null; + } + + List cells = new ArrayList(); + boolean starred = false; + boolean flagged = false; + + while (jp.nextToken() != JsonToken.END_OBJECT) { + String fieldName = jp.getCurrentName(); + jp.nextToken(); + + if (STARRED.equals(fieldName)) { + starred = jp.getBooleanValue(); + } else if (FLAGGED.equals(fieldName)) { + flagged = jp.getBooleanValue(); + } else if ("cells".equals(fieldName)) { + if (jp.getCurrentToken() != JsonToken.START_ARRAY) { + return null; + } + + while (jp.nextToken() != JsonToken.END_ARRAY) { + Cell cell = Cell.loadStreaming(jp, pool); + + cells.add(cell); + } + } + } + + return (cells.size() > 0) ? new Row(cells, flagged, starred) : null; + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/changes/CellChange.java b/src/main/java/com/metaweb/gridworks/model/changes/CellChange.java index 4339e506a..0326eb9ad 100644 --- a/src/main/java/com/metaweb/gridworks/model/changes/CellChange.java +++ b/src/main/java/com/metaweb/gridworks/model/changes/CellChange.java @@ -1,82 +1,82 @@ -package com.metaweb.gridworks.model.changes; - -import java.io.IOException; -import java.io.LineNumberReader; -import java.io.Writer; -import java.util.Properties; - -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.util.Pool; - -public class CellChange implements Change { - final public int row; - final public int cellIndex; - final public Cell oldCell; - final public Cell newCell; - - public CellChange(int row, int cellIndex, Cell oldCell, Cell newCell) { - this.row = row; - this.cellIndex = cellIndex; - this.oldCell = oldCell; - this.newCell = newCell; - } - - public void apply(Project project) { - project.rows.get(row).setCell(cellIndex, newCell); - - project.columnModel.getColumnByCellIndex(cellIndex).clearPrecomputes(); - } - - public void revert(Project project) { - project.rows.get(row).setCell(cellIndex, oldCell); - - project.columnModel.getColumnByCellIndex(cellIndex).clearPrecomputes(); - } - - public void save(Writer writer, Properties options) throws IOException { - writer.write("row="); writer.write(Integer.toString(row)); writer.write('\n'); - writer.write("cell="); writer.write(Integer.toString(cellIndex)); writer.write('\n'); - - writer.write("old="); - if (oldCell != null) { - oldCell.save(writer, options); // one liner - } - writer.write('\n'); - - writer.write("new="); - if (newCell != null) { - newCell.save(writer, options); // one liner - } - writer.write('\n'); - - writer.write("/ec/\n"); // end of change marker - } - - static public CellChange load(LineNumberReader reader, Pool pool) throws Exception { - int row = -1; - int cellIndex = -1; - Cell oldCell = null; - Cell newCell = null; - - String line; - while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { - int equal = line.indexOf('='); - CharSequence field = line.subSequence(0, equal); - String value = line.substring(equal + 1); - - if ("row".equals(field)) { - row = Integer.parseInt(value); - } else if ("cell".equals(field)) { - cellIndex = Integer.parseInt(value); - } else if ("new".equals(field) && value.length() > 0) { - newCell = Cell.loadStreaming(value, pool); - } else if ("old".equals(field) && value.length() > 0) { - oldCell = Cell.loadStreaming(value, pool); - } - } - - return new CellChange(row, cellIndex, oldCell, newCell); - } -} +package com.metaweb.gridworks.model.changes; + +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Writer; +import java.util.Properties; + +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.util.Pool; + +public class CellChange implements Change { + final public int row; + final public int cellIndex; + final public Cell oldCell; + final public Cell newCell; + + public CellChange(int row, int cellIndex, Cell oldCell, Cell newCell) { + this.row = row; + this.cellIndex = cellIndex; + this.oldCell = oldCell; + this.newCell = newCell; + } + + public void apply(Project project) { + project.rows.get(row).setCell(cellIndex, newCell); + + project.columnModel.getColumnByCellIndex(cellIndex).clearPrecomputes(); + } + + public void revert(Project project) { + project.rows.get(row).setCell(cellIndex, oldCell); + + project.columnModel.getColumnByCellIndex(cellIndex).clearPrecomputes(); + } + + public void save(Writer writer, Properties options) throws IOException { + writer.write("row="); writer.write(Integer.toString(row)); writer.write('\n'); + writer.write("cell="); writer.write(Integer.toString(cellIndex)); writer.write('\n'); + + writer.write("old="); + if (oldCell != null) { + oldCell.save(writer, options); // one liner + } + writer.write('\n'); + + writer.write("new="); + if (newCell != null) { + newCell.save(writer, options); // one liner + } + writer.write('\n'); + + writer.write("/ec/\n"); // end of change marker + } + + static public CellChange load(LineNumberReader reader, Pool pool) throws Exception { + int row = -1; + int cellIndex = -1; + Cell oldCell = null; + Cell newCell = null; + + String line; + while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { + int equal = line.indexOf('='); + CharSequence field = line.subSequence(0, equal); + String value = line.substring(equal + 1); + + if ("row".equals(field)) { + row = Integer.parseInt(value); + } else if ("cell".equals(field)) { + cellIndex = Integer.parseInt(value); + } else if ("new".equals(field) && value.length() > 0) { + newCell = Cell.loadStreaming(value, pool); + } else if ("old".equals(field) && value.length() > 0) { + oldCell = Cell.loadStreaming(value, pool); + } + } + + return new CellChange(row, cellIndex, oldCell, newCell); + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/changes/ColumnSplitChange.java b/src/main/java/com/metaweb/gridworks/model/changes/ColumnSplitChange.java index 5da9367a1..db99be7a6 100644 --- a/src/main/java/com/metaweb/gridworks/model/changes/ColumnSplitChange.java +++ b/src/main/java/com/metaweb/gridworks/model/changes/ColumnSplitChange.java @@ -19,10 +19,10 @@ import com.metaweb.gridworks.model.Row; import com.metaweb.gridworks.util.Pool; public class ColumnSplitChange implements Change { - final protected String _columnName; - - final protected List _columnNames; - final protected List _rowIndices; + final protected String _columnName; + + final protected List _columnNames; + final protected List _rowIndices; final protected List> _tuples; final protected boolean _removeOriginalColumn; @@ -30,20 +30,20 @@ public class ColumnSplitChange implements Change { protected Column _column; protected int _columnIndex; - protected int _firstNewCellIndex = -1; - protected List _oldRows; - protected List _newRows; + protected int _firstNewCellIndex = -1; + protected List _oldRows; + protected List _newRows; public ColumnSplitChange( - String columnName, - List columnNames, - List rowIndices, - List> tuples, - boolean removeOriginalColumn - ) { - _columnName = columnName; - - _columnNames = columnNames; + String columnName, + List columnNames, + List rowIndices, + List> tuples, + boolean removeOriginalColumn + ) { + _columnName = columnName; + + _columnNames = columnNames; _rowIndices = rowIndices; _tuples = tuples; @@ -60,10 +60,10 @@ public class ColumnSplitChange implements Change { Column column, int columnIndex, - int firstNewCellIndex, - List oldRows, - List newRows - ) { + int firstNewCellIndex, + List oldRows, + List newRows + ) { _columnName = columnName; _columnNames = columnNames; @@ -83,14 +83,14 @@ public class ColumnSplitChange implements Change { public void apply(Project project) { synchronized (project) { if (_firstNewCellIndex < 0) { - _firstNewCellIndex = project.columnModel.allocateNewCellIndex(); - for (int i = 1; i < _columnNames.size(); i++) { - project.columnModel.allocateNewCellIndex(); - } - - _column = project.columnModel.getColumnByName(_columnName); - _columnIndex = project.columnModel.getColumnIndexByName(_columnName); - + _firstNewCellIndex = project.columnModel.allocateNewCellIndex(); + for (int i = 1; i < _columnNames.size(); i++) { + project.columnModel.allocateNewCellIndex(); + } + + _column = project.columnModel.getColumnByName(_columnName); + _columnIndex = project.columnModel.getColumnIndexByName(_columnName); + _oldRows = new ArrayList(_rowIndices.size()); _newRows = new ArrayList(_rowIndices.size()); @@ -100,22 +100,22 @@ public class ColumnSplitChange implements Change { int r = _rowIndices.get(i); List tuple = _tuples.get(i); - Row oldRow = project.rows.get(r); - Row newRow = oldRow.dup(); - - _oldRows.add(oldRow); - _newRows.add(newRow); - - for (int c = 0; c < tuple.size(); c++) { - Serializable value = tuple.get(c); - if (value != null) { - newRow.setCell(_firstNewCellIndex + c, new Cell(value, null)); - } - } - - if (_removeOriginalColumn) { - newRow.setCell(cellIndex, null); - } + Row oldRow = project.rows.get(r); + Row newRow = oldRow.dup(); + + _oldRows.add(oldRow); + _newRows.add(newRow); + + for (int c = 0; c < tuple.size(); c++) { + Serializable value = tuple.get(c); + if (value != null) { + newRow.setCell(_firstNewCellIndex + c, new Cell(value, null)); + } + } + + if (_removeOriginalColumn) { + newRow.setCell(cellIndex, null); + } } } @@ -127,12 +127,12 @@ public class ColumnSplitChange implements Change { } for (int i = 0; i < _columnNames.size(); i++) { - String name = _columnNames.get(i); - int cellIndex = _firstNewCellIndex + i; - - Column column = new Column(cellIndex, name); - - project.columnModel.columns.add(_columnIndex + 1 + i, column); + String name = _columnNames.get(i); + int cellIndex = _firstNewCellIndex + i; + + Column column = new Column(cellIndex, name); + + project.columnModel.columns.add(_columnIndex + 1 + i, column); } if (_removeOriginalColumn) { @@ -158,7 +158,7 @@ public class ColumnSplitChange implements Change { } for (int i = 0; i < _columnNames.size(); i++) { - project.columnModel.columns.remove(_columnIndex + 1); + project.columnModel.columns.remove(_columnIndex + 1); } project.columnModel.update(); @@ -167,29 +167,29 @@ public class ColumnSplitChange implements Change { } public void save(Writer writer, Properties options) throws IOException { - writer.write("columnName="); writer.write(_columnName); writer.write('\n'); - + writer.write("columnName="); writer.write(_columnName); writer.write('\n'); + writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n'); for (String name : _columnNames) { - writer.write(name); writer.write('\n'); + writer.write(name); writer.write('\n'); } writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n'); for (Integer rowIndex : _rowIndices) { - writer.write(rowIndex.toString()); writer.write('\n'); + writer.write(rowIndex.toString()); writer.write('\n'); } writer.write("tupleCount="); writer.write(Integer.toString(_tuples.size())); writer.write('\n'); for (List tuple : _tuples) { writer.write(Integer.toString(tuple.size())); writer.write('\n'); for (Serializable value : tuple) { - if (value == null) { - writer.write("null"); - } else if (value instanceof String) { - writer.write(JSONObject.quote((String) value)); - } else { - writer.write(value.toString()); - } - writer.write('\n'); + if (value == null) { + writer.write("null"); + } else if (value instanceof String) { + writer.write(JSONObject.quote((String) value)); + } else { + writer.write(value.toString()); + } + writer.write('\n'); } } writer.write("removeOriginalColumn="); writer.write(Boolean.toString(_removeOriginalColumn)); writer.write('\n'); @@ -197,8 +197,8 @@ public class ColumnSplitChange implements Change { writer.write("column="); _column.save(writer); writer.write('\n'); writer.write("columnIndex="); writer.write(Integer.toString(_columnIndex)); writer.write('\n'); - writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n'); - + writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n'); + writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n'); for (Row row : _newRows) { row.save(writer, options); @@ -233,7 +233,7 @@ public class ColumnSplitChange implements Change { String value = line.substring(equal + 1); if ("columnName".equals(field)) { - columnName = value; + columnName = value; } else if ("columnNameCount".equals(field)) { int count = Integer.parseInt(value); @@ -267,8 +267,8 @@ public class ColumnSplitChange implements Change { List tuple = new ArrayList(valueCount); for (int r = 0; r < valueCount; r++) { - line = reader.readLine(); - + line = reader.readLine(); + JSONTokener t = new JSONTokener(line); Object o = t.nextValue(); diff --git a/src/main/java/com/metaweb/gridworks/model/changes/DataExtensionChange.java b/src/main/java/com/metaweb/gridworks/model/changes/DataExtensionChange.java index 5e2ca2014..15d20b1b4 100644 --- a/src/main/java/com/metaweb/gridworks/model/changes/DataExtensionChange.java +++ b/src/main/java/com/metaweb/gridworks/model/changes/DataExtensionChange.java @@ -30,35 +30,35 @@ import com.metaweb.gridworks.util.Pool; import com.metaweb.gridworks.util.FreebaseDataExtensionJob.DataExtension; public class DataExtensionChange implements Change { - final protected String _baseColumnName; - final protected int _columnInsertIndex; - - final protected List _columnNames; - final protected List _columnTypes; - - final protected List _rowIndices; + final protected String _baseColumnName; + final protected int _columnInsertIndex; + + final protected List _columnNames; + final protected List _columnTypes; + + final protected List _rowIndices; final protected List _dataExtensions; protected long _historyEntryID; - protected int _firstNewCellIndex = -1; - protected List _oldRows; - protected List _newRows; + protected int _firstNewCellIndex = -1; + protected List _oldRows; + protected List _newRows; public DataExtensionChange( - String baseColumnName, - int columnInsertIndex, - List columnNames, - List columnTypes, - List rowIndices, - List dataExtensions, - long historyEntryID - ) { - _baseColumnName = baseColumnName; - _columnInsertIndex = columnInsertIndex; - - _columnNames = columnNames; - _columnTypes = columnTypes; - + String baseColumnName, + int columnInsertIndex, + List columnNames, + List columnTypes, + List rowIndices, + List dataExtensions, + long historyEntryID + ) { + _baseColumnName = baseColumnName; + _columnInsertIndex = columnInsertIndex; + + _columnNames = columnNames; + _columnTypes = columnTypes; + _rowIndices = rowIndices; _dataExtensions = dataExtensions; @@ -66,22 +66,22 @@ public class DataExtensionChange implements Change { } protected DataExtensionChange( - String baseColumnName, - int columnInsertIndex, - - List columnNames, + String baseColumnName, + int columnInsertIndex, + + List columnNames, List columnTypes, - List rowIndices, - List dataExtensions, - int firstNewCellIndex, - List oldRows, - List newRows - ) { - _baseColumnName = baseColumnName; - _columnInsertIndex = columnInsertIndex; - - _columnNames = columnNames; + List rowIndices, + List dataExtensions, + int firstNewCellIndex, + List oldRows, + List newRows + ) { + _baseColumnName = baseColumnName; + _columnInsertIndex = columnInsertIndex; + + _columnNames = columnNames; _columnTypes = columnTypes; _rowIndices = rowIndices; @@ -95,11 +95,11 @@ public class DataExtensionChange implements Change { public void apply(Project project) { synchronized (project) { if (_firstNewCellIndex < 0) { - _firstNewCellIndex = project.columnModel.allocateNewCellIndex(); - for (int i = 1; i < _columnNames.size(); i++) { - project.columnModel.allocateNewCellIndex(); - } - + _firstNewCellIndex = project.columnModel.allocateNewCellIndex(); + for (int i = 1; i < _columnNames.size(); i++) { + project.columnModel.allocateNewCellIndex(); + } + _oldRows = new ArrayList(project.rows); _newRows = new ArrayList(project.rows.size()); @@ -116,19 +116,19 @@ public class DataExtensionChange implements Change { Map reconMap = new HashMap(); for (int r = 0; r < _oldRows.size(); r++) { - Row oldRow = _oldRows.get(r); - if (r < rowIndex) { - _newRows.add(oldRow.dup()); - continue; - } - - if (dataExtension == null || dataExtension.data.length == 0) { - _newRows.add(oldRow); - } else { - Row firstNewRow = oldRow.dup(); - extendRow(firstNewRow, dataExtension, 0, reconMap); - _newRows.add(firstNewRow); - + Row oldRow = _oldRows.get(r); + if (r < rowIndex) { + _newRows.add(oldRow.dup()); + continue; + } + + if (dataExtension == null || dataExtension.data.length == 0) { + _newRows.add(oldRow); + } else { + Row firstNewRow = oldRow.dup(); + extendRow(firstNewRow, dataExtension, 0, reconMap); + _newRows.add(firstNewRow); + int r2 = r + 1; for (int subR = 1; subR < dataExtension.data.length; subR++) { if (r2 < project.rows.size()) { @@ -153,8 +153,8 @@ public class DataExtensionChange implements Change { } r = r2 - 1; // r will be incremented by the for loop anyway - } - + } + rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size(); dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null; index++; @@ -165,14 +165,14 @@ public class DataExtensionChange implements Change { project.rows.addAll(_newRows); for (int i = 0; i < _columnNames.size(); i++) { - String name = _columnNames.get(i); - int cellIndex = _firstNewCellIndex + i; - - Column column = new Column(cellIndex, name); - column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i))); - column.setReconStats(ReconStats.create(project, cellIndex)); - - project.columnModel.columns.add(_columnInsertIndex + i, column); + String name = _columnNames.get(i); + int cellIndex = _firstNewCellIndex + i; + + Column column = new Column(cellIndex, name); + column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i))); + column.setReconStats(ReconStats.create(project, cellIndex)); + + project.columnModel.columns.add(_columnInsertIndex + i, column); } project.columnModel.update(); @@ -181,40 +181,40 @@ public class DataExtensionChange implements Change { } protected void extendRow( - Row row, - DataExtension dataExtension, - int extensionRowIndex, - Map reconMap - ) { - Object[] values = dataExtension.data[extensionRowIndex]; - for (int c = 0; c < values.length; c++) { - Object value = values[c]; - Cell cell = null; - - if (value instanceof ReconCandidate) { - ReconCandidate rc = (ReconCandidate) value; - Recon recon; - if (reconMap.containsKey(rc.topicGUID)) { - recon = reconMap.get(rc.topicGUID); - } else { - recon = new Recon(_historyEntryID); - recon.addCandidate(rc); - recon.service = "mql"; - recon.match = rc; - recon.matchRank = 0; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - recon.judgmentBatchSize = 1; - - reconMap.put(rc.topicGUID, recon); - } - cell = new Cell(rc.topicName, recon); - } else { - cell = new Cell((Serializable) value, null); - } - - row.setCell(_firstNewCellIndex + c, cell); - } + Row row, + DataExtension dataExtension, + int extensionRowIndex, + Map reconMap + ) { + Object[] values = dataExtension.data[extensionRowIndex]; + for (int c = 0; c < values.length; c++) { + Object value = values[c]; + Cell cell = null; + + if (value instanceof ReconCandidate) { + ReconCandidate rc = (ReconCandidate) value; + Recon recon; + if (reconMap.containsKey(rc.topicGUID)) { + recon = reconMap.get(rc.topicGUID); + } else { + recon = new Recon(_historyEntryID); + recon.addCandidate(rc); + recon.service = "mql"; + recon.match = rc; + recon.matchRank = 0; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.judgmentBatchSize = 1; + + reconMap.put(rc.topicGUID, recon); + } + cell = new Cell(rc.topicName, recon); + } else { + cell = new Cell((Serializable) value, null); + } + + row.setCell(_firstNewCellIndex + c, cell); + } } public void revert(Project project) { @@ -223,7 +223,7 @@ public class DataExtensionChange implements Change { project.rows.addAll(_oldRows); for (int i = 0; i < _columnNames.size(); i++) { - project.columnModel.columns.remove(_columnInsertIndex); + project.columnModel.columns.remove(_columnInsertIndex); } project.columnModel.update(); @@ -232,11 +232,11 @@ public class DataExtensionChange implements Change { } public void save(Writer writer, Properties options) throws IOException { - writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n'); - writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n'); + writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n'); + writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n'); writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n'); for (String name : _columnNames) { - writer.write(name); writer.write('\n'); + writer.write(name); writer.write('\n'); } writer.write("columnTypeCount="); writer.write(Integer.toString(_columnTypes.size())); writer.write('\n'); for (FreebaseType type : _columnTypes) { @@ -251,35 +251,35 @@ public class DataExtensionChange implements Change { } writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n'); for (Integer rowIndex : _rowIndices) { - writer.write(rowIndex.toString()); writer.write('\n'); + writer.write(rowIndex.toString()); writer.write('\n'); } writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n'); for (DataExtension dataExtension : _dataExtensions) { writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n'); for (Object[] values : dataExtension.data) { - for (Object value : values) { - if (value == null) { - writer.write("null"); - } else if (value instanceof ReconCandidate) { + for (Object value : values) { + if (value == null) { + writer.write("null"); + } else if (value instanceof ReconCandidate) { try { JSONWriter jsonWriter = new JSONWriter(writer); ((ReconCandidate) value).write(jsonWriter, options); } catch (JSONException e) { // ??? } - } else if (value instanceof String) { - writer.write(JSONObject.quote((String) value)); - } else { - writer.write(value.toString()); - } - writer.write('\n'); - } + } else if (value instanceof String) { + writer.write(JSONObject.quote((String) value)); + } else { + writer.write(value.toString()); + } + writer.write('\n'); + } } } - writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n'); - + writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n'); + writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n'); for (Row row : _newRows) { row.save(writer, options); @@ -294,15 +294,15 @@ public class DataExtensionChange implements Change { } static public Change load(LineNumberReader reader, Pool pool) throws Exception { - String baseColumnName = null; - int columnInsertIndex = -1; - - List columnNames = null; - List columnTypes = null; - - List rowIndices = null; - List dataExtensions = null; - + String baseColumnName = null; + int columnInsertIndex = -1; + + List columnNames = null; + List columnTypes = null; + + List rowIndices = null; + List dataExtensions = null; + List oldRows = null; List newRows = null; @@ -315,11 +315,11 @@ public class DataExtensionChange implements Change { String value = line.substring(equal + 1); if ("baseColumnName".equals(field)) { - baseColumnName = value; + baseColumnName = value; } else if ("columnInsertIndex".equals(field)) { - columnInsertIndex = Integer.parseInt(value); + columnInsertIndex = Integer.parseInt(value); } else if ("firstNewCellIndex".equals(field)) { - firstNewCellIndex = Integer.parseInt(value); + firstNewCellIndex = Integer.parseInt(value); } else if ("rowIndexCount".equals(field)) { int count = Integer.parseInt(value); @@ -361,14 +361,14 @@ public class DataExtensionChange implements Change { Object[][] data = new Object[rowCount][]; for (int r = 0; r < rowCount; r++) { - Object[] row = new Object[columnNames.size()]; - for (int c = 0; c < columnNames.size(); c++) { - line = reader.readLine(); - - row[c] = ReconCandidate.loadStreaming(line); - } - - data[r] = row; + Object[] row = new Object[columnNames.size()]; + for (int c = 0; c < columnNames.size(); c++) { + line = reader.readLine(); + + row[c] = ReconCandidate.loadStreaming(line); + } + + data[r] = row; } dataExtensions.add(new DataExtension(data)); @@ -398,15 +398,15 @@ public class DataExtensionChange implements Change { } DataExtensionChange change = new DataExtensionChange( - baseColumnName, - columnInsertIndex, - columnNames, - columnTypes, - rowIndices, - dataExtensions, - firstNewCellIndex, - oldRows, - newRows + baseColumnName, + columnInsertIndex, + columnNames, + columnTypes, + rowIndices, + dataExtensions, + firstNewCellIndex, + oldRows, + newRows ); diff --git a/src/main/java/com/metaweb/gridworks/model/changes/MassCellChange.java b/src/main/java/com/metaweb/gridworks/model/changes/MassCellChange.java index 445de2f35..bff8011a5 100644 --- a/src/main/java/com/metaweb/gridworks/model/changes/MassCellChange.java +++ b/src/main/java/com/metaweb/gridworks/model/changes/MassCellChange.java @@ -1,127 +1,127 @@ -package com.metaweb.gridworks.model.changes; - -import java.io.IOException; -import java.io.LineNumberReader; -import java.io.Writer; -import java.util.List; -import java.util.Properties; - -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.util.Pool; - -public class MassCellChange implements Change { - final protected CellChange[] _cellChanges; - final protected String _commonColumnName; - final protected boolean _updateRowContextDependencies; - - public MassCellChange( - CellChange[] cellChanges, - String commonColumnName, - boolean updateRowContextDependencies) { - - _cellChanges = cellChanges; - _commonColumnName = commonColumnName; - _updateRowContextDependencies = updateRowContextDependencies; - } - - public MassCellChange( - List cellChanges, - String commonColumnName, - boolean updateRowContextDependencies) { - - _cellChanges = new CellChange[cellChanges.size()]; - _commonColumnName = commonColumnName; - cellChanges.toArray(_cellChanges); - - _updateRowContextDependencies = updateRowContextDependencies; - } - - public MassCellChange(CellChange cellChange, String commonColumnName, boolean updateRowContextDependencies) { - _cellChanges = new CellChange[1]; - _cellChanges[0] = cellChange; - - _commonColumnName = commonColumnName; - - _updateRowContextDependencies = updateRowContextDependencies; - } - - public void apply(Project project) { - synchronized (project) { - List rows = project.rows; - - for (CellChange cellChange : _cellChanges) { - rows.get(cellChange.row).setCell(cellChange.cellIndex, cellChange.newCell); - } - - if (_commonColumnName != null) { - Column column = project.columnModel.getColumnByName(_commonColumnName); - column.clearPrecomputes(); - } - - if (_updateRowContextDependencies) { - project.recomputeRowContextDependencies(); - } - } - } - - public void revert(Project project) { - synchronized (project) { - List rows = project.rows; - - for (CellChange cellChange : _cellChanges) { - rows.get(cellChange.row).setCell(cellChange.cellIndex, cellChange.oldCell); - } - - if (_commonColumnName != null) { - Column column = project.columnModel.getColumnByName(_commonColumnName); - column.clearPrecomputes(); - } - - if (_updateRowContextDependencies) { - project.recomputeRowContextDependencies(); - } - } - } - - public void save(Writer writer, Properties options) throws IOException { - writer.write("commonColumnName="); writer.write(_commonColumnName); writer.write('\n'); - writer.write("updateRowContextDependencies="); writer.write(Boolean.toString(_updateRowContextDependencies)); writer.write('\n'); - writer.write("cellChangeCount="); writer.write(Integer.toString(_cellChanges.length)); writer.write('\n'); - for (CellChange c : _cellChanges) { - c.save(writer, options); - } - writer.write("/ec/\n"); // end of change marker - } - - static public Change load(LineNumberReader reader, Pool pool) throws Exception { - String commonColumnName = null; - boolean updateRowContextDependencies = false; - CellChange[] cellChanges = null; - - String line; - while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { - int equal = line.indexOf('='); - CharSequence field = line.subSequence(0, equal); - - if ("commonColumnName".equals(field)) { - commonColumnName = line.substring(equal + 1); - } else if ("updateRowContextDependencies".equals(field)) { - updateRowContextDependencies = Boolean.parseBoolean(line.substring(equal + 1)); - } else if ("cellChangeCount".equals(field)) { - int cellChangeCount = Integer.parseInt(line.substring(equal + 1)); - - cellChanges = new CellChange[cellChangeCount]; - for (int i = 0; i < cellChangeCount; i++) { - cellChanges[i] = CellChange.load(reader, pool); - } - } - } - - MassCellChange change = new MassCellChange(cellChanges, commonColumnName, updateRowContextDependencies); - - return change; - } -} +package com.metaweb.gridworks.model.changes; + +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Writer; +import java.util.List; +import java.util.Properties; + +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.util.Pool; + +public class MassCellChange implements Change { + final protected CellChange[] _cellChanges; + final protected String _commonColumnName; + final protected boolean _updateRowContextDependencies; + + public MassCellChange( + CellChange[] cellChanges, + String commonColumnName, + boolean updateRowContextDependencies) { + + _cellChanges = cellChanges; + _commonColumnName = commonColumnName; + _updateRowContextDependencies = updateRowContextDependencies; + } + + public MassCellChange( + List cellChanges, + String commonColumnName, + boolean updateRowContextDependencies) { + + _cellChanges = new CellChange[cellChanges.size()]; + _commonColumnName = commonColumnName; + cellChanges.toArray(_cellChanges); + + _updateRowContextDependencies = updateRowContextDependencies; + } + + public MassCellChange(CellChange cellChange, String commonColumnName, boolean updateRowContextDependencies) { + _cellChanges = new CellChange[1]; + _cellChanges[0] = cellChange; + + _commonColumnName = commonColumnName; + + _updateRowContextDependencies = updateRowContextDependencies; + } + + public void apply(Project project) { + synchronized (project) { + List rows = project.rows; + + for (CellChange cellChange : _cellChanges) { + rows.get(cellChange.row).setCell(cellChange.cellIndex, cellChange.newCell); + } + + if (_commonColumnName != null) { + Column column = project.columnModel.getColumnByName(_commonColumnName); + column.clearPrecomputes(); + } + + if (_updateRowContextDependencies) { + project.recomputeRowContextDependencies(); + } + } + } + + public void revert(Project project) { + synchronized (project) { + List rows = project.rows; + + for (CellChange cellChange : _cellChanges) { + rows.get(cellChange.row).setCell(cellChange.cellIndex, cellChange.oldCell); + } + + if (_commonColumnName != null) { + Column column = project.columnModel.getColumnByName(_commonColumnName); + column.clearPrecomputes(); + } + + if (_updateRowContextDependencies) { + project.recomputeRowContextDependencies(); + } + } + } + + public void save(Writer writer, Properties options) throws IOException { + writer.write("commonColumnName="); writer.write(_commonColumnName); writer.write('\n'); + writer.write("updateRowContextDependencies="); writer.write(Boolean.toString(_updateRowContextDependencies)); writer.write('\n'); + writer.write("cellChangeCount="); writer.write(Integer.toString(_cellChanges.length)); writer.write('\n'); + for (CellChange c : _cellChanges) { + c.save(writer, options); + } + writer.write("/ec/\n"); // end of change marker + } + + static public Change load(LineNumberReader reader, Pool pool) throws Exception { + String commonColumnName = null; + boolean updateRowContextDependencies = false; + CellChange[] cellChanges = null; + + String line; + while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { + int equal = line.indexOf('='); + CharSequence field = line.subSequence(0, equal); + + if ("commonColumnName".equals(field)) { + commonColumnName = line.substring(equal + 1); + } else if ("updateRowContextDependencies".equals(field)) { + updateRowContextDependencies = Boolean.parseBoolean(line.substring(equal + 1)); + } else if ("cellChangeCount".equals(field)) { + int cellChangeCount = Integer.parseInt(line.substring(equal + 1)); + + cellChanges = new CellChange[cellChangeCount]; + for (int i = 0; i < cellChangeCount; i++) { + cellChanges[i] = CellChange.load(reader, pool); + } + } + } + + MassCellChange change = new MassCellChange(cellChanges, commonColumnName, updateRowContextDependencies); + + return change; + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/changes/MassChange.java b/src/main/java/com/metaweb/gridworks/model/changes/MassChange.java index 7581e1eff..a9011308f 100644 --- a/src/main/java/com/metaweb/gridworks/model/changes/MassChange.java +++ b/src/main/java/com/metaweb/gridworks/model/changes/MassChange.java @@ -1,82 +1,82 @@ -package com.metaweb.gridworks.model.changes; - - import java.io.IOException; -import java.io.LineNumberReader; -import java.io.Writer; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.History; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.util.Pool; - -public class MassChange implements Change { - final protected List _changes; - final protected boolean _updateRowContextDependencies; - - public MassChange(List changes, boolean updateRowContextDependencies) { - _changes = changes; - _updateRowContextDependencies = updateRowContextDependencies; - } - - public void apply(Project project) { - synchronized (project) { - for (Change change : _changes) { - change.apply(project); - } - - if (_updateRowContextDependencies) { - project.recomputeRowContextDependencies(); - } - } - } - - public void revert(Project project) { - synchronized (project) { - for (Change change : _changes) { - change.revert(project); - } - - if (_updateRowContextDependencies) { - project.recomputeRowContextDependencies(); - } - } - } - - public void save(Writer writer, Properties options) throws IOException { - writer.write("updateRowContextDependencies="); writer.write(Boolean.toString(_updateRowContextDependencies)); writer.write('\n'); - writer.write("changeCount="); writer.write(Integer.toString(_changes.size())); writer.write('\n'); - for (Change c : _changes) { - c.save(writer, options); - } - writer.write("/ec/\n"); // end of change marker - } - - static public Change load(LineNumberReader reader, Pool pool) throws Exception { - boolean updateRowContextDependencies = false; - List changes = null; - - String line; - while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { - int equal = line.indexOf('='); - CharSequence field = line.subSequence(0, equal); - - if ("updateRowContextDependencies".equals(field)) { - updateRowContextDependencies = Boolean.parseBoolean(line.substring(equal + 1)); - } else if ("changeCount".equals(field)) { - int changeCount = Integer.parseInt(line.substring(equal + 1)); - - changes = new ArrayList(changeCount); - for (int i = 0; i < changeCount; i++) { - changes.add(History.readOneChange(reader, pool)); - } - } - } - - MassChange change = new MassChange(changes, updateRowContextDependencies); - - return change; - } -} +package com.metaweb.gridworks.model.changes; + + import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.History; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.util.Pool; + +public class MassChange implements Change { + final protected List _changes; + final protected boolean _updateRowContextDependencies; + + public MassChange(List changes, boolean updateRowContextDependencies) { + _changes = changes; + _updateRowContextDependencies = updateRowContextDependencies; + } + + public void apply(Project project) { + synchronized (project) { + for (Change change : _changes) { + change.apply(project); + } + + if (_updateRowContextDependencies) { + project.recomputeRowContextDependencies(); + } + } + } + + public void revert(Project project) { + synchronized (project) { + for (Change change : _changes) { + change.revert(project); + } + + if (_updateRowContextDependencies) { + project.recomputeRowContextDependencies(); + } + } + } + + public void save(Writer writer, Properties options) throws IOException { + writer.write("updateRowContextDependencies="); writer.write(Boolean.toString(_updateRowContextDependencies)); writer.write('\n'); + writer.write("changeCount="); writer.write(Integer.toString(_changes.size())); writer.write('\n'); + for (Change c : _changes) { + c.save(writer, options); + } + writer.write("/ec/\n"); // end of change marker + } + + static public Change load(LineNumberReader reader, Pool pool) throws Exception { + boolean updateRowContextDependencies = false; + List changes = null; + + String line; + while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { + int equal = line.indexOf('='); + CharSequence field = line.subSequence(0, equal); + + if ("updateRowContextDependencies".equals(field)) { + updateRowContextDependencies = Boolean.parseBoolean(line.substring(equal + 1)); + } else if ("changeCount".equals(field)) { + int changeCount = Integer.parseInt(line.substring(equal + 1)); + + changes = new ArrayList(changeCount); + for (int i = 0; i < changeCount; i++) { + changes.add(History.readOneChange(reader, pool)); + } + } + } + + MassChange change = new MassChange(changes, updateRowContextDependencies); + + return change; + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/changes/ReconChange.java b/src/main/java/com/metaweb/gridworks/model/changes/ReconChange.java index a02203217..b12b99180 100644 --- a/src/main/java/com/metaweb/gridworks/model/changes/ReconChange.java +++ b/src/main/java/com/metaweb/gridworks/model/changes/ReconChange.java @@ -1,174 +1,174 @@ -/** - * - */ -package com.metaweb.gridworks.model.changes; - -import java.io.IOException; -import java.io.LineNumberReader; -import java.io.Writer; -import java.util.List; -import java.util.Properties; - -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.ReconStats; -import com.metaweb.gridworks.model.recon.ReconConfig; -import com.metaweb.gridworks.util.ParsingUtilities; -import com.metaweb.gridworks.util.Pool; - -public class ReconChange extends MassCellChange { - final protected ReconConfig _newReconConfig; - protected ReconStats _newReconStats; - - protected ReconConfig _oldReconConfig; - protected ReconStats _oldReconStats; - - public ReconChange( - List cellChanges, - String commonColumnName, - ReconConfig newReconConfig, - ReconStats newReconStats // can be null - ) { - super(cellChanges, commonColumnName, false); - _newReconConfig = newReconConfig; - _newReconStats = newReconStats; - } - - public ReconChange( - CellChange[] cellChanges, - String commonColumnName, - ReconConfig newReconConfig, - ReconStats newReconStats // can be null - ) { - super(cellChanges, commonColumnName, false); - _newReconConfig = newReconConfig; - _newReconStats = newReconStats; - } - - public ReconChange( - CellChange cellChange, - String commonColumnName, - ReconConfig newReconConfig, - ReconStats newReconStats // can be null - ) { - super(cellChange, commonColumnName, false); - _newReconConfig = newReconConfig; - _newReconStats = newReconStats; - } - - @Override - public void apply(Project project) { - synchronized (project) { - super.apply(project); - - Column column = project.columnModel.getColumnByName(_commonColumnName); - - if (_newReconStats == null) { - _newReconStats = ReconStats.create(project, column.getCellIndex()); - } - - _oldReconConfig = column.getReconConfig(); - _oldReconStats = column.getReconStats(); - - column.setReconConfig(_newReconConfig); - column.setReconStats(_newReconStats); - - column.clearPrecomputes(); - } - } - - @Override - public void revert(Project project) { - synchronized (project) { - super.revert(project); - - Column column = project.columnModel.getColumnByName(_commonColumnName); - column.setReconConfig(_oldReconConfig); - column.setReconStats(_oldReconStats); - - column.clearPrecomputes(); - } - } - - public void save(Writer writer, Properties options) throws IOException { - writer.write("newReconConfig="); - if (_newReconConfig != null) { - _newReconConfig.save(writer); - } - writer.write('\n'); - - writer.write("newReconStats="); - if (_newReconStats != null) { - _newReconStats.save(writer); - } - writer.write('\n'); - - writer.write("oldReconConfig="); - if (_oldReconConfig != null) { - _oldReconConfig.save(writer); - } - writer.write('\n'); - - writer.write("oldReconStats="); - if (_oldReconStats != null) { - _oldReconStats.save(writer); - } - writer.write('\n'); - - super.save(writer, options); - } - - static public Change load(LineNumberReader reader, Pool pool) throws Exception { - ReconConfig newReconConfig = null; - ReconStats newReconStats = null; - ReconConfig oldReconConfig = null; - ReconStats oldReconStats = null; - - String commonColumnName = null; - CellChange[] cellChanges = null; - - String line; - while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { - int equal = line.indexOf('='); - - CharSequence field = line.subSequence(0, equal); - String value = line.substring(equal + 1); - - if ("newReconConfig".equals(field)) { - if (value.length() > 0) { - newReconConfig = ReconConfig.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); - } - } else if ("newReconStats".equals(field)) { - if (value.length() > 0) { - newReconStats = ReconStats.load(ParsingUtilities.evaluateJsonStringToObject(value)); - } - } else if ("oldReconConfig".equals(field)) { - if (value.length() > 0) { - oldReconConfig = ReconConfig.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); - } - } else if ("oldReconStats".equals(field)) { - if (value.length() > 0) { - oldReconStats = ReconStats.load(ParsingUtilities.evaluateJsonStringToObject(value)); - } - } else if ("commonColumnName".equals(field)) { - commonColumnName = value; - } else if ("cellChangeCount".equals(field)) { - int cellChangeCount = Integer.parseInt(value); - - cellChanges = new CellChange[cellChangeCount]; - for (int i = 0; i < cellChangeCount; i++) { - cellChanges[i] = CellChange.load(reader, pool); - } - } - } - - ReconChange change = new ReconChange( - cellChanges, commonColumnName, newReconConfig, newReconStats); - - change._oldReconConfig = oldReconConfig; - change._oldReconStats = oldReconStats; - - return change; - } +/** + * + */ +package com.metaweb.gridworks.model.changes; + +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Writer; +import java.util.List; +import java.util.Properties; + +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.ReconStats; +import com.metaweb.gridworks.model.recon.ReconConfig; +import com.metaweb.gridworks.util.ParsingUtilities; +import com.metaweb.gridworks.util.Pool; + +public class ReconChange extends MassCellChange { + final protected ReconConfig _newReconConfig; + protected ReconStats _newReconStats; + + protected ReconConfig _oldReconConfig; + protected ReconStats _oldReconStats; + + public ReconChange( + List cellChanges, + String commonColumnName, + ReconConfig newReconConfig, + ReconStats newReconStats // can be null + ) { + super(cellChanges, commonColumnName, false); + _newReconConfig = newReconConfig; + _newReconStats = newReconStats; + } + + public ReconChange( + CellChange[] cellChanges, + String commonColumnName, + ReconConfig newReconConfig, + ReconStats newReconStats // can be null + ) { + super(cellChanges, commonColumnName, false); + _newReconConfig = newReconConfig; + _newReconStats = newReconStats; + } + + public ReconChange( + CellChange cellChange, + String commonColumnName, + ReconConfig newReconConfig, + ReconStats newReconStats // can be null + ) { + super(cellChange, commonColumnName, false); + _newReconConfig = newReconConfig; + _newReconStats = newReconStats; + } + + @Override + public void apply(Project project) { + synchronized (project) { + super.apply(project); + + Column column = project.columnModel.getColumnByName(_commonColumnName); + + if (_newReconStats == null) { + _newReconStats = ReconStats.create(project, column.getCellIndex()); + } + + _oldReconConfig = column.getReconConfig(); + _oldReconStats = column.getReconStats(); + + column.setReconConfig(_newReconConfig); + column.setReconStats(_newReconStats); + + column.clearPrecomputes(); + } + } + + @Override + public void revert(Project project) { + synchronized (project) { + super.revert(project); + + Column column = project.columnModel.getColumnByName(_commonColumnName); + column.setReconConfig(_oldReconConfig); + column.setReconStats(_oldReconStats); + + column.clearPrecomputes(); + } + } + + public void save(Writer writer, Properties options) throws IOException { + writer.write("newReconConfig="); + if (_newReconConfig != null) { + _newReconConfig.save(writer); + } + writer.write('\n'); + + writer.write("newReconStats="); + if (_newReconStats != null) { + _newReconStats.save(writer); + } + writer.write('\n'); + + writer.write("oldReconConfig="); + if (_oldReconConfig != null) { + _oldReconConfig.save(writer); + } + writer.write('\n'); + + writer.write("oldReconStats="); + if (_oldReconStats != null) { + _oldReconStats.save(writer); + } + writer.write('\n'); + + super.save(writer, options); + } + + static public Change load(LineNumberReader reader, Pool pool) throws Exception { + ReconConfig newReconConfig = null; + ReconStats newReconStats = null; + ReconConfig oldReconConfig = null; + ReconStats oldReconStats = null; + + String commonColumnName = null; + CellChange[] cellChanges = null; + + String line; + while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { + int equal = line.indexOf('='); + + CharSequence field = line.subSequence(0, equal); + String value = line.substring(equal + 1); + + if ("newReconConfig".equals(field)) { + if (value.length() > 0) { + newReconConfig = ReconConfig.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); + } + } else if ("newReconStats".equals(field)) { + if (value.length() > 0) { + newReconStats = ReconStats.load(ParsingUtilities.evaluateJsonStringToObject(value)); + } + } else if ("oldReconConfig".equals(field)) { + if (value.length() > 0) { + oldReconConfig = ReconConfig.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); + } + } else if ("oldReconStats".equals(field)) { + if (value.length() > 0) { + oldReconStats = ReconStats.load(ParsingUtilities.evaluateJsonStringToObject(value)); + } + } else if ("commonColumnName".equals(field)) { + commonColumnName = value; + } else if ("cellChangeCount".equals(field)) { + int cellChangeCount = Integer.parseInt(value); + + cellChanges = new CellChange[cellChangeCount]; + for (int i = 0; i < cellChangeCount; i++) { + cellChanges[i] = CellChange.load(reader, pool); + } + } + } + + ReconChange change = new ReconChange( + cellChanges, commonColumnName, newReconConfig, newReconStats); + + change._oldReconConfig = oldReconConfig; + change._oldReconStats = oldReconStats; + + return change; + } } \ No newline at end of file diff --git a/src/main/java/com/metaweb/gridworks/model/recon/DataExtensionReconConfig.java b/src/main/java/com/metaweb/gridworks/model/recon/DataExtensionReconConfig.java index c0315fa3b..6591416de 100644 --- a/src/main/java/com/metaweb/gridworks/model/recon/DataExtensionReconConfig.java +++ b/src/main/java/com/metaweb/gridworks/model/recon/DataExtensionReconConfig.java @@ -1,63 +1,63 @@ -package com.metaweb.gridworks.model.recon; - -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.protograph.FreebaseType; - -public class DataExtensionReconConfig extends StrictReconConfig { - final public FreebaseType type; - - private final static String WARN = "Not implemented"; - - static public ReconConfig reconstruct(JSONObject obj) throws Exception { - JSONObject type = obj.getJSONObject("type"); - - return new DataExtensionReconConfig( - new FreebaseType( - type.getString("id"), - type.getString("name") - ) - ); - } - - public DataExtensionReconConfig(FreebaseType type) { - this.type = type; - } - - @Override - public ReconJob createJob(Project project, int rowIndex, Row row, - String columnName, Cell cell) { - throw new RuntimeException(WARN); - } - - @Override - public int getBatchSize() { - throw new RuntimeException(WARN); - } - - public void write(JSONWriter writer, Properties options) throws JSONException { - writer.object(); - writer.key("mode"); writer.value("extend"); - writer.key("type"); type.write(writer, options); - writer.endObject(); - } - - @Override - public List batchRecon(List jobs, long historyEntryID) { - throw new RuntimeException(WARN); - } - - @Override - public String getBriefDescription(Project project, String columnName) { - throw new RuntimeException(WARN); - } -} +package com.metaweb.gridworks.model.recon; + +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.protograph.FreebaseType; + +public class DataExtensionReconConfig extends StrictReconConfig { + final public FreebaseType type; + + private final static String WARN = "Not implemented"; + + static public ReconConfig reconstruct(JSONObject obj) throws Exception { + JSONObject type = obj.getJSONObject("type"); + + return new DataExtensionReconConfig( + new FreebaseType( + type.getString("id"), + type.getString("name") + ) + ); + } + + public DataExtensionReconConfig(FreebaseType type) { + this.type = type; + } + + @Override + public ReconJob createJob(Project project, int rowIndex, Row row, + String columnName, Cell cell) { + throw new RuntimeException(WARN); + } + + @Override + public int getBatchSize() { + throw new RuntimeException(WARN); + } + + public void write(JSONWriter writer, Properties options) throws JSONException { + writer.object(); + writer.key("mode"); writer.value("extend"); + writer.key("type"); type.write(writer, options); + writer.endObject(); + } + + @Override + public List batchRecon(List jobs, long historyEntryID) { + throw new RuntimeException(WARN); + } + + @Override + public String getBriefDescription(Project project, String columnName) { + throw new RuntimeException(WARN); + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/recon/GuidBasedReconConfig.java b/src/main/java/com/metaweb/gridworks/model/recon/GuidBasedReconConfig.java index 86e729cab..bcac3a03f 100644 --- a/src/main/java/com/metaweb/gridworks/model/recon/GuidBasedReconConfig.java +++ b/src/main/java/com/metaweb/gridworks/model/recon/GuidBasedReconConfig.java @@ -1,175 +1,175 @@ -package com.metaweb.gridworks.model.recon; - -import java.io.InputStream; -import java.io.StringWriter; -import java.net.URL; -import java.net.URLConnection; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class GuidBasedReconConfig extends StrictReconConfig { - static public ReconConfig reconstruct(JSONObject obj) throws Exception { - return new GuidBasedReconConfig(); - } - - public GuidBasedReconConfig() { - } - - static protected class GuidBasedReconJob extends ReconJob { - String guid; - - public int getKey() { - return guid.hashCode(); - } - } - - @Override - public ReconJob createJob(Project project, int rowIndex, Row row, - String columnName, Cell cell) { - - GuidBasedReconJob job = new GuidBasedReconJob(); - String s = cell.value.toString(); - - if (s.startsWith("/guid/")) { - s = "#" + s.substring(6); - } else if (!s.startsWith("#")) { - s = "#" + s; - } - - job.guid = s; - - return job; - } - - @Override - public int getBatchSize() { - return 10; - } - - @Override - public String getBriefDescription(Project project, String columnName) { - return "Reconcile cells in column " + columnName + " as Freebase IDs"; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("mode"); writer.value("strict"); - writer.key("match"); writer.value("id"); - writer.endObject(); - } - - @Override - public List batchRecon(List jobs, long historyEntryID) { - List recons = new ArrayList(jobs.size()); - Map guidToRecon = new HashMap(); - - try { - String query = null; - { - StringWriter stringWriter = new StringWriter(); - JSONWriter jsonWriter = new JSONWriter(stringWriter); - - jsonWriter.object(); - jsonWriter.key("query"); - jsonWriter.array(); - jsonWriter.object(); - - jsonWriter.key("id"); jsonWriter.value(null); - jsonWriter.key("name"); jsonWriter.value(null); - jsonWriter.key("guid"); jsonWriter.value(null); - jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); - - jsonWriter.key("guid|="); - jsonWriter.array(); - for (ReconJob job : jobs) { - jsonWriter.value(((GuidBasedReconJob) job).guid); - } - jsonWriter.endArray(); - - jsonWriter.endObject(); - jsonWriter.endArray(); - jsonWriter.endObject(); - - query = stringWriter.toString(); - } - - StringBuffer sb = new StringBuffer(1024); - sb.append(s_mqlreadService); - sb.append("?query="); - sb.append(ParsingUtilities.encode(query)); - - URL url = new URL(sb.toString()); - URLConnection connection = url.openConnection(); - connection.setConnectTimeout(5000); - connection.connect(); - - InputStream is = connection.getInputStream(); - try { - String s = ParsingUtilities.inputStreamToString(is); - JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); - JSONArray results = o.getJSONArray("result"); - int count = results.length(); - - for (int i = 0; i < count; i++) { - JSONObject result = results.getJSONObject(i); - - String guid = result.getString("guid"); - - JSONArray types = result.getJSONArray("type"); - String[] typeIDs = new String[types.length()]; - for (int j = 0; j < typeIDs.length; j++) { - typeIDs[j] = types.getString(j); - } - - ReconCandidate candidate = new ReconCandidate( - result.getString("id"), - guid, - result.getString("name"), - typeIDs, - 100 - ); - - Recon recon = new Recon(historyEntryID); - recon.addCandidate(candidate); - recon.service = "mql"; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - recon.match = candidate; - recon.matchRank = 0; - - guidToRecon.put(guid, recon); - } - } finally { - is.close(); - } - } catch (Exception e) { - e.printStackTrace(); - } - - for (int i = 0; i < jobs.size(); i++) { - String guid = ((GuidBasedReconJob) jobs.get(i)).guid; - Recon recon = guidToRecon.get(guid); - recons.add(recon); - } - - return recons; - } -} +package com.metaweb.gridworks.model.recon; + +import java.io.InputStream; +import java.io.StringWriter; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class GuidBasedReconConfig extends StrictReconConfig { + static public ReconConfig reconstruct(JSONObject obj) throws Exception { + return new GuidBasedReconConfig(); + } + + public GuidBasedReconConfig() { + } + + static protected class GuidBasedReconJob extends ReconJob { + String guid; + + public int getKey() { + return guid.hashCode(); + } + } + + @Override + public ReconJob createJob(Project project, int rowIndex, Row row, + String columnName, Cell cell) { + + GuidBasedReconJob job = new GuidBasedReconJob(); + String s = cell.value.toString(); + + if (s.startsWith("/guid/")) { + s = "#" + s.substring(6); + } else if (!s.startsWith("#")) { + s = "#" + s; + } + + job.guid = s; + + return job; + } + + @Override + public int getBatchSize() { + return 10; + } + + @Override + public String getBriefDescription(Project project, String columnName) { + return "Reconcile cells in column " + columnName + " as Freebase IDs"; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("mode"); writer.value("strict"); + writer.key("match"); writer.value("id"); + writer.endObject(); + } + + @Override + public List batchRecon(List jobs, long historyEntryID) { + List recons = new ArrayList(jobs.size()); + Map guidToRecon = new HashMap(); + + try { + String query = null; + { + StringWriter stringWriter = new StringWriter(); + JSONWriter jsonWriter = new JSONWriter(stringWriter); + + jsonWriter.object(); + jsonWriter.key("query"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("id"); jsonWriter.value(null); + jsonWriter.key("name"); jsonWriter.value(null); + jsonWriter.key("guid"); jsonWriter.value(null); + jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); + + jsonWriter.key("guid|="); + jsonWriter.array(); + for (ReconJob job : jobs) { + jsonWriter.value(((GuidBasedReconJob) job).guid); + } + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + jsonWriter.endObject(); + + query = stringWriter.toString(); + } + + StringBuffer sb = new StringBuffer(1024); + sb.append(s_mqlreadService); + sb.append("?query="); + sb.append(ParsingUtilities.encode(query)); + + URL url = new URL(sb.toString()); + URLConnection connection = url.openConnection(); + connection.setConnectTimeout(5000); + connection.connect(); + + InputStream is = connection.getInputStream(); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + JSONArray results = o.getJSONArray("result"); + int count = results.length(); + + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); + + String guid = result.getString("guid"); + + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); + } + + ReconCandidate candidate = new ReconCandidate( + result.getString("id"), + guid, + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = new Recon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + guidToRecon.put(guid, recon); + } + } finally { + is.close(); + } + } catch (Exception e) { + e.printStackTrace(); + } + + for (int i = 0; i < jobs.size(); i++) { + String guid = ((GuidBasedReconJob) jobs.get(i)).guid; + Recon recon = guidToRecon.get(guid); + recons.add(recon); + } + + return recons; + } +} diff --git a/src/main/java/com/metaweb/gridworks/model/recon/HeuristicReconConfig.java b/src/main/java/com/metaweb/gridworks/model/recon/HeuristicReconConfig.java index ccf9b48f9..15a97f4b5 100644 --- a/src/main/java/com/metaweb/gridworks/model/recon/HeuristicReconConfig.java +++ b/src/main/java/com/metaweb/gridworks/model/recon/HeuristicReconConfig.java @@ -300,8 +300,8 @@ public class HeuristicReconConfig extends ReconConfig { } if (count > 0) { - ReconCandidate candidate = recon.candidates.get(0); - + ReconCandidate candidate = recon.candidates.get(0); + recon.setFeature(Recon.Feature_nameMatch, text.equalsIgnoreCase(candidate.topicName)); recon.setFeature(Recon.Feature_nameLevenshtein, StringUtils.getLevenshteinDistance(text, candidate.topicName)); recon.setFeature(Recon.Feature_nameWordDistance, wordDistance(text, candidate.topicName)); @@ -311,14 +311,14 @@ public class HeuristicReconConfig extends ReconConfig { if (this.typeID.equals(typeID)) { recon.setFeature(Recon.Feature_typeMatch, true); if (autoMatch && candidate.score >= 100) { - if (count == 1 || - candidate.score / recon.candidates.get(1).score >= 1.5) { - - recon.match = candidate; - recon.matchRank = 0; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - } + if (count == 1 || + candidate.score / recon.candidates.get(1).score >= 1.5) { + + recon.match = candidate; + recon.matchRank = 0; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + } } break; } diff --git a/src/main/java/com/metaweb/gridworks/model/recon/IdBasedReconConfig.java b/src/main/java/com/metaweb/gridworks/model/recon/IdBasedReconConfig.java index a42c9c26c..22b1d9e3b 100644 --- a/src/main/java/com/metaweb/gridworks/model/recon/IdBasedReconConfig.java +++ b/src/main/java/com/metaweb/gridworks/model/recon/IdBasedReconConfig.java @@ -1,180 +1,180 @@ -package com.metaweb.gridworks.model.recon; - -import java.io.InputStream; -import java.io.StringWriter; -import java.net.URL; -import java.net.URLConnection; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class IdBasedReconConfig extends StrictReconConfig { - static public ReconConfig reconstruct(JSONObject obj) throws Exception { - return new IdBasedReconConfig(); - } - - public IdBasedReconConfig() { - } - - static protected class IdBasedReconJob extends ReconJob { - String id; - - public int getKey() { - return id.hashCode(); - } - } - - @Override - public ReconJob createJob(Project project, int rowIndex, Row row, - String columnName, Cell cell) { - - IdBasedReconJob job = new IdBasedReconJob(); - String s = cell.value.toString(); - - if (!s.startsWith("/")) { - if (s.startsWith("92")) { - s = "/guid/" + s; - } else if (!s.contains("/")){ - s = "/en/" + s; - } else { - s = "/" + s; - } - } - - job.id = s; - - return job; - } - - @Override - public int getBatchSize() { - return 10; - } - - @Override - public String getBriefDescription(Project project, String columnName) { - return "Reconcile cells in column " + columnName + " as Freebase IDs"; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("mode"); writer.value("strict"); - writer.key("match"); writer.value("id"); - writer.endObject(); - } - - @Override - public List batchRecon(List jobs, long historyEntryID) { - List recons = new ArrayList(jobs.size()); - Map idToRecon = new HashMap(); - - try { - String query = null; - { - StringWriter stringWriter = new StringWriter(); - JSONWriter jsonWriter = new JSONWriter(stringWriter); - - jsonWriter.object(); - jsonWriter.key("query"); - jsonWriter.array(); - jsonWriter.object(); - - jsonWriter.key("id"); jsonWriter.value(null); - jsonWriter.key("name"); jsonWriter.value(null); - jsonWriter.key("guid"); jsonWriter.value(null); - jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); - - jsonWriter.key("id|="); - jsonWriter.array(); - for (ReconJob job : jobs) { - jsonWriter.value(((IdBasedReconJob) job).id); - } - jsonWriter.endArray(); - - jsonWriter.endObject(); - jsonWriter.endArray(); - jsonWriter.endObject(); - - query = stringWriter.toString(); - } - - StringBuffer sb = new StringBuffer(1024); - sb.append(s_mqlreadService); - sb.append("?query="); - sb.append(ParsingUtilities.encode(query)); - - URL url = new URL(sb.toString()); - URLConnection connection = url.openConnection(); - connection.setConnectTimeout(5000); - connection.connect(); - - InputStream is = connection.getInputStream(); - try { - String s = ParsingUtilities.inputStreamToString(is); - JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); - JSONArray results = o.getJSONArray("result"); - int count = results.length(); - - for (int i = 0; i < count; i++) { - JSONObject result = results.getJSONObject(i); - - String id = result.getString("id"); - - JSONArray types = result.getJSONArray("type"); - String[] typeIDs = new String[types.length()]; - for (int j = 0; j < typeIDs.length; j++) { - typeIDs[j] = types.getString(j); - } - - ReconCandidate candidate = new ReconCandidate( - id, - result.getString("guid"), - result.getString("name"), - typeIDs, - 100 - ); - - Recon recon = new Recon(historyEntryID); - recon.addCandidate(candidate); - recon.service = "mql"; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - recon.match = candidate; - recon.matchRank = 0; - - idToRecon.put(id, recon); - } - } finally { - is.close(); - } - } catch (Exception e) { - e.printStackTrace(); - } - - for (int i = 0; i < jobs.size(); i++) { - String id = ((IdBasedReconJob) jobs.get(i)).id; - Recon recon = idToRecon.get(id); - recons.add(recon); - } - - return recons; - } - -} +package com.metaweb.gridworks.model.recon; + +import java.io.InputStream; +import java.io.StringWriter; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class IdBasedReconConfig extends StrictReconConfig { + static public ReconConfig reconstruct(JSONObject obj) throws Exception { + return new IdBasedReconConfig(); + } + + public IdBasedReconConfig() { + } + + static protected class IdBasedReconJob extends ReconJob { + String id; + + public int getKey() { + return id.hashCode(); + } + } + + @Override + public ReconJob createJob(Project project, int rowIndex, Row row, + String columnName, Cell cell) { + + IdBasedReconJob job = new IdBasedReconJob(); + String s = cell.value.toString(); + + if (!s.startsWith("/")) { + if (s.startsWith("92")) { + s = "/guid/" + s; + } else if (!s.contains("/")){ + s = "/en/" + s; + } else { + s = "/" + s; + } + } + + job.id = s; + + return job; + } + + @Override + public int getBatchSize() { + return 10; + } + + @Override + public String getBriefDescription(Project project, String columnName) { + return "Reconcile cells in column " + columnName + " as Freebase IDs"; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("mode"); writer.value("strict"); + writer.key("match"); writer.value("id"); + writer.endObject(); + } + + @Override + public List batchRecon(List jobs, long historyEntryID) { + List recons = new ArrayList(jobs.size()); + Map idToRecon = new HashMap(); + + try { + String query = null; + { + StringWriter stringWriter = new StringWriter(); + JSONWriter jsonWriter = new JSONWriter(stringWriter); + + jsonWriter.object(); + jsonWriter.key("query"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("id"); jsonWriter.value(null); + jsonWriter.key("name"); jsonWriter.value(null); + jsonWriter.key("guid"); jsonWriter.value(null); + jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); + + jsonWriter.key("id|="); + jsonWriter.array(); + for (ReconJob job : jobs) { + jsonWriter.value(((IdBasedReconJob) job).id); + } + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + jsonWriter.endObject(); + + query = stringWriter.toString(); + } + + StringBuffer sb = new StringBuffer(1024); + sb.append(s_mqlreadService); + sb.append("?query="); + sb.append(ParsingUtilities.encode(query)); + + URL url = new URL(sb.toString()); + URLConnection connection = url.openConnection(); + connection.setConnectTimeout(5000); + connection.connect(); + + InputStream is = connection.getInputStream(); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + JSONArray results = o.getJSONArray("result"); + int count = results.length(); + + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); + + String id = result.getString("id"); + + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); + } + + ReconCandidate candidate = new ReconCandidate( + id, + result.getString("guid"), + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = new Recon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + idToRecon.put(id, recon); + } + } finally { + is.close(); + } + } catch (Exception e) { + e.printStackTrace(); + } + + for (int i = 0; i < jobs.size(); i++) { + String id = ((IdBasedReconJob) jobs.get(i)).id; + Recon recon = idToRecon.get(id); + recons.add(recon); + } + + return recons; + } + +} diff --git a/src/main/java/com/metaweb/gridworks/model/recon/KeyBasedReconConfig.java b/src/main/java/com/metaweb/gridworks/model/recon/KeyBasedReconConfig.java index 4de94a1e4..1a3785d59 100644 --- a/src/main/java/com/metaweb/gridworks/model/recon/KeyBasedReconConfig.java +++ b/src/main/java/com/metaweb/gridworks/model/recon/KeyBasedReconConfig.java @@ -1,194 +1,194 @@ -package com.metaweb.gridworks.model.recon; - -import java.io.InputStream; -import java.io.StringWriter; -import java.net.URL; -import java.net.URLConnection; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.protograph.FreebaseTopic; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class KeyBasedReconConfig extends StrictReconConfig { - final public FreebaseTopic namespace; - - static public ReconConfig reconstruct(JSONObject obj) throws Exception { - JSONObject ns = obj.getJSONObject("namespace"); - - return new KeyBasedReconConfig( - new FreebaseTopic( - ns.getString("id"), - ns.getString("name") - ) - ); - } - - public KeyBasedReconConfig(FreebaseTopic namespace) { - this.namespace = namespace; - } - - static protected class KeyBasedReconJob extends ReconJob { - String key; - - public int getKey() { - return key.hashCode(); - } - } - - @Override - public ReconJob createJob(Project project, int rowIndex, Row row, - String columnName, Cell cell) { - - KeyBasedReconJob job = new KeyBasedReconJob(); - - job.key = cell.value.toString().replace(' ', '_'); - - return job; - } - - @Override - public int getBatchSize() { - return 10; - } - - @Override - public String getBriefDescription(Project project, String columnName) { - return "Reconcile cells in column " + columnName + " to topics with keys in namespace " + namespace.id; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("mode"); writer.value("strict"); - writer.key("match"); writer.value("key"); - writer.key("namespace"); namespace.write(writer, options); - writer.endObject(); - } - - @Override - public List batchRecon(List jobs, long historyEntryID) { - List recons = new ArrayList(jobs.size()); - Map keyToRecon = new HashMap(); - - try { - String query = null; - { - StringWriter stringWriter = new StringWriter(); - JSONWriter jsonWriter = new JSONWriter(stringWriter); - - jsonWriter.object(); - jsonWriter.key("query"); - jsonWriter.array(); - jsonWriter.object(); - - jsonWriter.key("id"); jsonWriter.value(null); - jsonWriter.key("name"); jsonWriter.value(null); - jsonWriter.key("guid"); jsonWriter.value(null); - jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); - - jsonWriter.key("key"); - jsonWriter.array(); - jsonWriter.object(); - - jsonWriter.key("namespace"); - jsonWriter.object(); - jsonWriter.key("id"); jsonWriter.value(namespace.id); - jsonWriter.endObject(); - - jsonWriter.key("value"); jsonWriter.value(null); - jsonWriter.key("value|="); - jsonWriter.array(); - for (ReconJob job : jobs) { - jsonWriter.value(((KeyBasedReconJob) job).key); - } - jsonWriter.endArray(); - - jsonWriter.endObject(); - jsonWriter.endArray(); - - jsonWriter.endObject(); - jsonWriter.endArray(); - jsonWriter.endObject(); - - query = stringWriter.toString(); - } - - StringBuffer sb = new StringBuffer(1024); - sb.append(s_mqlreadService); - sb.append("?query="); - sb.append(ParsingUtilities.encode(query)); - - URL url = new URL(sb.toString()); - URLConnection connection = url.openConnection(); - connection.setConnectTimeout(5000); - connection.connect(); - - InputStream is = connection.getInputStream(); - try { - String s = ParsingUtilities.inputStreamToString(is); - JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); - JSONArray results = o.getJSONArray("result"); - int count = results.length(); - - for (int i = 0; i < count; i++) { - JSONObject result = results.getJSONObject(i); - - String key = result.getJSONArray("key").getJSONObject(0).getString("value"); - - JSONArray types = result.getJSONArray("type"); - String[] typeIDs = new String[types.length()]; - for (int j = 0; j < typeIDs.length; j++) { - typeIDs[j] = types.getString(j); - } - - ReconCandidate candidate = new ReconCandidate( - result.getString("id"), - result.getString("guid"), - result.getString("name"), - typeIDs, - 100 - ); - - Recon recon = new Recon(historyEntryID); - recon.addCandidate(candidate); - recon.service = "mql"; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - recon.match = candidate; - recon.matchRank = 0; - - keyToRecon.put(key, recon); - } - } finally { - is.close(); - } - } catch (Exception e) { - e.printStackTrace(); - } - - for (int i = 0; i < jobs.size(); i++) { - String key = ((KeyBasedReconJob) jobs.get(i)).key; - Recon recon = keyToRecon.get(key); - recons.add(recon); - } - - return recons; - } - -} +package com.metaweb.gridworks.model.recon; + +import java.io.InputStream; +import java.io.StringWriter; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.protograph.FreebaseTopic; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class KeyBasedReconConfig extends StrictReconConfig { + final public FreebaseTopic namespace; + + static public ReconConfig reconstruct(JSONObject obj) throws Exception { + JSONObject ns = obj.getJSONObject("namespace"); + + return new KeyBasedReconConfig( + new FreebaseTopic( + ns.getString("id"), + ns.getString("name") + ) + ); + } + + public KeyBasedReconConfig(FreebaseTopic namespace) { + this.namespace = namespace; + } + + static protected class KeyBasedReconJob extends ReconJob { + String key; + + public int getKey() { + return key.hashCode(); + } + } + + @Override + public ReconJob createJob(Project project, int rowIndex, Row row, + String columnName, Cell cell) { + + KeyBasedReconJob job = new KeyBasedReconJob(); + + job.key = cell.value.toString().replace(' ', '_'); + + return job; + } + + @Override + public int getBatchSize() { + return 10; + } + + @Override + public String getBriefDescription(Project project, String columnName) { + return "Reconcile cells in column " + columnName + " to topics with keys in namespace " + namespace.id; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("mode"); writer.value("strict"); + writer.key("match"); writer.value("key"); + writer.key("namespace"); namespace.write(writer, options); + writer.endObject(); + } + + @Override + public List batchRecon(List jobs, long historyEntryID) { + List recons = new ArrayList(jobs.size()); + Map keyToRecon = new HashMap(); + + try { + String query = null; + { + StringWriter stringWriter = new StringWriter(); + JSONWriter jsonWriter = new JSONWriter(stringWriter); + + jsonWriter.object(); + jsonWriter.key("query"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("id"); jsonWriter.value(null); + jsonWriter.key("name"); jsonWriter.value(null); + jsonWriter.key("guid"); jsonWriter.value(null); + jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); + + jsonWriter.key("key"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("namespace"); + jsonWriter.object(); + jsonWriter.key("id"); jsonWriter.value(namespace.id); + jsonWriter.endObject(); + + jsonWriter.key("value"); jsonWriter.value(null); + jsonWriter.key("value|="); + jsonWriter.array(); + for (ReconJob job : jobs) { + jsonWriter.value(((KeyBasedReconJob) job).key); + } + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + jsonWriter.endObject(); + + query = stringWriter.toString(); + } + + StringBuffer sb = new StringBuffer(1024); + sb.append(s_mqlreadService); + sb.append("?query="); + sb.append(ParsingUtilities.encode(query)); + + URL url = new URL(sb.toString()); + URLConnection connection = url.openConnection(); + connection.setConnectTimeout(5000); + connection.connect(); + + InputStream is = connection.getInputStream(); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + JSONArray results = o.getJSONArray("result"); + int count = results.length(); + + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); + + String key = result.getJSONArray("key").getJSONObject(0).getString("value"); + + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); + } + + ReconCandidate candidate = new ReconCandidate( + result.getString("id"), + result.getString("guid"), + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = new Recon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + keyToRecon.put(key, recon); + } + } finally { + is.close(); + } + } catch (Exception e) { + e.printStackTrace(); + } + + for (int i = 0; i < jobs.size(); i++) { + String key = ((KeyBasedReconJob) jobs.get(i)).key; + Recon recon = keyToRecon.get(key); + recons.add(recon); + } + + return recons; + } + +} diff --git a/src/main/java/com/metaweb/gridworks/model/recon/ReconJob.java b/src/main/java/com/metaweb/gridworks/model/recon/ReconJob.java index 64111138f..ae1d747e6 100644 --- a/src/main/java/com/metaweb/gridworks/model/recon/ReconJob.java +++ b/src/main/java/com/metaweb/gridworks/model/recon/ReconJob.java @@ -1,5 +1,5 @@ -package com.metaweb.gridworks.model.recon; - -abstract public class ReconJob { - abstract public int getKey(); -} +package com.metaweb.gridworks.model.recon; + +abstract public class ReconJob { + abstract public int getKey(); +} diff --git a/src/main/java/com/metaweb/gridworks/operations/ColumnAdditionOperation.java b/src/main/java/com/metaweb/gridworks/operations/ColumnAdditionOperation.java index 902c9a7fc..acfd29922 100644 --- a/src/main/java/com/metaweb/gridworks/operations/ColumnAdditionOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/ColumnAdditionOperation.java @@ -1,174 +1,174 @@ -package com.metaweb.gridworks.operations; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.expr.MetaParser; -import com.metaweb.gridworks.expr.WrappedCell; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.CellAtRow; -import com.metaweb.gridworks.model.changes.ColumnAdditionChange; - -public class ColumnAdditionOperation extends EngineDependentOperation { - final protected String _baseColumnName; - final protected String _expression; - final protected OnError _onError; - - final protected String _newColumnName; - final protected int _columnInsertIndex; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - - return new ColumnAdditionOperation( - engineConfig, - obj.getString("baseColumnName"), - obj.getString("expression"), - TextTransformOperation.stringToOnError(obj.getString("onError")), - obj.getString("newColumnName"), - obj.getInt("columnInsertIndex") - ); - } - - public ColumnAdditionOperation( - JSONObject engineConfig, - String baseColumnName, - String expression, - OnError onError, - String newColumnName, - int columnInsertIndex - ) { - super(engineConfig); - - _baseColumnName = baseColumnName; - _expression = expression; - _onError = onError; - - _newColumnName = newColumnName; - _columnInsertIndex = columnInsertIndex; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("newColumnName"); writer.value(_newColumnName); - writer.key("columnInsertIndex"); writer.value(_columnInsertIndex); - writer.key("baseColumnName"); writer.value(_baseColumnName); - writer.key("expression"); writer.value(_expression); - writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError)); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Create column " + _newColumnName + - " at index " + _columnInsertIndex + - " based on column " + _baseColumnName + - " using expression " + _expression; - } - - protected String createDescription(Column column, List cellsAtRows) { - return "Create new column " + _newColumnName + - " based on column " + column.getName() + - " by filling " + cellsAtRows.size() + - " rows with " + _expression; - } - - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - Engine engine = createEngine(project); - - Column column = project.columnModel.getColumnByName(_baseColumnName); - if (column == null) { - throw new Exception("No column named " + _baseColumnName); - } - - List cellsAtRows = new ArrayList(project.rows.size()); - - FilteredRows filteredRows = engine.getAllFilteredRows(false); - filteredRows.accept(project, createRowVisitor(project, cellsAtRows)); - - String description = createDescription(column, cellsAtRows); - - Change change = new ColumnAdditionChange(_newColumnName, _columnInsertIndex, cellsAtRows); - - return new HistoryEntry( - historyEntryID, project, description, this, change); - } - - protected RowVisitor createRowVisitor(Project project, List cellsAtRows) throws Exception { - Column column = project.columnModel.getColumnByName(_baseColumnName); - - Evaluable eval = MetaParser.parse(_expression); - Properties bindings = ExpressionUtils.createBindings(project); - - return new RowVisitor() { - int cellIndex; - Properties bindings; - List cellsAtRows; - Evaluable eval; - - public RowVisitor init(int cellIndex, Properties bindings, List cellsAtRows, Evaluable eval) { - this.cellIndex = cellIndex; - this.bindings = bindings; - this.cellsAtRows = cellsAtRows; - this.eval = eval; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - Cell cell = row.getCell(cellIndex); - Cell newCell = null; - - ExpressionUtils.bind(bindings, row, rowIndex, _baseColumnName, cell); - - Object o = eval.evaluate(bindings); - if (o != null) { - if (o instanceof Cell) { - newCell = (Cell) o; - } else if (o instanceof WrappedCell) { - newCell = ((WrappedCell) o).cell; - } else { - Serializable v = ExpressionUtils.wrapStorable(o); - if (ExpressionUtils.isError(v)) { - if (_onError == OnError.SetToBlank) { - return false; - } else if (_onError == OnError.KeepOriginal) { - v = cell != null ? cell.value : null; - } - } - - if (v != null) { - newCell = new Cell(v, null); - } - } - } - - if (newCell != null) { - cellsAtRows.add(new CellAtRow(rowIndex, newCell)); - } - - return false; - } - }.init(column.getCellIndex(), bindings, cellsAtRows, eval); - } -} +package com.metaweb.gridworks.operations; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.expr.MetaParser; +import com.metaweb.gridworks.expr.WrappedCell; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.CellAtRow; +import com.metaweb.gridworks.model.changes.ColumnAdditionChange; + +public class ColumnAdditionOperation extends EngineDependentOperation { + final protected String _baseColumnName; + final protected String _expression; + final protected OnError _onError; + + final protected String _newColumnName; + final protected int _columnInsertIndex; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + + return new ColumnAdditionOperation( + engineConfig, + obj.getString("baseColumnName"), + obj.getString("expression"), + TextTransformOperation.stringToOnError(obj.getString("onError")), + obj.getString("newColumnName"), + obj.getInt("columnInsertIndex") + ); + } + + public ColumnAdditionOperation( + JSONObject engineConfig, + String baseColumnName, + String expression, + OnError onError, + String newColumnName, + int columnInsertIndex + ) { + super(engineConfig); + + _baseColumnName = baseColumnName; + _expression = expression; + _onError = onError; + + _newColumnName = newColumnName; + _columnInsertIndex = columnInsertIndex; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("newColumnName"); writer.value(_newColumnName); + writer.key("columnInsertIndex"); writer.value(_columnInsertIndex); + writer.key("baseColumnName"); writer.value(_baseColumnName); + writer.key("expression"); writer.value(_expression); + writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError)); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Create column " + _newColumnName + + " at index " + _columnInsertIndex + + " based on column " + _baseColumnName + + " using expression " + _expression; + } + + protected String createDescription(Column column, List cellsAtRows) { + return "Create new column " + _newColumnName + + " based on column " + column.getName() + + " by filling " + cellsAtRows.size() + + " rows with " + _expression; + } + + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + Engine engine = createEngine(project); + + Column column = project.columnModel.getColumnByName(_baseColumnName); + if (column == null) { + throw new Exception("No column named " + _baseColumnName); + } + + List cellsAtRows = new ArrayList(project.rows.size()); + + FilteredRows filteredRows = engine.getAllFilteredRows(false); + filteredRows.accept(project, createRowVisitor(project, cellsAtRows)); + + String description = createDescription(column, cellsAtRows); + + Change change = new ColumnAdditionChange(_newColumnName, _columnInsertIndex, cellsAtRows); + + return new HistoryEntry( + historyEntryID, project, description, this, change); + } + + protected RowVisitor createRowVisitor(Project project, List cellsAtRows) throws Exception { + Column column = project.columnModel.getColumnByName(_baseColumnName); + + Evaluable eval = MetaParser.parse(_expression); + Properties bindings = ExpressionUtils.createBindings(project); + + return new RowVisitor() { + int cellIndex; + Properties bindings; + List cellsAtRows; + Evaluable eval; + + public RowVisitor init(int cellIndex, Properties bindings, List cellsAtRows, Evaluable eval) { + this.cellIndex = cellIndex; + this.bindings = bindings; + this.cellsAtRows = cellsAtRows; + this.eval = eval; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + Cell cell = row.getCell(cellIndex); + Cell newCell = null; + + ExpressionUtils.bind(bindings, row, rowIndex, _baseColumnName, cell); + + Object o = eval.evaluate(bindings); + if (o != null) { + if (o instanceof Cell) { + newCell = (Cell) o; + } else if (o instanceof WrappedCell) { + newCell = ((WrappedCell) o).cell; + } else { + Serializable v = ExpressionUtils.wrapStorable(o); + if (ExpressionUtils.isError(v)) { + if (_onError == OnError.SetToBlank) { + return false; + } else if (_onError == OnError.KeepOriginal) { + v = cell != null ? cell.value : null; + } + } + + if (v != null) { + newCell = new Cell(v, null); + } + } + } + + if (newCell != null) { + cellsAtRows.add(new CellAtRow(rowIndex, newCell)); + } + + return false; + } + }.init(column.getCellIndex(), bindings, cellsAtRows, eval); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/ColumnRemovalOperation.java b/src/main/java/com/metaweb/gridworks/operations/ColumnRemovalOperation.java index 971106160..1c5e4ede6 100644 --- a/src/main/java/com/metaweb/gridworks/operations/ColumnRemovalOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/ColumnRemovalOperation.java @@ -1,58 +1,58 @@ -package com.metaweb.gridworks.operations; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.changes.ColumnRemovalChange; - -public class ColumnRemovalOperation extends AbstractOperation { - final protected String _columnName; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - return new ColumnRemovalOperation( - obj.getString("columnName") - ); - } - - public ColumnRemovalOperation( - String columnName - ) { - _columnName = columnName; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value("Remove column " + _columnName); - writer.key("columnName"); writer.value(_columnName); - writer.endObject(); - } - - - protected String getBriefDescription(Project project) { - return "Remove column " + _columnName; - } - - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - Column column = project.columnModel.getColumnByName(_columnName); - if (column == null) { - throw new Exception("No column named " + _columnName); - } - - String description = "Remove column " + column.getName(); - - Change change = new ColumnRemovalChange(project.columnModel.columns.indexOf(column)); - - return new HistoryEntry(historyEntryID, project, description, ColumnRemovalOperation.this, change); - } -} +package com.metaweb.gridworks.operations; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.changes.ColumnRemovalChange; + +public class ColumnRemovalOperation extends AbstractOperation { + final protected String _columnName; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + return new ColumnRemovalOperation( + obj.getString("columnName") + ); + } + + public ColumnRemovalOperation( + String columnName + ) { + _columnName = columnName; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value("Remove column " + _columnName); + writer.key("columnName"); writer.value(_columnName); + writer.endObject(); + } + + + protected String getBriefDescription(Project project) { + return "Remove column " + _columnName; + } + + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + Column column = project.columnModel.getColumnByName(_columnName); + if (column == null) { + throw new Exception("No column named " + _columnName); + } + + String description = "Remove column " + column.getName(); + + Change change = new ColumnRemovalChange(project.columnModel.columns.indexOf(column)); + + return new HistoryEntry(historyEntryID, project, description, ColumnRemovalOperation.this, change); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/ColumnRenameOperation.java b/src/main/java/com/metaweb/gridworks/operations/ColumnRenameOperation.java index 522e8dd9c..7107dc271 100644 --- a/src/main/java/com/metaweb/gridworks/operations/ColumnRenameOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/ColumnRenameOperation.java @@ -1,62 +1,62 @@ -package com.metaweb.gridworks.operations; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.changes.ColumnRenameChange; - -public class ColumnRenameOperation extends AbstractOperation { - final protected String _oldColumnName; - final protected String _newColumnName; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - return new ColumnRenameOperation( - obj.getString("oldColumnName"), - obj.getString("newColumnName") - ); - } - - public ColumnRenameOperation( - String oldColumnName, - String newColumnName - ) { - _oldColumnName = oldColumnName; - _newColumnName = newColumnName; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value("Rename column " + _oldColumnName + " to " + _newColumnName); - writer.key("oldColumnName"); writer.value(_oldColumnName); - writer.key("newColumnName"); writer.value(_newColumnName); - writer.endObject(); - } - - - protected String getBriefDescription(Project project) { - return "Rename column " + _oldColumnName + " to " + _newColumnName; - } - - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - if (project.columnModel.getColumnByName(_oldColumnName) == null) { - throw new Exception("No column named " + _oldColumnName); - } - if (project.columnModel.getColumnByName(_newColumnName) != null) { - throw new Exception("Another column already named " + _newColumnName); - } - - Change change = new ColumnRenameChange(_oldColumnName, _newColumnName); - - return new HistoryEntry(historyEntryID, project, getBriefDescription(null), ColumnRenameOperation.this, change); - } -} +package com.metaweb.gridworks.operations; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.changes.ColumnRenameChange; + +public class ColumnRenameOperation extends AbstractOperation { + final protected String _oldColumnName; + final protected String _newColumnName; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + return new ColumnRenameOperation( + obj.getString("oldColumnName"), + obj.getString("newColumnName") + ); + } + + public ColumnRenameOperation( + String oldColumnName, + String newColumnName + ) { + _oldColumnName = oldColumnName; + _newColumnName = newColumnName; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value("Rename column " + _oldColumnName + " to " + _newColumnName); + writer.key("oldColumnName"); writer.value(_oldColumnName); + writer.key("newColumnName"); writer.value(_newColumnName); + writer.endObject(); + } + + + protected String getBriefDescription(Project project) { + return "Rename column " + _oldColumnName + " to " + _newColumnName; + } + + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + if (project.columnModel.getColumnByName(_oldColumnName) == null) { + throw new Exception("No column named " + _oldColumnName); + } + if (project.columnModel.getColumnByName(_newColumnName) != null) { + throw new Exception("Another column already named " + _newColumnName); + } + + Change change = new ColumnRenameChange(_oldColumnName, _newColumnName); + + return new HistoryEntry(historyEntryID, project, getBriefDescription(null), ColumnRenameOperation.this, change); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/ColumnSplitOperation.java b/src/main/java/com/metaweb/gridworks/operations/ColumnSplitOperation.java index 365057875..13a06f5e4 100644 --- a/src/main/java/com/metaweb/gridworks/operations/ColumnSplitOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/ColumnSplitOperation.java @@ -1,281 +1,281 @@ -package com.metaweb.gridworks.operations; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import java.util.regex.Pattern; - -import org.apache.commons.lang.StringUtils; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import sun.reflect.generics.reflectiveObjects.NotImplementedException; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.importers.ImporterUtilities; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.ColumnSplitChange; -import com.metaweb.gridworks.util.JSONUtilities; - -public class ColumnSplitOperation extends EngineDependentOperation { - final protected String _columnName; - final protected boolean _guessCellType; - final protected boolean _removeOriginalColumn; - final protected String _mode; - - final protected String _separator; - final protected boolean _regex; - final protected int _maxColumns; - - final protected int[] _fieldLengths; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - String mode = obj.getString("mode"); - - if ("separator".equals(mode)) { - return new ColumnSplitOperation( - engineConfig, - obj.getString("columnName"), - obj.getBoolean("guessCellType"), - obj.getBoolean("removeOriginalColumn"), - obj.getString("separator"), - obj.getBoolean("regex"), - obj.getInt("maxColumns") - ); - } else { - return new ColumnSplitOperation( - engineConfig, - obj.getString("columnName"), - obj.getBoolean("guessCellType"), - obj.getBoolean("removeOriginalColumn"), - JSONUtilities.getIntArray(obj, "fieldLengths") - ); - } - } - - public ColumnSplitOperation( - JSONObject engineConfig, - String columnName, - boolean guessCellType, - boolean removeOriginalColumn, - String separator, - boolean regex, - int maxColumns - ) { - super(engineConfig); - - _columnName = columnName; - _guessCellType = guessCellType; - _removeOriginalColumn = removeOriginalColumn; - - _mode = "separator"; - _separator = separator; - _regex = regex; - _maxColumns = maxColumns; - - _fieldLengths = null; - } - - public ColumnSplitOperation( - JSONObject engineConfig, - String columnName, - boolean guessCellType, - boolean removeOriginalColumn, - int[] fieldLengths - ) { - super(engineConfig); - - _columnName = columnName; - _guessCellType = guessCellType; - _removeOriginalColumn = removeOriginalColumn; - - _mode = "lengths"; - _separator = null; - _regex = false; - _maxColumns = -1; - - _fieldLengths = fieldLengths; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("columnName"); writer.value(_columnName); - writer.key("guessCellType"); writer.value(_guessCellType); - writer.key("removeOriginalColumn"); writer.value(_removeOriginalColumn); - writer.key("mode"); writer.value(_mode); - if ("separator".equals(_mode)) { - writer.key("separator"); writer.value(_separator); - writer.key("regex"); writer.value(_regex); - writer.key("maxColumns"); writer.value(_maxColumns); - } else { - writer.key("fieldLengths"); writer.array(); - for (int l : _fieldLengths) { - writer.value(l); - } - writer.endArray(); - } - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Split column " + _columnName + - ("separator".equals(_mode) ? " by separator" : " by field lengths"); - } - - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - Engine engine = createEngine(project); - - Column column = project.columnModel.getColumnByName(_columnName); - if (column == null) { - throw new Exception("No column named " + _columnName); - } - - List columnNames = new ArrayList(); - List rowIndices = new ArrayList(project.rows.size()); - List> tuples = new ArrayList>(project.rows.size()); - - FilteredRows filteredRows = engine.getAllFilteredRows(false); - RowVisitor rowVisitor; - if ("lengths".equals(_mode)) { - rowVisitor = new ColumnSplitRowVisitor(project, column.getCellIndex(), columnNames, rowIndices, tuples) { - protected java.util.List split(String s) { - List results = new ArrayList(_fieldLengths.length + 1); - - int lastIndex = 0; - for (int i = 0; i < _fieldLengths.length; i++) { - int from = lastIndex; - int length = _fieldLengths[i]; - int to = Math.min(from + length, s.length()); - - results.add(stringToValue(s.substring(from, to))); - - lastIndex = to; - } - - return results; - }; - }; - } else if (_regex) { - Pattern pattern = Pattern.compile(_separator); - - rowVisitor = new ColumnSplitRowVisitor(project, column.getCellIndex(), columnNames, rowIndices, tuples) { - Pattern _pattern; - - protected java.util.List split(String s) { - return stringArrayToValueList(_pattern.split(s, _maxColumns)); - }; - - public RowVisitor init(Pattern pattern) { - _pattern = pattern; - return this; - } - }.init(pattern); - } else { - rowVisitor = new ColumnSplitRowVisitor(project, column.getCellIndex(), columnNames, rowIndices, tuples) { - protected java.util.List split(String s) { - return stringArrayToValueList( - StringUtils.splitByWholeSeparatorPreserveAllTokens(s, _separator, _maxColumns)); - }; - }; - } - - filteredRows.accept(project, rowVisitor); - - String description = - "Split " + rowIndices.size() + - " cell(s) in column " + _columnName + - " into several columns" + - ("separator".equals(_mode) ? " by separator" : " by field lengths"); - - Change change = new ColumnSplitChange( - _columnName, - columnNames, - rowIndices, - tuples, - _removeOriginalColumn - ); - - return new HistoryEntry( - historyEntryID, project, description, this, change); - } - - protected class ColumnSplitRowVisitor implements RowVisitor { - Project project; - int cellIndex; - List columnNames; - List rowIndices; - List> tuples; - - int columnNameIndex = 1; - - ColumnSplitRowVisitor( - Project project, - int cellIndex, - List columnNames, - List rowIndices, - List> tuples - ) { - this.project = project; - this.cellIndex = cellIndex; - this.columnNames = columnNames; - this.rowIndices = rowIndices; - this.tuples = tuples; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - Object value = row.getCellValue(cellIndex); - if (ExpressionUtils.isNonBlankData(value)) { - String s = value instanceof String ? ((String) value) : value.toString(); - - List tuple = split(s); - - rowIndices.add(rowIndex); - tuples.add(tuple); - - for (int i = columnNames.size(); i < tuple.size(); i++) { - while (true) { - String newColumnName = _columnName + " " + columnNameIndex++; - if (project.columnModel.getColumnByName(newColumnName) == null) { - columnNames.add(newColumnName); - break; - } - } - } - } - return false; - } - - protected List split(String s) { - throw new NotImplementedException(); - } - - protected Serializable stringToValue(String s) { - return _guessCellType ? ImporterUtilities.parseCellValue(s) : s; - } - - protected List stringArrayToValueList(String[] cells) { - List results = new ArrayList(cells.length); - for (String cell : cells) { - results.add(stringToValue(cell)); - } - - return results; - } - } -} +package com.metaweb.gridworks.operations; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.regex.Pattern; + +import org.apache.commons.lang.StringUtils; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.importers.ImporterUtilities; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.ColumnSplitChange; +import com.metaweb.gridworks.util.JSONUtilities; + +public class ColumnSplitOperation extends EngineDependentOperation { + final protected String _columnName; + final protected boolean _guessCellType; + final protected boolean _removeOriginalColumn; + final protected String _mode; + + final protected String _separator; + final protected boolean _regex; + final protected int _maxColumns; + + final protected int[] _fieldLengths; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + String mode = obj.getString("mode"); + + if ("separator".equals(mode)) { + return new ColumnSplitOperation( + engineConfig, + obj.getString("columnName"), + obj.getBoolean("guessCellType"), + obj.getBoolean("removeOriginalColumn"), + obj.getString("separator"), + obj.getBoolean("regex"), + obj.getInt("maxColumns") + ); + } else { + return new ColumnSplitOperation( + engineConfig, + obj.getString("columnName"), + obj.getBoolean("guessCellType"), + obj.getBoolean("removeOriginalColumn"), + JSONUtilities.getIntArray(obj, "fieldLengths") + ); + } + } + + public ColumnSplitOperation( + JSONObject engineConfig, + String columnName, + boolean guessCellType, + boolean removeOriginalColumn, + String separator, + boolean regex, + int maxColumns + ) { + super(engineConfig); + + _columnName = columnName; + _guessCellType = guessCellType; + _removeOriginalColumn = removeOriginalColumn; + + _mode = "separator"; + _separator = separator; + _regex = regex; + _maxColumns = maxColumns; + + _fieldLengths = null; + } + + public ColumnSplitOperation( + JSONObject engineConfig, + String columnName, + boolean guessCellType, + boolean removeOriginalColumn, + int[] fieldLengths + ) { + super(engineConfig); + + _columnName = columnName; + _guessCellType = guessCellType; + _removeOriginalColumn = removeOriginalColumn; + + _mode = "lengths"; + _separator = null; + _regex = false; + _maxColumns = -1; + + _fieldLengths = fieldLengths; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("columnName"); writer.value(_columnName); + writer.key("guessCellType"); writer.value(_guessCellType); + writer.key("removeOriginalColumn"); writer.value(_removeOriginalColumn); + writer.key("mode"); writer.value(_mode); + if ("separator".equals(_mode)) { + writer.key("separator"); writer.value(_separator); + writer.key("regex"); writer.value(_regex); + writer.key("maxColumns"); writer.value(_maxColumns); + } else { + writer.key("fieldLengths"); writer.array(); + for (int l : _fieldLengths) { + writer.value(l); + } + writer.endArray(); + } + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Split column " + _columnName + + ("separator".equals(_mode) ? " by separator" : " by field lengths"); + } + + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + Engine engine = createEngine(project); + + Column column = project.columnModel.getColumnByName(_columnName); + if (column == null) { + throw new Exception("No column named " + _columnName); + } + + List columnNames = new ArrayList(); + List rowIndices = new ArrayList(project.rows.size()); + List> tuples = new ArrayList>(project.rows.size()); + + FilteredRows filteredRows = engine.getAllFilteredRows(false); + RowVisitor rowVisitor; + if ("lengths".equals(_mode)) { + rowVisitor = new ColumnSplitRowVisitor(project, column.getCellIndex(), columnNames, rowIndices, tuples) { + protected java.util.List split(String s) { + List results = new ArrayList(_fieldLengths.length + 1); + + int lastIndex = 0; + for (int i = 0; i < _fieldLengths.length; i++) { + int from = lastIndex; + int length = _fieldLengths[i]; + int to = Math.min(from + length, s.length()); + + results.add(stringToValue(s.substring(from, to))); + + lastIndex = to; + } + + return results; + }; + }; + } else if (_regex) { + Pattern pattern = Pattern.compile(_separator); + + rowVisitor = new ColumnSplitRowVisitor(project, column.getCellIndex(), columnNames, rowIndices, tuples) { + Pattern _pattern; + + protected java.util.List split(String s) { + return stringArrayToValueList(_pattern.split(s, _maxColumns)); + }; + + public RowVisitor init(Pattern pattern) { + _pattern = pattern; + return this; + } + }.init(pattern); + } else { + rowVisitor = new ColumnSplitRowVisitor(project, column.getCellIndex(), columnNames, rowIndices, tuples) { + protected java.util.List split(String s) { + return stringArrayToValueList( + StringUtils.splitByWholeSeparatorPreserveAllTokens(s, _separator, _maxColumns)); + }; + }; + } + + filteredRows.accept(project, rowVisitor); + + String description = + "Split " + rowIndices.size() + + " cell(s) in column " + _columnName + + " into several columns" + + ("separator".equals(_mode) ? " by separator" : " by field lengths"); + + Change change = new ColumnSplitChange( + _columnName, + columnNames, + rowIndices, + tuples, + _removeOriginalColumn + ); + + return new HistoryEntry( + historyEntryID, project, description, this, change); + } + + protected class ColumnSplitRowVisitor implements RowVisitor { + Project project; + int cellIndex; + List columnNames; + List rowIndices; + List> tuples; + + int columnNameIndex = 1; + + ColumnSplitRowVisitor( + Project project, + int cellIndex, + List columnNames, + List rowIndices, + List> tuples + ) { + this.project = project; + this.cellIndex = cellIndex; + this.columnNames = columnNames; + this.rowIndices = rowIndices; + this.tuples = tuples; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + Object value = row.getCellValue(cellIndex); + if (ExpressionUtils.isNonBlankData(value)) { + String s = value instanceof String ? ((String) value) : value.toString(); + + List tuple = split(s); + + rowIndices.add(rowIndex); + tuples.add(tuple); + + for (int i = columnNames.size(); i < tuple.size(); i++) { + while (true) { + String newColumnName = _columnName + " " + columnNameIndex++; + if (project.columnModel.getColumnByName(newColumnName) == null) { + columnNames.add(newColumnName); + break; + } + } + } + } + return false; + } + + protected List split(String s) { + throw new NotImplementedException(); + } + + protected Serializable stringToValue(String s) { + return _guessCellType ? ImporterUtilities.parseCellValue(s) : s; + } + + protected List stringArrayToValueList(String[] cells) { + List results = new ArrayList(cells.length); + for (String cell : cells) { + results.add(stringToValue(cell)); + } + + return results; + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/DenormalizeOperation.java b/src/main/java/com/metaweb/gridworks/operations/DenormalizeOperation.java index a930468b8..598f67642 100644 --- a/src/main/java/com/metaweb/gridworks/operations/DenormalizeOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/DenormalizeOperation.java @@ -1,75 +1,75 @@ -package com.metaweb.gridworks.operations; - -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.MassRowChange; - -public class DenormalizeOperation extends AbstractOperation { - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - return new DenormalizeOperation(); - } - - public DenormalizeOperation() { - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value("Denormalize"); - writer.endObject(); - } - - - protected String getBriefDescription(Project project) { - return "Denormalize"; - } - - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - List newRows = new ArrayList(); - - List oldRows = project.rows; - for (int r = 0; r < oldRows.size(); r++) { - Row oldRow = oldRows.get(r); - Row newRow = null; - - if (oldRow.contextCellSlots != null && oldRow.contextRowSlots != null) { - newRow = oldRow.dup(); - - for (int c = 0; c < oldRow.contextCellSlots.length && c < oldRow.contextRowSlots.length; c++) { - int contextRowIndex = oldRow.contextRowSlots[c]; - int contextCellIndex = oldRow.contextCellSlots[c]; - - if (contextRowIndex >= 0 && contextRowIndex < oldRows.size()) { - Row contextRow = oldRows.get(contextRowIndex); - Cell contextCell = contextRow.getCell(contextCellIndex); - - newRow.setCell(contextCellIndex, contextCell); - } - } - } - - newRows.add(newRow != null ? newRow : oldRow); - } - - return new HistoryEntry( - historyEntryID, - project, - getBriefDescription(project), - DenormalizeOperation.this, - new MassRowChange(newRows) - ); - } -} +package com.metaweb.gridworks.operations; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.MassRowChange; + +public class DenormalizeOperation extends AbstractOperation { + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + return new DenormalizeOperation(); + } + + public DenormalizeOperation() { + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value("Denormalize"); + writer.endObject(); + } + + + protected String getBriefDescription(Project project) { + return "Denormalize"; + } + + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + List newRows = new ArrayList(); + + List oldRows = project.rows; + for (int r = 0; r < oldRows.size(); r++) { + Row oldRow = oldRows.get(r); + Row newRow = null; + + if (oldRow.contextCellSlots != null && oldRow.contextRowSlots != null) { + newRow = oldRow.dup(); + + for (int c = 0; c < oldRow.contextCellSlots.length && c < oldRow.contextRowSlots.length; c++) { + int contextRowIndex = oldRow.contextRowSlots[c]; + int contextCellIndex = oldRow.contextCellSlots[c]; + + if (contextRowIndex >= 0 && contextRowIndex < oldRows.size()) { + Row contextRow = oldRows.get(contextRowIndex); + Cell contextCell = contextRow.getCell(contextCellIndex); + + newRow.setCell(contextCellIndex, contextCell); + } + } + } + + newRows.add(newRow != null ? newRow : oldRow); + } + + return new HistoryEntry( + historyEntryID, + project, + getBriefDescription(project), + DenormalizeOperation.this, + new MassRowChange(newRows) + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/EngineDependentMassCellOperation.java b/src/main/java/com/metaweb/gridworks/operations/EngineDependentMassCellOperation.java index a1ea76cd5..78a686ddd 100644 --- a/src/main/java/com/metaweb/gridworks/operations/EngineDependentMassCellOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/EngineDependentMassCellOperation.java @@ -1,59 +1,59 @@ -package com.metaweb.gridworks.operations; - -import java.util.ArrayList; -import java.util.List; - -import org.json.JSONObject; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.changes.CellChange; -import com.metaweb.gridworks.model.changes.MassCellChange; - -abstract public class EngineDependentMassCellOperation extends EngineDependentOperation { - final protected String _columnName; - final protected boolean _updateRowContextDependencies; - - protected EngineDependentMassCellOperation( - JSONObject engineConfig, String columnName, boolean updateRowContextDependencies) { - super(engineConfig); - _columnName = columnName; - _updateRowContextDependencies = updateRowContextDependencies; - } - - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - Engine engine = createEngine(project); - - Column column = project.columnModel.getColumnByName(_columnName); - if (column == null) { - throw new Exception("No column named " + _columnName); - } - - List cellChanges = new ArrayList(project.rows.size()); - - FilteredRows filteredRows = engine.getAllFilteredRows(false); - try { - filteredRows.accept(project, createRowVisitor(project, cellChanges, historyEntryID)); - } catch (Exception e) { - e.printStackTrace(); - } - - String description = createDescription(column, cellChanges); - - return new HistoryEntry( - historyEntryID, project, description, this, createChange(project, column, cellChanges)); - } - - protected Change createChange(Project project, Column column, List cellChanges) { - return new MassCellChange( - cellChanges, column.getName(), _updateRowContextDependencies); - } - - abstract protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception; - abstract protected String createDescription(Column column, List cellChanges); -} +package com.metaweb.gridworks.operations; + +import java.util.ArrayList; +import java.util.List; + +import org.json.JSONObject; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.changes.CellChange; +import com.metaweb.gridworks.model.changes.MassCellChange; + +abstract public class EngineDependentMassCellOperation extends EngineDependentOperation { + final protected String _columnName; + final protected boolean _updateRowContextDependencies; + + protected EngineDependentMassCellOperation( + JSONObject engineConfig, String columnName, boolean updateRowContextDependencies) { + super(engineConfig); + _columnName = columnName; + _updateRowContextDependencies = updateRowContextDependencies; + } + + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + Engine engine = createEngine(project); + + Column column = project.columnModel.getColumnByName(_columnName); + if (column == null) { + throw new Exception("No column named " + _columnName); + } + + List cellChanges = new ArrayList(project.rows.size()); + + FilteredRows filteredRows = engine.getAllFilteredRows(false); + try { + filteredRows.accept(project, createRowVisitor(project, cellChanges, historyEntryID)); + } catch (Exception e) { + e.printStackTrace(); + } + + String description = createDescription(column, cellChanges); + + return new HistoryEntry( + historyEntryID, project, description, this, createChange(project, column, cellChanges)); + } + + protected Change createChange(Project project, Column column, List cellChanges) { + return new MassCellChange( + cellChanges, column.getName(), _updateRowContextDependencies); + } + + abstract protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception; + abstract protected String createDescription(Column column, List cellChanges); +} diff --git a/src/main/java/com/metaweb/gridworks/operations/EngineDependentOperation.java b/src/main/java/com/metaweb/gridworks/operations/EngineDependentOperation.java index cebd507b5..c052618de 100644 --- a/src/main/java/com/metaweb/gridworks/operations/EngineDependentOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/EngineDependentOperation.java @@ -1,38 +1,38 @@ -package com.metaweb.gridworks.operations; - -import org.json.JSONException; -import org.json.JSONObject; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.util.ParsingUtilities; - -abstract public class EngineDependentOperation extends AbstractOperation { - final private String _engineConfigString; - - transient protected JSONObject _engineConfig; - - protected EngineDependentOperation(JSONObject engineConfig) { - _engineConfig = engineConfig; - _engineConfigString = engineConfig == null || engineConfig.length() == 0 - ? null : engineConfig.toString(); - } - - protected Engine createEngine(Project project) throws Exception { - Engine engine = new Engine(project); - engine.initializeFromJSON(getEngineConfig()); - return engine; - } - - protected JSONObject getEngineConfig() { - if (_engineConfig == null && _engineConfigString != null) { - try { - _engineConfig = ParsingUtilities.evaluateJsonStringToObject(_engineConfigString); - } catch (JSONException e) { - // ignore - } - } - return _engineConfig; - } -} +package com.metaweb.gridworks.operations; + +import org.json.JSONException; +import org.json.JSONObject; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.util.ParsingUtilities; + +abstract public class EngineDependentOperation extends AbstractOperation { + final private String _engineConfigString; + + transient protected JSONObject _engineConfig; + + protected EngineDependentOperation(JSONObject engineConfig) { + _engineConfig = engineConfig; + _engineConfigString = engineConfig == null || engineConfig.length() == 0 + ? null : engineConfig.toString(); + } + + protected Engine createEngine(Project project) throws Exception { + Engine engine = new Engine(project); + engine.initializeFromJSON(getEngineConfig()); + return engine; + } + + protected JSONObject getEngineConfig() { + if (_engineConfig == null && _engineConfigString != null) { + try { + _engineConfig = ParsingUtilities.evaluateJsonStringToObject(_engineConfigString); + } catch (JSONException e) { + // ignore + } + } + return _engineConfig; + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/ExtendDataOperation.java b/src/main/java/com/metaweb/gridworks/operations/ExtendDataOperation.java index 0b1832271..068cf0493 100644 --- a/src/main/java/com/metaweb/gridworks/operations/ExtendDataOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/ExtendDataOperation.java @@ -1,263 +1,263 @@ -package com.metaweb.gridworks.operations; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.CellAtRow; -import com.metaweb.gridworks.model.changes.DataExtensionChange; -import com.metaweb.gridworks.process.LongRunningProcess; -import com.metaweb.gridworks.process.Process; -import com.metaweb.gridworks.protograph.FreebaseType; -import com.metaweb.gridworks.util.FreebaseDataExtensionJob; -import com.metaweb.gridworks.util.FreebaseDataExtensionJob.ColumnInfo; -import com.metaweb.gridworks.util.FreebaseDataExtensionJob.DataExtension; - -public class ExtendDataOperation extends EngineDependentOperation { - final protected String _baseColumnName; - final protected JSONObject _extension; - final protected int _columnInsertIndex; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - - return new ExtendDataOperation( - engineConfig, - obj.getString("baseColumnName"), - obj.getJSONObject("extension"), - obj.getInt("columnInsertIndex") - ); - } - - public ExtendDataOperation( - JSONObject engineConfig, - String baseColumnName, - JSONObject extension, - int columnInsertIndex - ) { - super(engineConfig); - - _baseColumnName = baseColumnName; - _extension = extension; - _columnInsertIndex = columnInsertIndex; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("columnInsertIndex"); writer.value(_columnInsertIndex); - writer.key("baseColumnName"); writer.value(_baseColumnName); - writer.key("extension"); writer.value(_extension); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Extend data at index " + _columnInsertIndex + - " based on column " + _baseColumnName; - } - - protected String createDescription(Column column, List cellsAtRows) { - return "Extend data at index " + _columnInsertIndex + - " based on column " + column.getName() + - " by filling " + cellsAtRows.size(); - } - - public Process createProcess(Project project, Properties options) throws Exception { - return new ExtendDataProcess( - project, - getEngineConfig(), - getBriefDescription(null) - ); - } - - public class ExtendDataProcess extends LongRunningProcess implements Runnable { - final protected Project _project; - final protected JSONObject _engineConfig; - final protected long _historyEntryID; - protected int _cellIndex; - protected FreebaseDataExtensionJob _job; - - public ExtendDataProcess( - Project project, - JSONObject engineConfig, - String description - ) throws JSONException { - super(description); - _project = project; - _engineConfig = engineConfig; - _historyEntryID = HistoryEntry.allocateID(); - - _job = new FreebaseDataExtensionJob(_extension); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("id"); writer.value(hashCode()); - writer.key("description"); writer.value(_description); - writer.key("immediate"); writer.value(false); - writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done")); - writer.key("progress"); writer.value(_progress); - writer.endObject(); - } - - protected Runnable getRunnable() { - return this; - } - - protected void populateRowsWithMatches(List rowIndices) throws Exception { - Engine engine = new Engine(_project); - engine.initializeFromJSON(_engineConfig); - - Column column = _project.columnModel.getColumnByName(_baseColumnName); - if (column == null) { - throw new Exception("No column named " + _baseColumnName); - } - - _cellIndex = column.getCellIndex(); - - FilteredRows filteredRows = engine.getAllFilteredRows(false); - filteredRows.accept(_project, new RowVisitor() { - List _rowIndices; - - public RowVisitor init(List rowIndices) { - _rowIndices = rowIndices; - return this; - } - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - if (!includeContextual) { - Cell cell = row.getCell(_cellIndex); - if (cell != null && cell.recon != null && cell.recon.match != null) { - _rowIndices.add(rowIndex); - } - } - return false; - } - }.init(rowIndices)); - } - - protected int extendRows( - List rowIndices, - List dataExtensions, - int from, - int limit, - Map reconCandidateMap - ) { - Set guids = new HashSet(); - - int end; - for (end = from; end < limit && guids.size() < 10; end++) { - int index = rowIndices.get(end); - Row row = _project.rows.get(index); - Cell cell = row.getCell(_cellIndex); - - guids.add(cell.recon.match.topicGUID); - } - - Map map = null; - try { - map = _job.extend(guids, reconCandidateMap); - } catch (Exception e) { - map = new HashMap(); - } - - for (int i = from; i < end; i++) { - int index = rowIndices.get(i); - Row row = _project.rows.get(index); - Cell cell = row.getCell(_cellIndex); - String guid = cell.recon.match.topicGUID; - - if (map.containsKey(guid)) { - dataExtensions.add(map.get(guid)); - } else { - dataExtensions.add(null); - } - } - - return end; - } - - public void run() { - List rowIndices = new ArrayList(); - List dataExtensions = new ArrayList(); - - try { - populateRowsWithMatches(rowIndices); - } catch (Exception e2) { - // TODO : Not sure what to do here? - e2.printStackTrace(); - } - - int start = 0; - Map reconCandidateMap = new HashMap(); - - while (start < rowIndices.size()) { - int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap); - start = end; - - _progress = end * 100 / rowIndices.size(); - try { - Thread.sleep(200); - } catch (InterruptedException e) { - if (_canceled) { - break; - } - } - } - - if (!_canceled) { - List columnNames = new ArrayList(); - for (ColumnInfo info : _job.columns) { - columnNames.add(StringUtils.join(info.names, " - ")); - } - - List columnTypes = new ArrayList(); - for (ColumnInfo info : _job.columns) { - columnTypes.add(info.expectedType); - } - - HistoryEntry historyEntry = new HistoryEntry( - _historyEntryID, - _project, - _description, - ExtendDataOperation.this, - new DataExtensionChange( - _baseColumnName, - _columnInsertIndex, - columnNames, - columnTypes, - rowIndices, - dataExtensions, - _historyEntryID) - ); - - _project.history.addEntry(historyEntry); - _project.processManager.onDoneProcess(this); - } - } - } -} +package com.metaweb.gridworks.operations; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.CellAtRow; +import com.metaweb.gridworks.model.changes.DataExtensionChange; +import com.metaweb.gridworks.process.LongRunningProcess; +import com.metaweb.gridworks.process.Process; +import com.metaweb.gridworks.protograph.FreebaseType; +import com.metaweb.gridworks.util.FreebaseDataExtensionJob; +import com.metaweb.gridworks.util.FreebaseDataExtensionJob.ColumnInfo; +import com.metaweb.gridworks.util.FreebaseDataExtensionJob.DataExtension; + +public class ExtendDataOperation extends EngineDependentOperation { + final protected String _baseColumnName; + final protected JSONObject _extension; + final protected int _columnInsertIndex; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + + return new ExtendDataOperation( + engineConfig, + obj.getString("baseColumnName"), + obj.getJSONObject("extension"), + obj.getInt("columnInsertIndex") + ); + } + + public ExtendDataOperation( + JSONObject engineConfig, + String baseColumnName, + JSONObject extension, + int columnInsertIndex + ) { + super(engineConfig); + + _baseColumnName = baseColumnName; + _extension = extension; + _columnInsertIndex = columnInsertIndex; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("columnInsertIndex"); writer.value(_columnInsertIndex); + writer.key("baseColumnName"); writer.value(_baseColumnName); + writer.key("extension"); writer.value(_extension); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Extend data at index " + _columnInsertIndex + + " based on column " + _baseColumnName; + } + + protected String createDescription(Column column, List cellsAtRows) { + return "Extend data at index " + _columnInsertIndex + + " based on column " + column.getName() + + " by filling " + cellsAtRows.size(); + } + + public Process createProcess(Project project, Properties options) throws Exception { + return new ExtendDataProcess( + project, + getEngineConfig(), + getBriefDescription(null) + ); + } + + public class ExtendDataProcess extends LongRunningProcess implements Runnable { + final protected Project _project; + final protected JSONObject _engineConfig; + final protected long _historyEntryID; + protected int _cellIndex; + protected FreebaseDataExtensionJob _job; + + public ExtendDataProcess( + Project project, + JSONObject engineConfig, + String description + ) throws JSONException { + super(description); + _project = project; + _engineConfig = engineConfig; + _historyEntryID = HistoryEntry.allocateID(); + + _job = new FreebaseDataExtensionJob(_extension); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("id"); writer.value(hashCode()); + writer.key("description"); writer.value(_description); + writer.key("immediate"); writer.value(false); + writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done")); + writer.key("progress"); writer.value(_progress); + writer.endObject(); + } + + protected Runnable getRunnable() { + return this; + } + + protected void populateRowsWithMatches(List rowIndices) throws Exception { + Engine engine = new Engine(_project); + engine.initializeFromJSON(_engineConfig); + + Column column = _project.columnModel.getColumnByName(_baseColumnName); + if (column == null) { + throw new Exception("No column named " + _baseColumnName); + } + + _cellIndex = column.getCellIndex(); + + FilteredRows filteredRows = engine.getAllFilteredRows(false); + filteredRows.accept(_project, new RowVisitor() { + List _rowIndices; + + public RowVisitor init(List rowIndices) { + _rowIndices = rowIndices; + return this; + } + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + if (!includeContextual) { + Cell cell = row.getCell(_cellIndex); + if (cell != null && cell.recon != null && cell.recon.match != null) { + _rowIndices.add(rowIndex); + } + } + return false; + } + }.init(rowIndices)); + } + + protected int extendRows( + List rowIndices, + List dataExtensions, + int from, + int limit, + Map reconCandidateMap + ) { + Set guids = new HashSet(); + + int end; + for (end = from; end < limit && guids.size() < 10; end++) { + int index = rowIndices.get(end); + Row row = _project.rows.get(index); + Cell cell = row.getCell(_cellIndex); + + guids.add(cell.recon.match.topicGUID); + } + + Map map = null; + try { + map = _job.extend(guids, reconCandidateMap); + } catch (Exception e) { + map = new HashMap(); + } + + for (int i = from; i < end; i++) { + int index = rowIndices.get(i); + Row row = _project.rows.get(index); + Cell cell = row.getCell(_cellIndex); + String guid = cell.recon.match.topicGUID; + + if (map.containsKey(guid)) { + dataExtensions.add(map.get(guid)); + } else { + dataExtensions.add(null); + } + } + + return end; + } + + public void run() { + List rowIndices = new ArrayList(); + List dataExtensions = new ArrayList(); + + try { + populateRowsWithMatches(rowIndices); + } catch (Exception e2) { + // TODO : Not sure what to do here? + e2.printStackTrace(); + } + + int start = 0; + Map reconCandidateMap = new HashMap(); + + while (start < rowIndices.size()) { + int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap); + start = end; + + _progress = end * 100 / rowIndices.size(); + try { + Thread.sleep(200); + } catch (InterruptedException e) { + if (_canceled) { + break; + } + } + } + + if (!_canceled) { + List columnNames = new ArrayList(); + for (ColumnInfo info : _job.columns) { + columnNames.add(StringUtils.join(info.names, " - ")); + } + + List columnTypes = new ArrayList(); + for (ColumnInfo info : _job.columns) { + columnTypes.add(info.expectedType); + } + + HistoryEntry historyEntry = new HistoryEntry( + _historyEntryID, + _project, + _description, + ExtendDataOperation.this, + new DataExtensionChange( + _baseColumnName, + _columnInsertIndex, + columnNames, + columnTypes, + rowIndices, + dataExtensions, + _historyEntryID) + ); + + _project.history.addEntry(historyEntry); + _project.processManager.onDoneProcess(this); + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/MassEditOperation.java b/src/main/java/com/metaweb/gridworks/operations/MassEditOperation.java index 2808554d8..61ac96bb5 100644 --- a/src/main/java/com/metaweb/gridworks/operations/MassEditOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/MassEditOperation.java @@ -1,230 +1,230 @@ -package com.metaweb.gridworks.operations; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.expr.MetaParser; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.CellChange; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class MassEditOperation extends EngineDependentMassCellOperation { - final protected String _expression; - final protected List _edits; - - static public class Edit implements Jsonizable { - final public List from; - final public boolean fromBlank; - final public boolean fromError; - final public Serializable to; - - public Edit(List from, boolean fromBlank, boolean fromError, Serializable to) { - this.from = from; - this.fromBlank = fromBlank; - this.fromError = fromError; - this.to = to; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("fromBlank"); writer.value(fromBlank); - writer.key("fromError"); writer.value(fromError); - writer.key("from"); - writer.array(); - for (String s : from) { - writer.value(s); - } - writer.endArray(); - writer.key("to"); writer.value(to); - writer.endObject(); - } - } - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.has("engineConfig") && !obj.isNull("engineConfig") ? - obj.getJSONObject("engineConfig") : null; - - return new MassEditOperation( - engineConfig, - obj.getString("columnName"), - obj.getString("expression"), - reconstructEdits(obj.getJSONArray("edits")) - ); - } - - static public List reconstructEdits(JSONArray editsA) throws Exception { - int editCount = editsA.length(); - - List edits = new ArrayList(editCount); - for (int i = 0; i < editCount; i++) { - JSONObject editO = editsA.getJSONObject(i); - - List from = null; - if (editO.has("from") && !editO.isNull("from")) { - JSONArray fromA = editO.getJSONArray("from"); - int fromCount = fromA.length(); - - from = new ArrayList(fromCount); - for (int j = 0; j < fromCount; j++) { - from.add(fromA.getString(j)); - } - } else { - from = new ArrayList(); - } - - boolean fromBlank = editO.has("fromBlank") && editO.getBoolean("fromBlank"); - boolean fromError = editO.has("fromError") && editO.getBoolean("fromError"); - - Serializable to = (Serializable) editO.get("to"); - if (editO.has("type")) { - String type = editO.getString("type"); - if ("date".equals(type)) { - to = ParsingUtilities.stringToDate((String) to); - } - } - - edits.add(new Edit(from, fromBlank, fromError, to)); - } - - return edits; - } - - public MassEditOperation(JSONObject engineConfig, String columnName, String expression, List edits) { - super(engineConfig, columnName, true); - _expression = expression; - _edits = edits; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("columnName"); writer.value(_columnName); - writer.key("expression"); writer.value(_expression); - writer.key("edits"); - writer.array(); - for (Edit edit : _edits) { - edit.write(writer, options); - } - writer.endArray(); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Mass edit cells in column " + _columnName; - } - - protected String createDescription(Column column, - List cellChanges) { - - return "Mass edit " + cellChanges.size() + - " cells in column " + column.getName(); - } - - protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { - Column column = project.columnModel.getColumnByName(_columnName); - - Evaluable eval = MetaParser.parse(_expression); - Properties bindings = ExpressionUtils.createBindings(project); - - Map fromTo = new HashMap(); - Serializable fromBlankTo = null; - Serializable fromErrorTo = null; - - for (Edit edit : _edits) { - for (String s : edit.from) { - fromTo.put(s, edit.to); - } - - // the last edit wins - if (edit.fromBlank) { - fromBlankTo = edit.to; - } - if (edit.fromError) { - fromErrorTo = edit.to; - } - } - - return new RowVisitor() { - int cellIndex; - Properties bindings; - List cellChanges; - Evaluable eval; - - Map fromTo; - Serializable fromBlankTo; - Serializable fromErrorTo; - - public RowVisitor init( - int cellIndex, - Properties bindings, - List cellChanges, - Evaluable eval, - Map fromTo, - Serializable fromBlankTo, - Serializable fromErrorTo - ) { - this.cellIndex = cellIndex; - this.bindings = bindings; - this.cellChanges = cellChanges; - this.eval = eval; - this.fromTo = fromTo; - this.fromBlankTo = fromBlankTo; - this.fromErrorTo = fromErrorTo; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - Cell cell = row.getCell(cellIndex); - Cell newCell = null; - - ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); - - Object v = eval.evaluate(bindings); - if (ExpressionUtils.isError(v)) { - if (fromErrorTo != null) { - newCell = new Cell(fromErrorTo, (cell != null) ? cell.recon : null); - } - } else if (ExpressionUtils.isNonBlankData(v)) { - String from = v.toString(); - Serializable to = fromTo.get(from); - if (to != null) { - newCell = new Cell(to, (cell != null) ? cell.recon : null); - } - } else { - if (fromBlankTo != null) { - newCell = new Cell(fromBlankTo, (cell != null) ? cell.recon : null); - } - } - - if (newCell != null) { - CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); - cellChanges.add(cellChange); - } - return false; - } - }.init(column.getCellIndex(), bindings, cellChanges, eval, fromTo, fromBlankTo, fromErrorTo); - } -} +package com.metaweb.gridworks.operations; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.expr.MetaParser; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.CellChange; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class MassEditOperation extends EngineDependentMassCellOperation { + final protected String _expression; + final protected List _edits; + + static public class Edit implements Jsonizable { + final public List from; + final public boolean fromBlank; + final public boolean fromError; + final public Serializable to; + + public Edit(List from, boolean fromBlank, boolean fromError, Serializable to) { + this.from = from; + this.fromBlank = fromBlank; + this.fromError = fromError; + this.to = to; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("fromBlank"); writer.value(fromBlank); + writer.key("fromError"); writer.value(fromError); + writer.key("from"); + writer.array(); + for (String s : from) { + writer.value(s); + } + writer.endArray(); + writer.key("to"); writer.value(to); + writer.endObject(); + } + } + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.has("engineConfig") && !obj.isNull("engineConfig") ? + obj.getJSONObject("engineConfig") : null; + + return new MassEditOperation( + engineConfig, + obj.getString("columnName"), + obj.getString("expression"), + reconstructEdits(obj.getJSONArray("edits")) + ); + } + + static public List reconstructEdits(JSONArray editsA) throws Exception { + int editCount = editsA.length(); + + List edits = new ArrayList(editCount); + for (int i = 0; i < editCount; i++) { + JSONObject editO = editsA.getJSONObject(i); + + List from = null; + if (editO.has("from") && !editO.isNull("from")) { + JSONArray fromA = editO.getJSONArray("from"); + int fromCount = fromA.length(); + + from = new ArrayList(fromCount); + for (int j = 0; j < fromCount; j++) { + from.add(fromA.getString(j)); + } + } else { + from = new ArrayList(); + } + + boolean fromBlank = editO.has("fromBlank") && editO.getBoolean("fromBlank"); + boolean fromError = editO.has("fromError") && editO.getBoolean("fromError"); + + Serializable to = (Serializable) editO.get("to"); + if (editO.has("type")) { + String type = editO.getString("type"); + if ("date".equals(type)) { + to = ParsingUtilities.stringToDate((String) to); + } + } + + edits.add(new Edit(from, fromBlank, fromError, to)); + } + + return edits; + } + + public MassEditOperation(JSONObject engineConfig, String columnName, String expression, List edits) { + super(engineConfig, columnName, true); + _expression = expression; + _edits = edits; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("columnName"); writer.value(_columnName); + writer.key("expression"); writer.value(_expression); + writer.key("edits"); + writer.array(); + for (Edit edit : _edits) { + edit.write(writer, options); + } + writer.endArray(); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Mass edit cells in column " + _columnName; + } + + protected String createDescription(Column column, + List cellChanges) { + + return "Mass edit " + cellChanges.size() + + " cells in column " + column.getName(); + } + + protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { + Column column = project.columnModel.getColumnByName(_columnName); + + Evaluable eval = MetaParser.parse(_expression); + Properties bindings = ExpressionUtils.createBindings(project); + + Map fromTo = new HashMap(); + Serializable fromBlankTo = null; + Serializable fromErrorTo = null; + + for (Edit edit : _edits) { + for (String s : edit.from) { + fromTo.put(s, edit.to); + } + + // the last edit wins + if (edit.fromBlank) { + fromBlankTo = edit.to; + } + if (edit.fromError) { + fromErrorTo = edit.to; + } + } + + return new RowVisitor() { + int cellIndex; + Properties bindings; + List cellChanges; + Evaluable eval; + + Map fromTo; + Serializable fromBlankTo; + Serializable fromErrorTo; + + public RowVisitor init( + int cellIndex, + Properties bindings, + List cellChanges, + Evaluable eval, + Map fromTo, + Serializable fromBlankTo, + Serializable fromErrorTo + ) { + this.cellIndex = cellIndex; + this.bindings = bindings; + this.cellChanges = cellChanges; + this.eval = eval; + this.fromTo = fromTo; + this.fromBlankTo = fromBlankTo; + this.fromErrorTo = fromErrorTo; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + Cell cell = row.getCell(cellIndex); + Cell newCell = null; + + ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); + + Object v = eval.evaluate(bindings); + if (ExpressionUtils.isError(v)) { + if (fromErrorTo != null) { + newCell = new Cell(fromErrorTo, (cell != null) ? cell.recon : null); + } + } else if (ExpressionUtils.isNonBlankData(v)) { + String from = v.toString(); + Serializable to = fromTo.get(from); + if (to != null) { + newCell = new Cell(to, (cell != null) ? cell.recon : null); + } + } else { + if (fromBlankTo != null) { + newCell = new Cell(fromBlankTo, (cell != null) ? cell.recon : null); + } + } + + if (newCell != null) { + CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); + cellChanges.add(cellChange); + } + return false; + } + }.init(column.getCellIndex(), bindings, cellChanges, eval, fromTo, fromBlankTo, fromErrorTo); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/MultiValuedCellJoinOperation.java b/src/main/java/com/metaweb/gridworks/operations/MultiValuedCellJoinOperation.java index 0ee7e6c31..fb382df0c 100644 --- a/src/main/java/com/metaweb/gridworks/operations/MultiValuedCellJoinOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/MultiValuedCellJoinOperation.java @@ -1,129 +1,129 @@ -package com.metaweb.gridworks.operations; - -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.MassRowChange; - -public class MultiValuedCellJoinOperation extends AbstractOperation { - final protected String _columnName; - final protected String _keyColumnName; - final protected String _separator; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - return new MultiValuedCellJoinOperation( - obj.getString("columnName"), - obj.getString("keyColumnName"), - obj.getString("separator") - ); - } - - public MultiValuedCellJoinOperation( - String columnName, - String keyColumnName, - String separator - ) { - _columnName = columnName; - _keyColumnName = keyColumnName; - _separator = separator; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("columnName"); writer.value(_columnName); - writer.key("keyColumnName"); writer.value(_keyColumnName); - writer.key("separator"); writer.value(_separator); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Join multi-valued cells in column " + _columnName; - } - - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - Column column = project.columnModel.getColumnByName(_columnName); - if (column == null) { - throw new Exception("No column named " + _columnName); - } - int cellIndex = column.getCellIndex(); - - Column keyColumn = project.columnModel.getColumnByName(_keyColumnName); - if (keyColumn == null) { - throw new Exception("No key column named " + _keyColumnName); - } - int keyCellIndex = keyColumn.getCellIndex(); - - List newRows = new ArrayList(); - - int oldRowCount = project.rows.size(); - for (int r = 0; r < oldRowCount; r++) { - Row oldRow = project.rows.get(r); - - if (oldRow.isCellBlank(keyCellIndex)) { - newRows.add(oldRow.dup()); - continue; - } - - int r2 = r + 1; - while (r2 < oldRowCount && project.rows.get(r2).isCellBlank(keyCellIndex)) { - r2++; - } - - if (r2 == r + 1) { - newRows.add(oldRow.dup()); - continue; - } - - StringBuffer sb = new StringBuffer(); - for (int r3 = r; r3 < r2; r3++) { - Object value = project.rows.get(r3).getCellValue(cellIndex); - if (ExpressionUtils.isNonBlankData(value)) { - if (sb.length() > 0) { - sb.append(_separator); - } - sb.append(value.toString()); - } - } - - for (int r3 = r; r3 < r2; r3++) { - Row newRow = project.rows.get(r3).dup(); - if (r3 == r) { - newRow.setCell(cellIndex, new Cell(sb.toString(), null)); - } else { - newRow.setCell(cellIndex, null); - } - - if (!newRow.isEmpty()) { - newRows.add(newRow); - } - } - - r = r2 - 1; // r will be incremented by the for loop anyway - } - - return new HistoryEntry( - historyEntryID, - project, - getBriefDescription(null), - this, - new MassRowChange(newRows) - ); - } - -} +package com.metaweb.gridworks.operations; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.MassRowChange; + +public class MultiValuedCellJoinOperation extends AbstractOperation { + final protected String _columnName; + final protected String _keyColumnName; + final protected String _separator; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + return new MultiValuedCellJoinOperation( + obj.getString("columnName"), + obj.getString("keyColumnName"), + obj.getString("separator") + ); + } + + public MultiValuedCellJoinOperation( + String columnName, + String keyColumnName, + String separator + ) { + _columnName = columnName; + _keyColumnName = keyColumnName; + _separator = separator; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("columnName"); writer.value(_columnName); + writer.key("keyColumnName"); writer.value(_keyColumnName); + writer.key("separator"); writer.value(_separator); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Join multi-valued cells in column " + _columnName; + } + + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + Column column = project.columnModel.getColumnByName(_columnName); + if (column == null) { + throw new Exception("No column named " + _columnName); + } + int cellIndex = column.getCellIndex(); + + Column keyColumn = project.columnModel.getColumnByName(_keyColumnName); + if (keyColumn == null) { + throw new Exception("No key column named " + _keyColumnName); + } + int keyCellIndex = keyColumn.getCellIndex(); + + List newRows = new ArrayList(); + + int oldRowCount = project.rows.size(); + for (int r = 0; r < oldRowCount; r++) { + Row oldRow = project.rows.get(r); + + if (oldRow.isCellBlank(keyCellIndex)) { + newRows.add(oldRow.dup()); + continue; + } + + int r2 = r + 1; + while (r2 < oldRowCount && project.rows.get(r2).isCellBlank(keyCellIndex)) { + r2++; + } + + if (r2 == r + 1) { + newRows.add(oldRow.dup()); + continue; + } + + StringBuffer sb = new StringBuffer(); + for (int r3 = r; r3 < r2; r3++) { + Object value = project.rows.get(r3).getCellValue(cellIndex); + if (ExpressionUtils.isNonBlankData(value)) { + if (sb.length() > 0) { + sb.append(_separator); + } + sb.append(value.toString()); + } + } + + for (int r3 = r; r3 < r2; r3++) { + Row newRow = project.rows.get(r3).dup(); + if (r3 == r) { + newRow.setCell(cellIndex, new Cell(sb.toString(), null)); + } else { + newRow.setCell(cellIndex, null); + } + + if (!newRow.isEmpty()) { + newRows.add(newRow); + } + } + + r = r2 - 1; // r will be incremented by the for loop anyway + } + + return new HistoryEntry( + historyEntryID, + project, + getBriefDescription(null), + this, + new MassRowChange(newRows) + ); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/operations/MultiValuedCellSplitOperation.java b/src/main/java/com/metaweb/gridworks/operations/MultiValuedCellSplitOperation.java index bfc695938..f170523d5 100644 --- a/src/main/java/com/metaweb/gridworks/operations/MultiValuedCellSplitOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/MultiValuedCellSplitOperation.java @@ -1,146 +1,146 @@ -package com.metaweb.gridworks.operations; - - import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.apache.commons.lang.StringUtils; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.MassRowChange; - -public class MultiValuedCellSplitOperation extends AbstractOperation { - final protected String _columnName; - final protected String _keyColumnName; - final protected String _separator; - final protected String _mode; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - return new MultiValuedCellSplitOperation( - obj.getString("columnName"), - obj.getString("keyColumnName"), - obj.getString("separator"), - obj.getString("mode") - ); - } - - public MultiValuedCellSplitOperation( - String columnName, - String keyColumnName, - String separator, - String mode - ) { - _columnName = columnName; - _keyColumnName = keyColumnName; - _separator = separator; - _mode = mode; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value("Split multi-valued cells in column " + _columnName); - writer.key("columnName"); writer.value(_columnName); - writer.key("keyColumnName"); writer.value(_keyColumnName); - writer.key("separator"); writer.value(_separator); - writer.key("mode"); writer.value(_mode); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Split multi-valued cells in column " + _columnName; - } - - @Override - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - Column column = project.columnModel.getColumnByName(_columnName); - if (column == null) { - throw new Exception("No column named " + _columnName); - } - int cellIndex = column.getCellIndex(); - - Column keyColumn = project.columnModel.getColumnByName(_keyColumnName); - if (keyColumn == null) { - throw new Exception("No key column named " + _keyColumnName); - } - int keyCellIndex = keyColumn.getCellIndex(); - - List newRows = new ArrayList(); - - int oldRowCount = project.rows.size(); - for (int r = 0; r < oldRowCount; r++) { - Row oldRow = project.rows.get(r); - if (oldRow.isCellBlank(cellIndex)) { - newRows.add(oldRow.dup()); - continue; - } - - Object value = oldRow.getCellValue(cellIndex); - String s = value instanceof String ? ((String) value) : value.toString(); - String[] values = null; - if (_mode.equals("regex")) { - values = s.split(_separator); - } else { - values = StringUtils.splitByWholeSeparator(s, _separator); - } - - if (values.length < 2) { - newRows.add(oldRow.dup()); - continue; - } - - // First value goes into the same row - { - Row firstNewRow = oldRow.dup(); - firstNewRow.setCell(cellIndex, new Cell(values[0].trim(), null)); - - newRows.add(firstNewRow); - } - - int r2 = r + 1; - for (int v = 1; v < values.length; v++) { - Cell newCell = new Cell(values[v].trim(), null); - - if (r2 < project.rows.size()) { - Row oldRow2 = project.rows.get(r2); - if (oldRow2.isCellBlank(cellIndex) && - oldRow2.isCellBlank(keyCellIndex)) { - - Row newRow = oldRow2.dup(); - newRow.setCell(cellIndex, newCell); - - newRows.add(newRow); - r2++; - - continue; - } - } - - Row newRow = new Row(cellIndex + 1); - newRow.setCell(cellIndex, newCell); - - newRows.add(newRow); - } - - r = r2 - 1; // r will be incremented by the for loop anyway - } - - return new HistoryEntry( - historyEntryID, - project, - getBriefDescription(null), - this, - new MassRowChange(newRows) - ); - } -} +package com.metaweb.gridworks.operations; + + import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.apache.commons.lang.StringUtils; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.MassRowChange; + +public class MultiValuedCellSplitOperation extends AbstractOperation { + final protected String _columnName; + final protected String _keyColumnName; + final protected String _separator; + final protected String _mode; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + return new MultiValuedCellSplitOperation( + obj.getString("columnName"), + obj.getString("keyColumnName"), + obj.getString("separator"), + obj.getString("mode") + ); + } + + public MultiValuedCellSplitOperation( + String columnName, + String keyColumnName, + String separator, + String mode + ) { + _columnName = columnName; + _keyColumnName = keyColumnName; + _separator = separator; + _mode = mode; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value("Split multi-valued cells in column " + _columnName); + writer.key("columnName"); writer.value(_columnName); + writer.key("keyColumnName"); writer.value(_keyColumnName); + writer.key("separator"); writer.value(_separator); + writer.key("mode"); writer.value(_mode); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Split multi-valued cells in column " + _columnName; + } + + @Override + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + Column column = project.columnModel.getColumnByName(_columnName); + if (column == null) { + throw new Exception("No column named " + _columnName); + } + int cellIndex = column.getCellIndex(); + + Column keyColumn = project.columnModel.getColumnByName(_keyColumnName); + if (keyColumn == null) { + throw new Exception("No key column named " + _keyColumnName); + } + int keyCellIndex = keyColumn.getCellIndex(); + + List newRows = new ArrayList(); + + int oldRowCount = project.rows.size(); + for (int r = 0; r < oldRowCount; r++) { + Row oldRow = project.rows.get(r); + if (oldRow.isCellBlank(cellIndex)) { + newRows.add(oldRow.dup()); + continue; + } + + Object value = oldRow.getCellValue(cellIndex); + String s = value instanceof String ? ((String) value) : value.toString(); + String[] values = null; + if (_mode.equals("regex")) { + values = s.split(_separator); + } else { + values = StringUtils.splitByWholeSeparator(s, _separator); + } + + if (values.length < 2) { + newRows.add(oldRow.dup()); + continue; + } + + // First value goes into the same row + { + Row firstNewRow = oldRow.dup(); + firstNewRow.setCell(cellIndex, new Cell(values[0].trim(), null)); + + newRows.add(firstNewRow); + } + + int r2 = r + 1; + for (int v = 1; v < values.length; v++) { + Cell newCell = new Cell(values[v].trim(), null); + + if (r2 < project.rows.size()) { + Row oldRow2 = project.rows.get(r2); + if (oldRow2.isCellBlank(cellIndex) && + oldRow2.isCellBlank(keyCellIndex)) { + + Row newRow = oldRow2.dup(); + newRow.setCell(cellIndex, newCell); + + newRows.add(newRow); + r2++; + + continue; + } + } + + Row newRow = new Row(cellIndex + 1); + newRow.setCell(cellIndex, newCell); + + newRows.add(newRow); + } + + r = r2 - 1; // r will be incremented by the for loop anyway + } + + return new HistoryEntry( + historyEntryID, + project, + getBriefDescription(null), + this, + new MassRowChange(newRows) + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/OnError.java b/src/main/java/com/metaweb/gridworks/operations/OnError.java index 072842ed5..3e82f434d 100644 --- a/src/main/java/com/metaweb/gridworks/operations/OnError.java +++ b/src/main/java/com/metaweb/gridworks/operations/OnError.java @@ -4,7 +4,7 @@ package com.metaweb.gridworks.operations; public enum OnError { - KeepOriginal, - SetToBlank, - StoreError + KeepOriginal, + SetToBlank, + StoreError } \ No newline at end of file diff --git a/src/main/java/com/metaweb/gridworks/operations/ReconDiscardJudgmentsOperation.java b/src/main/java/com/metaweb/gridworks/operations/ReconDiscardJudgmentsOperation.java index 82d4ff60d..37ba73c6c 100644 --- a/src/main/java/com/metaweb/gridworks/operations/ReconDiscardJudgmentsOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/ReconDiscardJudgmentsOperation.java @@ -1,113 +1,113 @@ -package com.metaweb.gridworks.operations; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.model.changes.CellChange; -import com.metaweb.gridworks.model.changes.ReconChange; - -public class ReconDiscardJudgmentsOperation extends EngineDependentMassCellOperation { - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - String columnName = obj.getString("columnName"); - - return new ReconDiscardJudgmentsOperation( - engineConfig, - columnName - ); - } - - public ReconDiscardJudgmentsOperation(JSONObject engineConfig, String columnName) { - super(engineConfig, columnName, false); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("columnName"); writer.value(_columnName); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Discard recon judgments for cells in column " + _columnName; - } - - protected String createDescription(Column column, - List cellChanges) { - - return "Discard recon judgments for " + cellChanges.size() + - " cells in column " + column.getName(); - } - - protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { - Column column = project.columnModel.getColumnByName(_columnName); - - return new RowVisitor() { - int cellIndex; - List cellChanges; - Map dupReconMap = new HashMap(); - long historyEntryID; - - public RowVisitor init(int cellIndex, List cellChanges, long historyEntryID) { - this.cellIndex = cellIndex; - this.cellChanges = cellChanges; - this.historyEntryID = historyEntryID; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - Cell cell = row.getCell(cellIndex); - if (cell != null && cell.recon != null) { - Recon newRecon; - if (dupReconMap.containsKey(cell.recon.id)) { - newRecon = dupReconMap.get(cell.recon.id); - newRecon.judgmentBatchSize++; - } else { - newRecon = cell.recon.dup(historyEntryID); - newRecon.match = null; - newRecon.matchRank = -1; - newRecon.judgment = Judgment.None; - newRecon.judgmentAction = "mass"; - newRecon.judgmentBatchSize = 1; - - dupReconMap.put(cell.recon.id, newRecon); - } - - Cell newCell = new Cell(cell.value, newRecon); - - CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); - cellChanges.add(cellChange); - } - return false; - } - }.init(column.getCellIndex(), cellChanges, historyEntryID); - } - - protected Change createChange(Project project, Column column, List cellChanges) { - return new ReconChange( - cellChanges, - _columnName, - column.getReconConfig(), - null - ); - } -} +package com.metaweb.gridworks.operations; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.model.changes.CellChange; +import com.metaweb.gridworks.model.changes.ReconChange; + +public class ReconDiscardJudgmentsOperation extends EngineDependentMassCellOperation { + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + String columnName = obj.getString("columnName"); + + return new ReconDiscardJudgmentsOperation( + engineConfig, + columnName + ); + } + + public ReconDiscardJudgmentsOperation(JSONObject engineConfig, String columnName) { + super(engineConfig, columnName, false); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("columnName"); writer.value(_columnName); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Discard recon judgments for cells in column " + _columnName; + } + + protected String createDescription(Column column, + List cellChanges) { + + return "Discard recon judgments for " + cellChanges.size() + + " cells in column " + column.getName(); + } + + protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { + Column column = project.columnModel.getColumnByName(_columnName); + + return new RowVisitor() { + int cellIndex; + List cellChanges; + Map dupReconMap = new HashMap(); + long historyEntryID; + + public RowVisitor init(int cellIndex, List cellChanges, long historyEntryID) { + this.cellIndex = cellIndex; + this.cellChanges = cellChanges; + this.historyEntryID = historyEntryID; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + Cell cell = row.getCell(cellIndex); + if (cell != null && cell.recon != null) { + Recon newRecon; + if (dupReconMap.containsKey(cell.recon.id)) { + newRecon = dupReconMap.get(cell.recon.id); + newRecon.judgmentBatchSize++; + } else { + newRecon = cell.recon.dup(historyEntryID); + newRecon.match = null; + newRecon.matchRank = -1; + newRecon.judgment = Judgment.None; + newRecon.judgmentAction = "mass"; + newRecon.judgmentBatchSize = 1; + + dupReconMap.put(cell.recon.id, newRecon); + } + + Cell newCell = new Cell(cell.value, newRecon); + + CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); + cellChanges.add(cellChange); + } + return false; + } + }.init(column.getCellIndex(), cellChanges, historyEntryID); + } + + protected Change createChange(Project project, Column column, List cellChanges) { + return new ReconChange( + cellChanges, + _columnName, + column.getReconConfig(), + null + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/ReconJudgeSimilarCellsOperation.java b/src/main/java/com/metaweb/gridworks/operations/ReconJudgeSimilarCellsOperation.java index f819b0dda..d100174b4 100644 --- a/src/main/java/com/metaweb/gridworks/operations/ReconJudgeSimilarCellsOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/ReconJudgeSimilarCellsOperation.java @@ -1,235 +1,235 @@ -package com.metaweb.gridworks.operations; - - import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.model.changes.CellChange; -import com.metaweb.gridworks.model.changes.ReconChange; - -public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOperation { - final protected String _similarValue; - final protected Judgment _judgment; - final protected ReconCandidate _match; - final protected boolean _shareNewTopics; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - - ReconCandidate match = null; - if (obj.has("match")) { - JSONObject matchObj = obj.getJSONObject("match"); - - JSONArray types = matchObj.getJSONArray("types"); - String[] typeIDs = new String[types.length()]; - for (int i = 0; i < typeIDs.length; i++) { - typeIDs[i] = types.getString(i); - } - - match = new ReconCandidate( - matchObj.getString("id"), - matchObj.getString("guid"), - matchObj.getString("name"), - typeIDs, - matchObj.getDouble("score") - ); - } - - Judgment judgment = Judgment.None; - if (obj.has("judgment")) { - judgment = Recon.stringToJudgment(obj.getString("judgment")); - } - - return new ReconJudgeSimilarCellsOperation( - engineConfig, - obj.getString("columnName"), - obj.getString("similarValue"), - judgment, - match, - obj.has("shareNewTopics") ? obj.getBoolean("shareNewTopics") : false - ); - } - - public ReconJudgeSimilarCellsOperation( - JSONObject engineConfig, - String columnName, - String similarValue, - Judgment judgment, - ReconCandidate match, - boolean shareNewTopics - ) { - super(engineConfig, columnName, false); - this._similarValue = similarValue; - this._judgment = judgment; - this._match = match; - this._shareNewTopics = shareNewTopics; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("columnName"); writer.value(_columnName); - writer.key("similarValue"); writer.value(_similarValue); - writer.key("judgment"); writer.value(Recon.judgmentToString(_judgment)); - if (_match != null) { - writer.key("match"); _match.write(writer, options); - } - writer.key("shareNewTopics"); writer.value(_shareNewTopics); - - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - if (_judgment == Judgment.None) { - return "Discard recon judgments for cells containing \"" + - _similarValue + "\" in column " + _columnName; - } else if (_judgment == Judgment.New) { - if (_shareNewTopics) { - return "Mark to create one single new topic for all cells containing \"" + - _similarValue + "\" in column " + _columnName; - } else { - return "Mark to create one new topic for each cell containing \"" + - _similarValue + "\" in column " + _columnName; - } - } else if (_judgment == Judgment.Matched) { - return "Match topic " + - _match.topicName + " (" + - _match.topicID + ") for cells containing \"" + - _similarValue + "\" in column " + _columnName; - } - throw new InternalError("Can't get here"); - } - - protected String createDescription(Column column, - List cellChanges) { - - if (_judgment == Judgment.None) { - return "Discard recon judgments for " + cellChanges.size() + " cells containing \"" + - _similarValue + "\" in column " + _columnName; - } else if (_judgment == Judgment.New) { - if (_shareNewTopics) { - return "Mark to create one single new topic for " + cellChanges.size() + " cells containing \"" + - _similarValue + "\" in column " + _columnName; - } else { - return "Mark to create one new topic for each of " + cellChanges.size() + " cells containing \"" + - _similarValue + "\" in column " + _columnName; - } - } else if (_judgment == Judgment.Matched) { - return "Match topic " + - _match.topicName + " (" + - _match.topicID + ") for " + - cellChanges.size() + " cells containing \"" + - _similarValue + "\" in column " + _columnName; - } - throw new InternalError("Can't get here"); - } - - protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { - Column column = project.columnModel.getColumnByName(_columnName); - - return new RowVisitor() { - int _cellIndex; - List _cellChanges; - Recon _sharedNewRecon = null; - Map _dupReconMap = new HashMap(); - long _historyEntryID; - - public RowVisitor init(int cellIndex, List cellChanges, long historyEntryID) { - _cellIndex = cellIndex; - _cellChanges = cellChanges; - _historyEntryID = historyEntryID; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - Cell cell = row.getCell(_cellIndex); - if (cell != null && - ExpressionUtils.isNonBlankData(cell.value) && - _similarValue.equals(cell.value)) { - - Recon recon = null; - if (_judgment == Judgment.New && _shareNewTopics) { - if (_sharedNewRecon == null) { - _sharedNewRecon = new Recon(_historyEntryID); - _sharedNewRecon.judgment = Judgment.New; - _sharedNewRecon.judgmentBatchSize = 0; - _sharedNewRecon.judgmentAction = "similar"; - } - _sharedNewRecon.judgmentBatchSize++; - - recon = _sharedNewRecon; - } else { - if (_dupReconMap.containsKey(cell.recon.id)) { - recon = _dupReconMap.get(cell.recon.id); - recon.judgmentBatchSize++; - } else { - recon = cell.recon.dup(_historyEntryID); - recon.judgmentBatchSize = 1; - recon.matchRank = -1; - recon.judgmentAction = "similar"; - - if (_judgment == Judgment.Matched) { - recon.judgment = Recon.Judgment.Matched; - recon.match = _match; - - if (recon.candidates != null) { - for (int m = 0; m < recon.candidates.size(); m++) { - if (recon.candidates.get(m).topicGUID.equals(_match.topicGUID)) { - recon.matchRank = m; - break; - } - } - } - } else if (_judgment == Judgment.New) { - recon.judgment = Recon.Judgment.New; - recon.match = null; - } else if (_judgment == Judgment.None) { - recon.judgment = Recon.Judgment.None; - recon.match = null; - } - - _dupReconMap.put(cell.recon.id, recon); - } - } - - Cell newCell = new Cell(cell.value, recon); - - CellChange cellChange = new CellChange(rowIndex, _cellIndex, cell, newCell); - _cellChanges.add(cellChange); - } - return false; - } - }.init(column.getCellIndex(), cellChanges, historyEntryID); - } - - - protected Change createChange(Project project, Column column, List cellChanges) { - return new ReconChange( - cellChanges, - _columnName, - column.getReconConfig(), - null - ); - } -} +package com.metaweb.gridworks.operations; + + import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.model.changes.CellChange; +import com.metaweb.gridworks.model.changes.ReconChange; + +public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOperation { + final protected String _similarValue; + final protected Judgment _judgment; + final protected ReconCandidate _match; + final protected boolean _shareNewTopics; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + + ReconCandidate match = null; + if (obj.has("match")) { + JSONObject matchObj = obj.getJSONObject("match"); + + JSONArray types = matchObj.getJSONArray("types"); + String[] typeIDs = new String[types.length()]; + for (int i = 0; i < typeIDs.length; i++) { + typeIDs[i] = types.getString(i); + } + + match = new ReconCandidate( + matchObj.getString("id"), + matchObj.getString("guid"), + matchObj.getString("name"), + typeIDs, + matchObj.getDouble("score") + ); + } + + Judgment judgment = Judgment.None; + if (obj.has("judgment")) { + judgment = Recon.stringToJudgment(obj.getString("judgment")); + } + + return new ReconJudgeSimilarCellsOperation( + engineConfig, + obj.getString("columnName"), + obj.getString("similarValue"), + judgment, + match, + obj.has("shareNewTopics") ? obj.getBoolean("shareNewTopics") : false + ); + } + + public ReconJudgeSimilarCellsOperation( + JSONObject engineConfig, + String columnName, + String similarValue, + Judgment judgment, + ReconCandidate match, + boolean shareNewTopics + ) { + super(engineConfig, columnName, false); + this._similarValue = similarValue; + this._judgment = judgment; + this._match = match; + this._shareNewTopics = shareNewTopics; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("columnName"); writer.value(_columnName); + writer.key("similarValue"); writer.value(_similarValue); + writer.key("judgment"); writer.value(Recon.judgmentToString(_judgment)); + if (_match != null) { + writer.key("match"); _match.write(writer, options); + } + writer.key("shareNewTopics"); writer.value(_shareNewTopics); + + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + if (_judgment == Judgment.None) { + return "Discard recon judgments for cells containing \"" + + _similarValue + "\" in column " + _columnName; + } else if (_judgment == Judgment.New) { + if (_shareNewTopics) { + return "Mark to create one single new topic for all cells containing \"" + + _similarValue + "\" in column " + _columnName; + } else { + return "Mark to create one new topic for each cell containing \"" + + _similarValue + "\" in column " + _columnName; + } + } else if (_judgment == Judgment.Matched) { + return "Match topic " + + _match.topicName + " (" + + _match.topicID + ") for cells containing \"" + + _similarValue + "\" in column " + _columnName; + } + throw new InternalError("Can't get here"); + } + + protected String createDescription(Column column, + List cellChanges) { + + if (_judgment == Judgment.None) { + return "Discard recon judgments for " + cellChanges.size() + " cells containing \"" + + _similarValue + "\" in column " + _columnName; + } else if (_judgment == Judgment.New) { + if (_shareNewTopics) { + return "Mark to create one single new topic for " + cellChanges.size() + " cells containing \"" + + _similarValue + "\" in column " + _columnName; + } else { + return "Mark to create one new topic for each of " + cellChanges.size() + " cells containing \"" + + _similarValue + "\" in column " + _columnName; + } + } else if (_judgment == Judgment.Matched) { + return "Match topic " + + _match.topicName + " (" + + _match.topicID + ") for " + + cellChanges.size() + " cells containing \"" + + _similarValue + "\" in column " + _columnName; + } + throw new InternalError("Can't get here"); + } + + protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { + Column column = project.columnModel.getColumnByName(_columnName); + + return new RowVisitor() { + int _cellIndex; + List _cellChanges; + Recon _sharedNewRecon = null; + Map _dupReconMap = new HashMap(); + long _historyEntryID; + + public RowVisitor init(int cellIndex, List cellChanges, long historyEntryID) { + _cellIndex = cellIndex; + _cellChanges = cellChanges; + _historyEntryID = historyEntryID; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + Cell cell = row.getCell(_cellIndex); + if (cell != null && + ExpressionUtils.isNonBlankData(cell.value) && + _similarValue.equals(cell.value)) { + + Recon recon = null; + if (_judgment == Judgment.New && _shareNewTopics) { + if (_sharedNewRecon == null) { + _sharedNewRecon = new Recon(_historyEntryID); + _sharedNewRecon.judgment = Judgment.New; + _sharedNewRecon.judgmentBatchSize = 0; + _sharedNewRecon.judgmentAction = "similar"; + } + _sharedNewRecon.judgmentBatchSize++; + + recon = _sharedNewRecon; + } else { + if (_dupReconMap.containsKey(cell.recon.id)) { + recon = _dupReconMap.get(cell.recon.id); + recon.judgmentBatchSize++; + } else { + recon = cell.recon.dup(_historyEntryID); + recon.judgmentBatchSize = 1; + recon.matchRank = -1; + recon.judgmentAction = "similar"; + + if (_judgment == Judgment.Matched) { + recon.judgment = Recon.Judgment.Matched; + recon.match = _match; + + if (recon.candidates != null) { + for (int m = 0; m < recon.candidates.size(); m++) { + if (recon.candidates.get(m).topicGUID.equals(_match.topicGUID)) { + recon.matchRank = m; + break; + } + } + } + } else if (_judgment == Judgment.New) { + recon.judgment = Recon.Judgment.New; + recon.match = null; + } else if (_judgment == Judgment.None) { + recon.judgment = Recon.Judgment.None; + recon.match = null; + } + + _dupReconMap.put(cell.recon.id, recon); + } + } + + Cell newCell = new Cell(cell.value, recon); + + CellChange cellChange = new CellChange(rowIndex, _cellIndex, cell, newCell); + _cellChanges.add(cellChange); + } + return false; + } + }.init(column.getCellIndex(), cellChanges, historyEntryID); + } + + + protected Change createChange(Project project, Column column, List cellChanges) { + return new ReconChange( + cellChanges, + _columnName, + column.getReconConfig(), + null + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/ReconMarkNewTopicsOperation.java b/src/main/java/com/metaweb/gridworks/operations/ReconMarkNewTopicsOperation.java index a5589240c..3d8736514 100644 --- a/src/main/java/com/metaweb/gridworks/operations/ReconMarkNewTopicsOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/ReconMarkNewTopicsOperation.java @@ -1,140 +1,140 @@ -package com.metaweb.gridworks.operations; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.model.changes.CellChange; -import com.metaweb.gridworks.model.changes.ReconChange; - -public class ReconMarkNewTopicsOperation extends EngineDependentMassCellOperation { - final protected boolean _shareNewTopics; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - - return new ReconMarkNewTopicsOperation( - engineConfig, - obj.getString("columnName"), - obj.has("shareNewTopics") ? obj.getBoolean("shareNewTopics") : false - ); - } - - public ReconMarkNewTopicsOperation(JSONObject engineConfig, String columnName, boolean shareNewTopics) { - super(engineConfig, columnName, false); - _shareNewTopics = shareNewTopics; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("columnName"); writer.value(_columnName); - writer.key("shareNewTopics"); writer.value(_shareNewTopics); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Mark to create new topics for cells in column " + _columnName + - (_shareNewTopics ? - ", one topic for each group of similar cells" : - ", one topic for each cell"); - } - - protected String createDescription(Column column, - List cellChanges) { - - return "Mark to create new topics for " + cellChanges.size() + - " cells in column " + column.getName() + - (_shareNewTopics ? - ", one topic for each group of similar cells" : - ", one topic for each cell"); - } - - protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { - Column column = project.columnModel.getColumnByName(_columnName); - - return new RowVisitor() { - int cellIndex; - List cellChanges; - Map sharedRecons = new HashMap(); - Map dupReconMap = new HashMap(); - long historyEntryID; - - public RowVisitor init(int cellIndex, List cellChanges, long historyEntryID) { - this.cellIndex = cellIndex; - this.cellChanges = cellChanges; - this.historyEntryID = historyEntryID; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - Cell cell = row.getCell(cellIndex); - if (cell != null) { - Recon recon = null; - if (_shareNewTopics) { - String s = cell.value == null ? "" : cell.value.toString(); - if (sharedRecons.containsKey(s)) { - recon = sharedRecons.get(s); - recon.judgmentBatchSize++; - } else { - recon = new Recon(historyEntryID); - recon.judgment = Judgment.New; - recon.judgmentBatchSize = 1; - recon.judgmentAction = "mass"; - - sharedRecons.put(s, recon); - } - } else { - long reconID = cell.recon == null ? 0 : cell.recon.id; - if (dupReconMap.containsKey(reconID)) { - recon = dupReconMap.get(reconID); - recon.judgmentBatchSize++; - } else { - recon = cell.recon == null ? new Recon(historyEntryID) : cell.recon.dup(historyEntryID); - recon.match = null; - recon.matchRank = -1; - recon.judgment = Judgment.New; - recon.judgmentBatchSize = 1; - recon.judgmentAction = "mass"; - - dupReconMap.put(reconID, recon); - } - } - - Cell newCell = new Cell(cell.value, recon); - - CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); - cellChanges.add(cellChange); - } - return false; - } - }.init(column.getCellIndex(), cellChanges, historyEntryID); - } - - protected Change createChange(Project project, Column column, List cellChanges) { - return new ReconChange( - cellChanges, - _columnName, - column.getReconConfig(), - null - ); - } -} +package com.metaweb.gridworks.operations; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.model.changes.CellChange; +import com.metaweb.gridworks.model.changes.ReconChange; + +public class ReconMarkNewTopicsOperation extends EngineDependentMassCellOperation { + final protected boolean _shareNewTopics; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + + return new ReconMarkNewTopicsOperation( + engineConfig, + obj.getString("columnName"), + obj.has("shareNewTopics") ? obj.getBoolean("shareNewTopics") : false + ); + } + + public ReconMarkNewTopicsOperation(JSONObject engineConfig, String columnName, boolean shareNewTopics) { + super(engineConfig, columnName, false); + _shareNewTopics = shareNewTopics; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("columnName"); writer.value(_columnName); + writer.key("shareNewTopics"); writer.value(_shareNewTopics); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Mark to create new topics for cells in column " + _columnName + + (_shareNewTopics ? + ", one topic for each group of similar cells" : + ", one topic for each cell"); + } + + protected String createDescription(Column column, + List cellChanges) { + + return "Mark to create new topics for " + cellChanges.size() + + " cells in column " + column.getName() + + (_shareNewTopics ? + ", one topic for each group of similar cells" : + ", one topic for each cell"); + } + + protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { + Column column = project.columnModel.getColumnByName(_columnName); + + return new RowVisitor() { + int cellIndex; + List cellChanges; + Map sharedRecons = new HashMap(); + Map dupReconMap = new HashMap(); + long historyEntryID; + + public RowVisitor init(int cellIndex, List cellChanges, long historyEntryID) { + this.cellIndex = cellIndex; + this.cellChanges = cellChanges; + this.historyEntryID = historyEntryID; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + Cell cell = row.getCell(cellIndex); + if (cell != null) { + Recon recon = null; + if (_shareNewTopics) { + String s = cell.value == null ? "" : cell.value.toString(); + if (sharedRecons.containsKey(s)) { + recon = sharedRecons.get(s); + recon.judgmentBatchSize++; + } else { + recon = new Recon(historyEntryID); + recon.judgment = Judgment.New; + recon.judgmentBatchSize = 1; + recon.judgmentAction = "mass"; + + sharedRecons.put(s, recon); + } + } else { + long reconID = cell.recon == null ? 0 : cell.recon.id; + if (dupReconMap.containsKey(reconID)) { + recon = dupReconMap.get(reconID); + recon.judgmentBatchSize++; + } else { + recon = cell.recon == null ? new Recon(historyEntryID) : cell.recon.dup(historyEntryID); + recon.match = null; + recon.matchRank = -1; + recon.judgment = Judgment.New; + recon.judgmentBatchSize = 1; + recon.judgmentAction = "mass"; + + dupReconMap.put(reconID, recon); + } + } + + Cell newCell = new Cell(cell.value, recon); + + CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); + cellChanges.add(cellChange); + } + return false; + } + }.init(column.getCellIndex(), cellChanges, historyEntryID); + } + + protected Change createChange(Project project, Column column, List cellChanges) { + return new ReconChange( + cellChanges, + _columnName, + column.getReconConfig(), + null + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/ReconMatchBestCandidatesOperation.java b/src/main/java/com/metaweb/gridworks/operations/ReconMatchBestCandidatesOperation.java index 2a22f5cb9..4150202d0 100644 --- a/src/main/java/com/metaweb/gridworks/operations/ReconMatchBestCandidatesOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/ReconMatchBestCandidatesOperation.java @@ -1,121 +1,121 @@ -package com.metaweb.gridworks.operations; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.model.changes.CellChange; -import com.metaweb.gridworks.model.changes.ReconChange; - -public class ReconMatchBestCandidatesOperation extends EngineDependentMassCellOperation { - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - String columnName = obj.getString("columnName"); - - return new ReconMatchBestCandidatesOperation( - engineConfig, - columnName - ); - } - - public ReconMatchBestCandidatesOperation(JSONObject engineConfig, String columnName) { - super(engineConfig, columnName, false); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("columnName"); writer.value(_columnName); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Match each cell to its best recon candidate in column " + _columnName; - } - - protected String createDescription(Column column, - List cellChanges) { - - return "Match each of " + cellChanges.size() + - " cells to its best candidate in column " + column.getName(); - } - - protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { - Column column = project.columnModel.getColumnByName(_columnName); - - return new RowVisitor() { - int cellIndex; - List cellChanges; - Map dupReconMap = new HashMap(); - long historyEntryID; - - public RowVisitor init(int cellIndex, List cellChanges, long historyEntryID) { - this.cellIndex = cellIndex; - this.cellChanges = cellChanges; - this.historyEntryID = historyEntryID; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - if (cellIndex < row.cells.size()) { - Cell cell = row.cells.get(cellIndex); - if (cell != null && cell.recon != null) { - ReconCandidate candidate = cell.recon.getBestCandidate(); - if (candidate != null) { - Recon newRecon; - if (dupReconMap.containsKey(cell.recon.id)) { - newRecon = dupReconMap.get(cell.recon.id); - newRecon.judgmentBatchSize++; - } else { - newRecon = cell.recon.dup(historyEntryID); - newRecon.judgmentBatchSize = 1; - newRecon.match = candidate; - newRecon.matchRank = 0; - newRecon.judgment = Judgment.Matched; - newRecon.judgmentAction = "mass"; - - dupReconMap.put(cell.recon.id, newRecon); - } - Cell newCell = new Cell( - cell.value, - newRecon - ); - - CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); - cellChanges.add(cellChange); - } - } - } - return false; - } - }.init(column.getCellIndex(), cellChanges, historyEntryID); - } - - protected Change createChange(Project project, Column column, List cellChanges) { - return new ReconChange( - cellChanges, - _columnName, - column.getReconConfig(), - null - ); - } -} +package com.metaweb.gridworks.operations; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.model.changes.CellChange; +import com.metaweb.gridworks.model.changes.ReconChange; + +public class ReconMatchBestCandidatesOperation extends EngineDependentMassCellOperation { + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + String columnName = obj.getString("columnName"); + + return new ReconMatchBestCandidatesOperation( + engineConfig, + columnName + ); + } + + public ReconMatchBestCandidatesOperation(JSONObject engineConfig, String columnName) { + super(engineConfig, columnName, false); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("columnName"); writer.value(_columnName); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Match each cell to its best recon candidate in column " + _columnName; + } + + protected String createDescription(Column column, + List cellChanges) { + + return "Match each of " + cellChanges.size() + + " cells to its best candidate in column " + column.getName(); + } + + protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { + Column column = project.columnModel.getColumnByName(_columnName); + + return new RowVisitor() { + int cellIndex; + List cellChanges; + Map dupReconMap = new HashMap(); + long historyEntryID; + + public RowVisitor init(int cellIndex, List cellChanges, long historyEntryID) { + this.cellIndex = cellIndex; + this.cellChanges = cellChanges; + this.historyEntryID = historyEntryID; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + if (cellIndex < row.cells.size()) { + Cell cell = row.cells.get(cellIndex); + if (cell != null && cell.recon != null) { + ReconCandidate candidate = cell.recon.getBestCandidate(); + if (candidate != null) { + Recon newRecon; + if (dupReconMap.containsKey(cell.recon.id)) { + newRecon = dupReconMap.get(cell.recon.id); + newRecon.judgmentBatchSize++; + } else { + newRecon = cell.recon.dup(historyEntryID); + newRecon.judgmentBatchSize = 1; + newRecon.match = candidate; + newRecon.matchRank = 0; + newRecon.judgment = Judgment.Matched; + newRecon.judgmentAction = "mass"; + + dupReconMap.put(cell.recon.id, newRecon); + } + Cell newCell = new Cell( + cell.value, + newRecon + ); + + CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); + cellChanges.add(cellChange); + } + } + } + return false; + } + }.init(column.getCellIndex(), cellChanges, historyEntryID); + } + + protected Change createChange(Project project, Column column, List cellChanges) { + return new ReconChange( + cellChanges, + _columnName, + column.getReconConfig(), + null + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/ReconMatchSpecificTopicOperation.java b/src/main/java/com/metaweb/gridworks/operations/ReconMatchSpecificTopicOperation.java index 2ec4c7028..7cb22a6d8 100644 --- a/src/main/java/com/metaweb/gridworks/operations/ReconMatchSpecificTopicOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/ReconMatchSpecificTopicOperation.java @@ -1,152 +1,152 @@ -package com.metaweb.gridworks.operations; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.ReconCandidate; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.model.changes.CellChange; -import com.metaweb.gridworks.model.changes.ReconChange; - -public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOperation { - final protected ReconCandidate match; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - - JSONObject match = obj.getJSONObject("match"); - - JSONArray types = obj.getJSONArray("types"); - String[] typeIDs = new String[types.length()]; - for (int i = 0; i < typeIDs.length; i++) { - typeIDs[i] = types.getString(i); - } - - return new ReconMatchSpecificTopicOperation( - engineConfig, - obj.getString("columnName"), - new ReconCandidate( - match.getString("id"), - match.getString("guid"), - match.getString("name"), - typeIDs, - 100 - ) - ); - } - - public ReconMatchSpecificTopicOperation(JSONObject engineConfig, String columnName, ReconCandidate match) { - super(engineConfig, columnName, false); - this.match = match; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("columnName"); writer.value(_columnName); - writer.key("match"); - writer.object(); - writer.key("id"); writer.value(match.topicID); - writer.key("guid"); writer.value(match.topicGUID); - writer.key("name"); writer.value(match.topicName); - writer.key("types"); - writer.array(); - for (String typeID : match.typeIDs) { - writer.value(typeID); - } - writer.endArray(); - writer.endObject(); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Match specific topic " + - match.topicName + " (" + - match.topicID + ") to cells in column " + _columnName; - } - - protected String createDescription(Column column, - List cellChanges) { - return "Match specific topic " + - match.topicName + " (" + - match.topicID + ") to " + cellChanges.size() + - " cells in column " + column.getName(); - } - - protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { - Column column = project.columnModel.getColumnByName(_columnName); - - return new RowVisitor() { - int cellIndex; - List cellChanges; - Map dupReconMap = new HashMap(); - long historyEntryID; - - public RowVisitor init(int cellIndex, List cellChanges, long historyEntryID) { - this.cellIndex = cellIndex; - this.cellChanges = cellChanges; - this.historyEntryID = historyEntryID; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - Cell cell = row.getCell(cellIndex); - if (cell != null) { - long reconID = cell.recon != null ? cell.recon.id : 0; - - Recon newRecon; - if (dupReconMap.containsKey(reconID)) { - newRecon = dupReconMap.get(reconID); - newRecon.judgmentBatchSize++; - } else { - newRecon = cell.recon != null ? cell.recon.dup(historyEntryID) : new Recon(historyEntryID); - newRecon.match = match; - newRecon.matchRank = -1; - newRecon.judgment = Judgment.Matched; - newRecon.judgmentAction = "mass"; - newRecon.judgmentBatchSize = 1; - - dupReconMap.put(reconID, newRecon); - } - - Cell newCell = new Cell( - cell.value, - newRecon - ); - - CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); - cellChanges.add(cellChange); - } - return false; - } - }.init(column.getCellIndex(), cellChanges, historyEntryID); - } - - protected Change createChange(Project project, Column column, List cellChanges) { - return new ReconChange( - cellChanges, - _columnName, - column.getReconConfig(), - null - ); - } -} +package com.metaweb.gridworks.operations; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.ReconCandidate; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.model.changes.CellChange; +import com.metaweb.gridworks.model.changes.ReconChange; + +public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOperation { + final protected ReconCandidate match; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + + JSONObject match = obj.getJSONObject("match"); + + JSONArray types = obj.getJSONArray("types"); + String[] typeIDs = new String[types.length()]; + for (int i = 0; i < typeIDs.length; i++) { + typeIDs[i] = types.getString(i); + } + + return new ReconMatchSpecificTopicOperation( + engineConfig, + obj.getString("columnName"), + new ReconCandidate( + match.getString("id"), + match.getString("guid"), + match.getString("name"), + typeIDs, + 100 + ) + ); + } + + public ReconMatchSpecificTopicOperation(JSONObject engineConfig, String columnName, ReconCandidate match) { + super(engineConfig, columnName, false); + this.match = match; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("columnName"); writer.value(_columnName); + writer.key("match"); + writer.object(); + writer.key("id"); writer.value(match.topicID); + writer.key("guid"); writer.value(match.topicGUID); + writer.key("name"); writer.value(match.topicName); + writer.key("types"); + writer.array(); + for (String typeID : match.typeIDs) { + writer.value(typeID); + } + writer.endArray(); + writer.endObject(); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Match specific topic " + + match.topicName + " (" + + match.topicID + ") to cells in column " + _columnName; + } + + protected String createDescription(Column column, + List cellChanges) { + return "Match specific topic " + + match.topicName + " (" + + match.topicID + ") to " + cellChanges.size() + + " cells in column " + column.getName(); + } + + protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { + Column column = project.columnModel.getColumnByName(_columnName); + + return new RowVisitor() { + int cellIndex; + List cellChanges; + Map dupReconMap = new HashMap(); + long historyEntryID; + + public RowVisitor init(int cellIndex, List cellChanges, long historyEntryID) { + this.cellIndex = cellIndex; + this.cellChanges = cellChanges; + this.historyEntryID = historyEntryID; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + Cell cell = row.getCell(cellIndex); + if (cell != null) { + long reconID = cell.recon != null ? cell.recon.id : 0; + + Recon newRecon; + if (dupReconMap.containsKey(reconID)) { + newRecon = dupReconMap.get(reconID); + newRecon.judgmentBatchSize++; + } else { + newRecon = cell.recon != null ? cell.recon.dup(historyEntryID) : new Recon(historyEntryID); + newRecon.match = match; + newRecon.matchRank = -1; + newRecon.judgment = Judgment.Matched; + newRecon.judgmentAction = "mass"; + newRecon.judgmentBatchSize = 1; + + dupReconMap.put(reconID, newRecon); + } + + Cell newCell = new Cell( + cell.value, + newRecon + ); + + CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); + cellChanges.add(cellChange); + } + return false; + } + }.init(column.getCellIndex(), cellChanges, historyEntryID); + } + + protected Change createChange(Project project, Column column, List cellChanges) { + return new ReconChange( + cellChanges, + _columnName, + column.getReconConfig(), + null + ); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/ReconOperation.java b/src/main/java/com/metaweb/gridworks/operations/ReconOperation.java index eca322405..b734bb784 100644 --- a/src/main/java/com/metaweb/gridworks/operations/ReconOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/ReconOperation.java @@ -1,285 +1,285 @@ -package com.metaweb.gridworks.operations; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.CellChange; -import com.metaweb.gridworks.model.changes.ReconChange; -import com.metaweb.gridworks.model.recon.HeuristicReconConfig; -import com.metaweb.gridworks.model.recon.ReconConfig; -import com.metaweb.gridworks.model.recon.ReconJob; -import com.metaweb.gridworks.process.LongRunningProcess; -import com.metaweb.gridworks.process.Process; - -public class ReconOperation extends EngineDependentOperation { - final protected String _columnName; - final protected ReconConfig _reconConfig; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - - return new ReconOperation( - engineConfig, - obj.getString("columnName"), - ReconConfig.reconstruct(obj.getJSONObject("config")) - ); - } - - public ReconOperation( - JSONObject engineConfig, - String columnName, - ReconConfig reconConfig - ) { - super(engineConfig); - _columnName = columnName; - _reconConfig = reconConfig; - } - - public Process createProcess(Project project, Properties options) throws Exception { - return new ReconProcess( - project, - getEngineConfig(), - getBriefDescription(null) - ); - } - - protected String getBriefDescription(Project project) { - return _reconConfig.getBriefDescription(project, _columnName); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("columnName"); writer.value(_columnName); - writer.key("config"); _reconConfig.write(writer, options); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.endObject(); - } - - static protected class ReconEntry { - final public int rowIndex; - final public Cell cell; - - public ReconEntry(int rowIndex, Cell cell) { - this.rowIndex = rowIndex; - this.cell = cell; - } - } - static protected class JobGroup { - final public ReconJob job; - final public List entries = new ArrayList(); - - public JobGroup(ReconJob job) { - this.job = job; - } - } - - public class ReconProcess extends LongRunningProcess implements Runnable { - final protected Project _project; - final protected JSONObject _engineConfig; - final protected long _historyEntryID; - protected List _entries; - protected int _cellIndex; - - public ReconProcess( - Project project, - JSONObject engineConfig, - String description - ) { - super(description); - _project = project; - _engineConfig = engineConfig; - _historyEntryID = HistoryEntry.allocateID(); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("id"); writer.value(hashCode()); - writer.key("description"); writer.value(_description); - writer.key("immediate"); writer.value(false); - writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done")); - writer.key("progress"); writer.value(_progress); - writer.key("onDone"); - writer.array(); - writer.object(); - writer.key("action"); writer.value("createFacet"); - writer.key("facetType"); writer.value("list"); - writer.key("facetConfig"); - writer.object(); - writer.key("name"); writer.value(_columnName + ": judgment"); - writer.key("columnName"); writer.value(_columnName); - writer.key("expression"); writer.value("cell.recon.judgment"); - writer.key("omitError"); writer.value(true); - writer.endObject(); - writer.key("facetOptions"); - writer.object(); - writer.key("scroll"); writer.value(false); - writer.endObject(); - writer.endObject(); - - if (_reconConfig instanceof HeuristicReconConfig) { - writer.object(); - writer.key("action"); writer.value("createFacet"); - writer.key("facetType"); writer.value("range"); - writer.key("facetConfig"); - writer.object(); - writer.key("name"); writer.value(_columnName + ": best candidate's score"); - writer.key("columnName"); writer.value(_columnName); - writer.key("expression"); writer.value("cell.recon.best.score"); - writer.key("mode"); writer.value("range"); - writer.endObject(); - writer.endObject(); - } - writer.endArray(); - writer.endObject(); - } - - protected Runnable getRunnable() { - return this; - } - - protected void populateEntries() throws Exception { - Engine engine = new Engine(_project); - engine.initializeFromJSON(_engineConfig); - - Column column = _project.columnModel.getColumnByName(_columnName); - if (column == null) { - throw new Exception("No column named " + _columnName); - } - - _entries = new ArrayList(_project.rows.size()); - _cellIndex = column.getCellIndex(); - - FilteredRows filteredRows = engine.getAllFilteredRows(false); - filteredRows.accept(_project, new RowVisitor() { - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - if (_cellIndex < row.cells.size()) { - Cell cell = row.cells.get(_cellIndex); - if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) { - _entries.add(new ReconEntry(rowIndex, cell)); - } - } - return false; - } - }); - } - - public void run() { - try { - populateEntries(); - } catch (Exception e2) { - // TODO : Not sure what to do here? - e2.printStackTrace(); - } - - Map jobKeyToGroup = new HashMap(); - - for (ReconEntry entry : _entries) { - ReconJob job = _reconConfig.createJob( - _project, - entry.rowIndex, - _project.rows.get(entry.rowIndex), - _columnName, - entry.cell - ); - - int key = job.getKey(); - JobGroup group = jobKeyToGroup.get(key); - if (group == null) { - group = new JobGroup(job); - jobKeyToGroup.put(key, group); - } - group.entries.add(entry); - } - - List cellChanges = new ArrayList(_entries.size()); - List groups = new ArrayList(jobKeyToGroup.values()); - - int batchSize = _reconConfig.getBatchSize(); - for (int i = 0; i < groups.size(); i += batchSize) { - int to = Math.min(i + batchSize, groups.size()); - - List jobs = new ArrayList(to - i); - for (int j = i; j < to; j++) { - jobs.add(groups.get(j).job); - } - - List recons = _reconConfig.batchRecon(jobs, _historyEntryID); - for (int j = i; j < to; j++) { - Recon recon = recons.get(j - i); - List entries = groups.get(j).entries; - - if (recon != null) { - recon.judgmentBatchSize = entries.size(); - } - - for (ReconEntry entry : entries) { - Cell oldCell = entry.cell; - Cell newCell = new Cell(oldCell.value, recon); - - CellChange cellChange = new CellChange( - entry.rowIndex, - _cellIndex, - oldCell, - newCell - ); - cellChanges.add(cellChange); - } - } - - _progress = i * 100 / groups.size(); - try { - Thread.sleep(50); - } catch (InterruptedException e) { - if (_canceled) { - break; - } - } - } - - if (!_canceled) { - Change reconChange = new ReconChange( - cellChanges, - _columnName, - _reconConfig, - null - ); - - HistoryEntry historyEntry = new HistoryEntry( - _historyEntryID, - _project, - _description, - ReconOperation.this, - reconChange - ); - - _project.history.addEntry(historyEntry); - _project.processManager.onDoneProcess(this); - } - } - } -} +package com.metaweb.gridworks.operations; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.CellChange; +import com.metaweb.gridworks.model.changes.ReconChange; +import com.metaweb.gridworks.model.recon.HeuristicReconConfig; +import com.metaweb.gridworks.model.recon.ReconConfig; +import com.metaweb.gridworks.model.recon.ReconJob; +import com.metaweb.gridworks.process.LongRunningProcess; +import com.metaweb.gridworks.process.Process; + +public class ReconOperation extends EngineDependentOperation { + final protected String _columnName; + final protected ReconConfig _reconConfig; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + + return new ReconOperation( + engineConfig, + obj.getString("columnName"), + ReconConfig.reconstruct(obj.getJSONObject("config")) + ); + } + + public ReconOperation( + JSONObject engineConfig, + String columnName, + ReconConfig reconConfig + ) { + super(engineConfig); + _columnName = columnName; + _reconConfig = reconConfig; + } + + public Process createProcess(Project project, Properties options) throws Exception { + return new ReconProcess( + project, + getEngineConfig(), + getBriefDescription(null) + ); + } + + protected String getBriefDescription(Project project) { + return _reconConfig.getBriefDescription(project, _columnName); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("columnName"); writer.value(_columnName); + writer.key("config"); _reconConfig.write(writer, options); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.endObject(); + } + + static protected class ReconEntry { + final public int rowIndex; + final public Cell cell; + + public ReconEntry(int rowIndex, Cell cell) { + this.rowIndex = rowIndex; + this.cell = cell; + } + } + static protected class JobGroup { + final public ReconJob job; + final public List entries = new ArrayList(); + + public JobGroup(ReconJob job) { + this.job = job; + } + } + + public class ReconProcess extends LongRunningProcess implements Runnable { + final protected Project _project; + final protected JSONObject _engineConfig; + final protected long _historyEntryID; + protected List _entries; + protected int _cellIndex; + + public ReconProcess( + Project project, + JSONObject engineConfig, + String description + ) { + super(description); + _project = project; + _engineConfig = engineConfig; + _historyEntryID = HistoryEntry.allocateID(); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("id"); writer.value(hashCode()); + writer.key("description"); writer.value(_description); + writer.key("immediate"); writer.value(false); + writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done")); + writer.key("progress"); writer.value(_progress); + writer.key("onDone"); + writer.array(); + writer.object(); + writer.key("action"); writer.value("createFacet"); + writer.key("facetType"); writer.value("list"); + writer.key("facetConfig"); + writer.object(); + writer.key("name"); writer.value(_columnName + ": judgment"); + writer.key("columnName"); writer.value(_columnName); + writer.key("expression"); writer.value("cell.recon.judgment"); + writer.key("omitError"); writer.value(true); + writer.endObject(); + writer.key("facetOptions"); + writer.object(); + writer.key("scroll"); writer.value(false); + writer.endObject(); + writer.endObject(); + + if (_reconConfig instanceof HeuristicReconConfig) { + writer.object(); + writer.key("action"); writer.value("createFacet"); + writer.key("facetType"); writer.value("range"); + writer.key("facetConfig"); + writer.object(); + writer.key("name"); writer.value(_columnName + ": best candidate's score"); + writer.key("columnName"); writer.value(_columnName); + writer.key("expression"); writer.value("cell.recon.best.score"); + writer.key("mode"); writer.value("range"); + writer.endObject(); + writer.endObject(); + } + writer.endArray(); + writer.endObject(); + } + + protected Runnable getRunnable() { + return this; + } + + protected void populateEntries() throws Exception { + Engine engine = new Engine(_project); + engine.initializeFromJSON(_engineConfig); + + Column column = _project.columnModel.getColumnByName(_columnName); + if (column == null) { + throw new Exception("No column named " + _columnName); + } + + _entries = new ArrayList(_project.rows.size()); + _cellIndex = column.getCellIndex(); + + FilteredRows filteredRows = engine.getAllFilteredRows(false); + filteredRows.accept(_project, new RowVisitor() { + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + if (_cellIndex < row.cells.size()) { + Cell cell = row.cells.get(_cellIndex); + if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) { + _entries.add(new ReconEntry(rowIndex, cell)); + } + } + return false; + } + }); + } + + public void run() { + try { + populateEntries(); + } catch (Exception e2) { + // TODO : Not sure what to do here? + e2.printStackTrace(); + } + + Map jobKeyToGroup = new HashMap(); + + for (ReconEntry entry : _entries) { + ReconJob job = _reconConfig.createJob( + _project, + entry.rowIndex, + _project.rows.get(entry.rowIndex), + _columnName, + entry.cell + ); + + int key = job.getKey(); + JobGroup group = jobKeyToGroup.get(key); + if (group == null) { + group = new JobGroup(job); + jobKeyToGroup.put(key, group); + } + group.entries.add(entry); + } + + List cellChanges = new ArrayList(_entries.size()); + List groups = new ArrayList(jobKeyToGroup.values()); + + int batchSize = _reconConfig.getBatchSize(); + for (int i = 0; i < groups.size(); i += batchSize) { + int to = Math.min(i + batchSize, groups.size()); + + List jobs = new ArrayList(to - i); + for (int j = i; j < to; j++) { + jobs.add(groups.get(j).job); + } + + List recons = _reconConfig.batchRecon(jobs, _historyEntryID); + for (int j = i; j < to; j++) { + Recon recon = recons.get(j - i); + List entries = groups.get(j).entries; + + if (recon != null) { + recon.judgmentBatchSize = entries.size(); + } + + for (ReconEntry entry : entries) { + Cell oldCell = entry.cell; + Cell newCell = new Cell(oldCell.value, recon); + + CellChange cellChange = new CellChange( + entry.rowIndex, + _cellIndex, + oldCell, + newCell + ); + cellChanges.add(cellChange); + } + } + + _progress = i * 100 / groups.size(); + try { + Thread.sleep(50); + } catch (InterruptedException e) { + if (_canceled) { + break; + } + } + } + + if (!_canceled) { + Change reconChange = new ReconChange( + cellChanges, + _columnName, + _reconConfig, + null + ); + + HistoryEntry historyEntry = new HistoryEntry( + _historyEntryID, + _project, + _description, + ReconOperation.this, + reconChange + ); + + _project.history.addEntry(historyEntry); + _project.processManager.onDoneProcess(this); + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/RowFlagOperation.java b/src/main/java/com/metaweb/gridworks/operations/RowFlagOperation.java index aa7a5af3b..c885f1644 100644 --- a/src/main/java/com/metaweb/gridworks/operations/RowFlagOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/RowFlagOperation.java @@ -1,91 +1,91 @@ -package com.metaweb.gridworks.operations; - -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.MassChange; -import com.metaweb.gridworks.model.changes.RowFlagChange; - -public class RowFlagOperation extends EngineDependentOperation { - final protected boolean _flagged; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - boolean flagged = obj.getBoolean("flagged"); - - return new RowFlagOperation( - engineConfig, - flagged - ); - } - - public RowFlagOperation(JSONObject engineConfig, boolean flagged) { - super(engineConfig); - _flagged = flagged; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("flagged"); writer.value(_flagged); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return (_flagged ? "Flag rows" : "Unflag rows"); - } - - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - Engine engine = createEngine(project); - - List changes = new ArrayList(project.rows.size()); - - FilteredRows filteredRows = engine.getAllFilteredRows(false); - filteredRows.accept(project, createRowVisitor(project, changes)); - - return new HistoryEntry( - historyEntryID, - project, - (_flagged ? "Flag" : "Unflag") + " " + changes.size() + " rows", - this, - new MassChange(changes, false) - ); - } - - protected RowVisitor createRowVisitor(Project project, List changes) throws Exception { - return new RowVisitor() { - List changes; - - public RowVisitor init(List changes) { - this.changes = changes; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - if (row.flagged != _flagged) { - RowFlagChange change = new RowFlagChange(rowIndex, _flagged); - - changes.add(change); - } - return false; - } - }.init(changes); - } -} +package com.metaweb.gridworks.operations; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.MassChange; +import com.metaweb.gridworks.model.changes.RowFlagChange; + +public class RowFlagOperation extends EngineDependentOperation { + final protected boolean _flagged; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + boolean flagged = obj.getBoolean("flagged"); + + return new RowFlagOperation( + engineConfig, + flagged + ); + } + + public RowFlagOperation(JSONObject engineConfig, boolean flagged) { + super(engineConfig); + _flagged = flagged; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("flagged"); writer.value(_flagged); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return (_flagged ? "Flag rows" : "Unflag rows"); + } + + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + Engine engine = createEngine(project); + + List changes = new ArrayList(project.rows.size()); + + FilteredRows filteredRows = engine.getAllFilteredRows(false); + filteredRows.accept(project, createRowVisitor(project, changes)); + + return new HistoryEntry( + historyEntryID, + project, + (_flagged ? "Flag" : "Unflag") + " " + changes.size() + " rows", + this, + new MassChange(changes, false) + ); + } + + protected RowVisitor createRowVisitor(Project project, List changes) throws Exception { + return new RowVisitor() { + List changes; + + public RowVisitor init(List changes) { + this.changes = changes; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + if (row.flagged != _flagged) { + RowFlagChange change = new RowFlagChange(rowIndex, _flagged); + + changes.add(change); + } + return false; + } + }.init(changes); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/RowRemovalOperation.java b/src/main/java/com/metaweb/gridworks/operations/RowRemovalOperation.java index f750d11ce..c08753a2e 100644 --- a/src/main/java/com/metaweb/gridworks/operations/RowRemovalOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/RowRemovalOperation.java @@ -1,81 +1,81 @@ -package com.metaweb.gridworks.operations; - - import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.RowRemovalChange; - -public class RowRemovalOperation extends EngineDependentOperation { - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - - return new RowRemovalOperation( - engineConfig - ); - } - - public RowRemovalOperation(JSONObject engineConfig) { - super(engineConfig); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Remove rows"; - } - - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - Engine engine = createEngine(project); - - List rowIndices = new ArrayList(); - - FilteredRows filteredRows = engine.getAllFilteredRows(false); - filteredRows.accept(project, createRowVisitor(project, rowIndices)); - - return new HistoryEntry( - historyEntryID, - project, - "Remove " + rowIndices.size() + " rows", - this, - new RowRemovalChange(rowIndices) - ); - } - - protected RowVisitor createRowVisitor(Project project, List rowIndices) throws Exception { - return new RowVisitor() { - List rowIndices; - - public RowVisitor init(List rowIndices) { - this.rowIndices = rowIndices; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - if (!includeContextual) { - rowIndices.add(rowIndex); - } - return false; - } - }.init(rowIndices); - } -} +package com.metaweb.gridworks.operations; + + import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.RowRemovalChange; + +public class RowRemovalOperation extends EngineDependentOperation { + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + + return new RowRemovalOperation( + engineConfig + ); + } + + public RowRemovalOperation(JSONObject engineConfig) { + super(engineConfig); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Remove rows"; + } + + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + Engine engine = createEngine(project); + + List rowIndices = new ArrayList(); + + FilteredRows filteredRows = engine.getAllFilteredRows(false); + filteredRows.accept(project, createRowVisitor(project, rowIndices)); + + return new HistoryEntry( + historyEntryID, + project, + "Remove " + rowIndices.size() + " rows", + this, + new RowRemovalChange(rowIndices) + ); + } + + protected RowVisitor createRowVisitor(Project project, List rowIndices) throws Exception { + return new RowVisitor() { + List rowIndices; + + public RowVisitor init(List rowIndices) { + this.rowIndices = rowIndices; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + if (!includeContextual) { + rowIndices.add(rowIndex); + } + return false; + } + }.init(rowIndices); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/RowStarOperation.java b/src/main/java/com/metaweb/gridworks/operations/RowStarOperation.java index 74439b722..ba188a957 100644 --- a/src/main/java/com/metaweb/gridworks/operations/RowStarOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/RowStarOperation.java @@ -1,91 +1,91 @@ -package com.metaweb.gridworks.operations; - - import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.Engine; -import com.metaweb.gridworks.browsing.FilteredRows; -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.MassChange; -import com.metaweb.gridworks.model.changes.RowStarChange; - -public class RowStarOperation extends EngineDependentOperation { - final protected boolean _starred; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - boolean starred = obj.getBoolean("starred"); - - return new RowStarOperation( - engineConfig, - starred - ); - } - - public RowStarOperation(JSONObject engineConfig, boolean starred) { - super(engineConfig); - _starred = starred; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("starred"); writer.value(_starred); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return (_starred ? "Star rows" : "Unstar rows"); - } - - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - Engine engine = createEngine(project); - - List changes = new ArrayList(project.rows.size()); - - FilteredRows filteredRows = engine.getAllFilteredRows(false); - filteredRows.accept(project, createRowVisitor(project, changes)); - - return new HistoryEntry( - historyEntryID, - project, - (_starred ? "Star" : "Unstar") + " " + changes.size() + " rows", - this, - new MassChange(changes, false) - ); - } - - protected RowVisitor createRowVisitor(Project project, List changes) throws Exception { - return new RowVisitor() { - List changes; - - public RowVisitor init(List changes) { - this.changes = changes; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - if (row.starred != _starred) { - RowStarChange change = new RowStarChange(rowIndex, _starred); - - changes.add(change); - } - return false; - } - }.init(changes); - } -} +package com.metaweb.gridworks.operations; + + import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.Engine; +import com.metaweb.gridworks.browsing.FilteredRows; +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.MassChange; +import com.metaweb.gridworks.model.changes.RowStarChange; + +public class RowStarOperation extends EngineDependentOperation { + final protected boolean _starred; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + boolean starred = obj.getBoolean("starred"); + + return new RowStarOperation( + engineConfig, + starred + ); + } + + public RowStarOperation(JSONObject engineConfig, boolean starred) { + super(engineConfig); + _starred = starred; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("starred"); writer.value(_starred); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return (_starred ? "Star rows" : "Unstar rows"); + } + + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + Engine engine = createEngine(project); + + List changes = new ArrayList(project.rows.size()); + + FilteredRows filteredRows = engine.getAllFilteredRows(false); + filteredRows.accept(project, createRowVisitor(project, changes)); + + return new HistoryEntry( + historyEntryID, + project, + (_starred ? "Star" : "Unstar") + " " + changes.size() + " rows", + this, + new MassChange(changes, false) + ); + } + + protected RowVisitor createRowVisitor(Project project, List changes) throws Exception { + return new RowVisitor() { + List changes; + + public RowVisitor init(List changes) { + this.changes = changes; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + if (row.starred != _starred) { + RowStarChange change = new RowStarChange(rowIndex, _starred); + + changes.add(change); + } + return false; + } + }.init(changes); + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/SaveProtographOperation.java b/src/main/java/com/metaweb/gridworks/operations/SaveProtographOperation.java index e43a75283..8e50a3c99 100644 --- a/src/main/java/com/metaweb/gridworks/operations/SaveProtographOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/SaveProtographOperation.java @@ -1,117 +1,117 @@ -package com.metaweb.gridworks.operations; - -import java.io.IOException; -import java.io.LineNumberReader; -import java.io.Writer; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.history.Change; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.protograph.Protograph; -import com.metaweb.gridworks.util.ParsingUtilities; -import com.metaweb.gridworks.util.Pool; - -public class SaveProtographOperation extends AbstractOperation { - final protected Protograph _protograph; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - return new SaveProtographOperation( - Protograph.reconstruct(obj.getJSONObject("protograph")) - ); - } - - public SaveProtographOperation(Protograph protograph) { - _protograph = protograph; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value("Save protograph"); - writer.key("protograph"); _protograph.write(writer, options); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Save schema skeleton"; - } - - @Override - protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { - String description = "Save schema-alignment protograph"; - - Change change = new ProtographChange(_protograph); - - return new HistoryEntry(historyEntryID, project, description, SaveProtographOperation.this, change); - } - - static public class ProtographChange implements Change { - final protected Protograph _newProtograph; - protected Protograph _oldProtograph; - - public ProtographChange(Protograph protograph) { - _newProtograph = protograph; - } - - public void apply(Project project) { - synchronized (project) { - _oldProtograph = project.protograph; - project.protograph = _newProtograph; - } - } - - public void revert(Project project) { - synchronized (project) { - project.protograph = _oldProtograph; - } - } - - public void save(Writer writer, Properties options) throws IOException { - writer.write("newProtograph="); writeProtograph(_newProtograph, writer); writer.write('\n'); - writer.write("oldProtograph="); writeProtograph(_oldProtograph, writer); writer.write('\n'); - writer.write("/ec/\n"); // end of change marker - } - - static public Change load(LineNumberReader reader, Pool pool) throws Exception { - Protograph oldProtograph = null; - Protograph newProtograph = null; - - String line; - while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { - int equal = line.indexOf('='); - CharSequence field = line.subSequence(0, equal); - String value = line.substring(equal + 1); - - if ("oldProtograph".equals(field) && value.length() > 0) { - oldProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); - } else if ("newProtograph".equals(field) && value.length() > 0) { - newProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); - } - } - - ProtographChange change = new ProtographChange(newProtograph); - change._oldProtograph = oldProtograph; - - return change; - } - - static protected void writeProtograph(Protograph p, Writer writer) throws IOException { - if (p != null) { - JSONWriter jsonWriter = new JSONWriter(writer); - try { - p.write(jsonWriter, new Properties()); - } catch (JSONException e) { - e.printStackTrace(); - } - } - } - } -} +package com.metaweb.gridworks.operations; + +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Writer; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.history.Change; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.protograph.Protograph; +import com.metaweb.gridworks.util.ParsingUtilities; +import com.metaweb.gridworks.util.Pool; + +public class SaveProtographOperation extends AbstractOperation { + final protected Protograph _protograph; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + return new SaveProtographOperation( + Protograph.reconstruct(obj.getJSONObject("protograph")) + ); + } + + public SaveProtographOperation(Protograph protograph) { + _protograph = protograph; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value("Save protograph"); + writer.key("protograph"); _protograph.write(writer, options); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Save schema skeleton"; + } + + @Override + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + String description = "Save schema-alignment protograph"; + + Change change = new ProtographChange(_protograph); + + return new HistoryEntry(historyEntryID, project, description, SaveProtographOperation.this, change); + } + + static public class ProtographChange implements Change { + final protected Protograph _newProtograph; + protected Protograph _oldProtograph; + + public ProtographChange(Protograph protograph) { + _newProtograph = protograph; + } + + public void apply(Project project) { + synchronized (project) { + _oldProtograph = project.protograph; + project.protograph = _newProtograph; + } + } + + public void revert(Project project) { + synchronized (project) { + project.protograph = _oldProtograph; + } + } + + public void save(Writer writer, Properties options) throws IOException { + writer.write("newProtograph="); writeProtograph(_newProtograph, writer); writer.write('\n'); + writer.write("oldProtograph="); writeProtograph(_oldProtograph, writer); writer.write('\n'); + writer.write("/ec/\n"); // end of change marker + } + + static public Change load(LineNumberReader reader, Pool pool) throws Exception { + Protograph oldProtograph = null; + Protograph newProtograph = null; + + String line; + while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { + int equal = line.indexOf('='); + CharSequence field = line.subSequence(0, equal); + String value = line.substring(equal + 1); + + if ("oldProtograph".equals(field) && value.length() > 0) { + oldProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); + } else if ("newProtograph".equals(field) && value.length() > 0) { + newProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); + } + } + + ProtographChange change = new ProtographChange(newProtograph); + change._oldProtograph = oldProtograph; + + return change; + } + + static protected void writeProtograph(Protograph p, Writer writer) throws IOException { + if (p != null) { + JSONWriter jsonWriter = new JSONWriter(writer); + try { + p.write(jsonWriter, new Properties()); + } catch (JSONException e) { + e.printStackTrace(); + } + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/operations/TextTransformOperation.java b/src/main/java/com/metaweb/gridworks/operations/TextTransformOperation.java index ce13f48bd..9b30ee939 100644 --- a/src/main/java/com/metaweb/gridworks/operations/TextTransformOperation.java +++ b/src/main/java/com/metaweb/gridworks/operations/TextTransformOperation.java @@ -1,176 +1,176 @@ -package com.metaweb.gridworks.operations; - -import java.io.Serializable; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.browsing.RowVisitor; -import com.metaweb.gridworks.expr.Evaluable; -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.expr.MetaParser; -import com.metaweb.gridworks.expr.WrappedCell; -import com.metaweb.gridworks.model.AbstractOperation; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.changes.CellChange; - -public class TextTransformOperation extends EngineDependentMassCellOperation { - final protected String _expression; - final protected OnError _onError; - final protected boolean _repeat; - final protected int _repeatCount; - - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - - return new TextTransformOperation( - engineConfig, - obj.getString("columnName"), - obj.getString("expression"), - stringToOnError(obj.getString("onError")), - obj.getBoolean("repeat"), - obj.getInt("repeatCount") - ); - } - - static public OnError stringToOnError(String s) { - if ("set-to-blank".equalsIgnoreCase(s)) { - return OnError.SetToBlank; - } else if ("store-error".equalsIgnoreCase(s)) { - return OnError.StoreError; - } else { - return OnError.KeepOriginal; - } - } - static public String onErrorToString(OnError onError) { - if (onError == OnError.SetToBlank) { - return "set-to-blank"; - } else if (onError == OnError.StoreError) { - return "store-error"; - } else { - return "keep-original"; - } - } - - public TextTransformOperation( - JSONObject engineConfig, - String columnName, - String expression, - OnError onError, - boolean repeat, - int repeatCount - ) { - super(engineConfig, columnName, true); - _expression = expression; - _onError = onError; - _repeat = repeat; - _repeatCount = repeatCount; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); - writer.key("description"); writer.value(getBriefDescription(null)); - writer.key("engineConfig"); writer.value(getEngineConfig()); - writer.key("columnName"); writer.value(_columnName); - writer.key("expression"); writer.value(_expression); - writer.key("onError"); writer.value(onErrorToString(_onError)); - writer.key("repeat"); writer.value(_repeat); - writer.key("repeatCount"); writer.value(_repeatCount); - writer.endObject(); - } - - protected String getBriefDescription(Project project) { - return "Text transform on cells in column " + _columnName + " using expression " + _expression; - } - - protected String createDescription(Column column, - List cellChanges) { - - return "Text transform on " + cellChanges.size() + - " cells in column " + column.getName() + ": " + _expression; - } - - protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { - Column column = project.columnModel.getColumnByName(_columnName); - - Evaluable eval = MetaParser.parse(_expression); - Properties bindings = ExpressionUtils.createBindings(project); - - return new RowVisitor() { - int cellIndex; - Properties bindings; - List cellChanges; - Evaluable eval; - - public RowVisitor init(int cellIndex, Properties bindings, List cellChanges, Evaluable eval) { - this.cellIndex = cellIndex; - this.bindings = bindings; - this.cellChanges = cellChanges; - this.eval = eval; - return this; - } - - public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { - Cell cell = row.getCell(cellIndex); - Cell newCell = null; - - Object oldValue = cell != null ? cell.value : null; - - ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); - - Object o = eval.evaluate(bindings); - if (o != null) { - if (o instanceof Cell) { - newCell = (Cell) o; - } else if (o instanceof WrappedCell) { - newCell = ((WrappedCell) o).cell; - } else { - Serializable newValue = ExpressionUtils.wrapStorable(o); - if (ExpressionUtils.isError(newValue)) { - if (_onError == OnError.KeepOriginal) { - return false; - } else if (_onError == OnError.SetToBlank) { - newValue = null; - } - } - - if (!ExpressionUtils.sameValue(oldValue, newValue)) { - newCell = new Cell(newValue, (cell != null) ? cell.recon : null); - - if (_repeat) { - for (int i = 0; i < _repeatCount; i++) { - ExpressionUtils.bind(bindings, row, rowIndex, _columnName, newCell); - - newValue = ExpressionUtils.wrapStorable(eval.evaluate(bindings)); - if (ExpressionUtils.isError(newValue)) { - break; - } else if (ExpressionUtils.sameValue(newCell.value, newValue)) { - break; - } - - newCell = new Cell(newValue, newCell.recon); - } - } - } - } - - if (newCell != null) { - CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); - cellChanges.add(cellChange); - } - } - - return false; - } - }.init(column.getCellIndex(), bindings, cellChanges, eval); - } -} +package com.metaweb.gridworks.operations; + +import java.io.Serializable; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.browsing.RowVisitor; +import com.metaweb.gridworks.expr.Evaluable; +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.expr.MetaParser; +import com.metaweb.gridworks.expr.WrappedCell; +import com.metaweb.gridworks.model.AbstractOperation; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.changes.CellChange; + +public class TextTransformOperation extends EngineDependentMassCellOperation { + final protected String _expression; + final protected OnError _onError; + final protected boolean _repeat; + final protected int _repeatCount; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + + return new TextTransformOperation( + engineConfig, + obj.getString("columnName"), + obj.getString("expression"), + stringToOnError(obj.getString("onError")), + obj.getBoolean("repeat"), + obj.getInt("repeatCount") + ); + } + + static public OnError stringToOnError(String s) { + if ("set-to-blank".equalsIgnoreCase(s)) { + return OnError.SetToBlank; + } else if ("store-error".equalsIgnoreCase(s)) { + return OnError.StoreError; + } else { + return OnError.KeepOriginal; + } + } + static public String onErrorToString(OnError onError) { + if (onError == OnError.SetToBlank) { + return "set-to-blank"; + } else if (onError == OnError.StoreError) { + return "store-error"; + } else { + return "keep-original"; + } + } + + public TextTransformOperation( + JSONObject engineConfig, + String columnName, + String expression, + OnError onError, + boolean repeat, + int repeatCount + ) { + super(engineConfig, columnName, true); + _expression = expression; + _onError = onError; + _repeat = repeat; + _repeatCount = repeatCount; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("columnName"); writer.value(_columnName); + writer.key("expression"); writer.value(_expression); + writer.key("onError"); writer.value(onErrorToString(_onError)); + writer.key("repeat"); writer.value(_repeat); + writer.key("repeatCount"); writer.value(_repeatCount); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Text transform on cells in column " + _columnName + " using expression " + _expression; + } + + protected String createDescription(Column column, + List cellChanges) { + + return "Text transform on " + cellChanges.size() + + " cells in column " + column.getName() + ": " + _expression; + } + + protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { + Column column = project.columnModel.getColumnByName(_columnName); + + Evaluable eval = MetaParser.parse(_expression); + Properties bindings = ExpressionUtils.createBindings(project); + + return new RowVisitor() { + int cellIndex; + Properties bindings; + List cellChanges; + Evaluable eval; + + public RowVisitor init(int cellIndex, Properties bindings, List cellChanges, Evaluable eval) { + this.cellIndex = cellIndex; + this.bindings = bindings; + this.cellChanges = cellChanges; + this.eval = eval; + return this; + } + + public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { + Cell cell = row.getCell(cellIndex); + Cell newCell = null; + + Object oldValue = cell != null ? cell.value : null; + + ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell); + + Object o = eval.evaluate(bindings); + if (o != null) { + if (o instanceof Cell) { + newCell = (Cell) o; + } else if (o instanceof WrappedCell) { + newCell = ((WrappedCell) o).cell; + } else { + Serializable newValue = ExpressionUtils.wrapStorable(o); + if (ExpressionUtils.isError(newValue)) { + if (_onError == OnError.KeepOriginal) { + return false; + } else if (_onError == OnError.SetToBlank) { + newValue = null; + } + } + + if (!ExpressionUtils.sameValue(oldValue, newValue)) { + newCell = new Cell(newValue, (cell != null) ? cell.recon : null); + + if (_repeat) { + for (int i = 0; i < _repeatCount; i++) { + ExpressionUtils.bind(bindings, row, rowIndex, _columnName, newCell); + + newValue = ExpressionUtils.wrapStorable(eval.evaluate(bindings)); + if (ExpressionUtils.isError(newValue)) { + break; + } else if (ExpressionUtils.sameValue(newCell.value, newValue)) { + break; + } + + newCell = new Cell(newValue, newCell.recon); + } + } + } + } + + if (newCell != null) { + CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); + cellChanges.add(cellChange); + } + } + + return false; + } + }.init(column.getCellIndex(), bindings, cellChanges, eval); + } +} diff --git a/src/main/java/com/metaweb/gridworks/process/LongRunningProcess.java b/src/main/java/com/metaweb/gridworks/process/LongRunningProcess.java index cbceab887..7bc20709f 100644 --- a/src/main/java/com/metaweb/gridworks/process/LongRunningProcess.java +++ b/src/main/java/com/metaweb/gridworks/process/LongRunningProcess.java @@ -1,71 +1,71 @@ -package com.metaweb.gridworks.process; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.history.HistoryEntry; - -abstract public class LongRunningProcess extends Process { - final protected String _description; - protected ProcessManager _manager; - protected Thread _thread; - protected int _progress; // out of 100 - protected boolean _canceled; - - protected LongRunningProcess(String description) { - _description = description; - } - - public void cancel() { - _canceled = true; - if (_thread != null && _thread.isAlive()) { - _thread.interrupt(); - } - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("id"); writer.value(hashCode()); - writer.key("description"); writer.value(_description); - writer.key("immediate"); writer.value(false); - writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done")); - writer.key("progress"); writer.value(_progress); - writer.endObject(); - } - - @Override - public boolean isImmediate() { - return false; - } - - @Override - public boolean isRunning() { - return _thread != null && _thread.isAlive(); - } - - @Override - public boolean isDone() { - return _thread != null && !_thread.isAlive(); - } - - @Override - public HistoryEntry performImmediate() { - throw new RuntimeException("Not an immediate process"); - } - - @Override - public void startPerforming(ProcessManager manager) { - if (_thread == null) { - _manager = manager; - - _thread = new Thread(getRunnable()); - _thread.start(); - } - } - - abstract protected Runnable getRunnable(); -} +package com.metaweb.gridworks.process; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.history.HistoryEntry; + +abstract public class LongRunningProcess extends Process { + final protected String _description; + protected ProcessManager _manager; + protected Thread _thread; + protected int _progress; // out of 100 + protected boolean _canceled; + + protected LongRunningProcess(String description) { + _description = description; + } + + public void cancel() { + _canceled = true; + if (_thread != null && _thread.isAlive()) { + _thread.interrupt(); + } + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("id"); writer.value(hashCode()); + writer.key("description"); writer.value(_description); + writer.key("immediate"); writer.value(false); + writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done")); + writer.key("progress"); writer.value(_progress); + writer.endObject(); + } + + @Override + public boolean isImmediate() { + return false; + } + + @Override + public boolean isRunning() { + return _thread != null && _thread.isAlive(); + } + + @Override + public boolean isDone() { + return _thread != null && !_thread.isAlive(); + } + + @Override + public HistoryEntry performImmediate() { + throw new RuntimeException("Not an immediate process"); + } + + @Override + public void startPerforming(ProcessManager manager) { + if (_thread == null) { + _manager = manager; + + _thread = new Thread(getRunnable()); + _thread.start(); + } + } + + abstract protected Runnable getRunnable(); +} diff --git a/src/main/java/com/metaweb/gridworks/process/Process.java b/src/main/java/com/metaweb/gridworks/process/Process.java index 94d0089c7..cd4f0cdb7 100644 --- a/src/main/java/com/metaweb/gridworks/process/Process.java +++ b/src/main/java/com/metaweb/gridworks/process/Process.java @@ -1,16 +1,16 @@ -package com.metaweb.gridworks.process; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.history.HistoryEntry; - -public abstract class Process implements Jsonizable { - abstract public boolean isImmediate(); - - abstract public boolean isRunning(); - abstract public boolean isDone(); - - abstract public HistoryEntry performImmediate() throws Exception; - - abstract public void startPerforming(ProcessManager manager); - abstract public void cancel(); -} +package com.metaweb.gridworks.process; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.history.HistoryEntry; + +public abstract class Process implements Jsonizable { + abstract public boolean isImmediate(); + + abstract public boolean isRunning(); + abstract public boolean isDone(); + + abstract public HistoryEntry performImmediate() throws Exception; + + abstract public void startPerforming(ProcessManager manager); + abstract public void cancel(); +} diff --git a/src/main/java/com/metaweb/gridworks/process/ProcessManager.java b/src/main/java/com/metaweb/gridworks/process/ProcessManager.java index 2eaf1f8c8..37298d034 100644 --- a/src/main/java/com/metaweb/gridworks/process/ProcessManager.java +++ b/src/main/java/com/metaweb/gridworks/process/ProcessManager.java @@ -1,105 +1,105 @@ -package com.metaweb.gridworks.process; - -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.history.HistoryProcess; - -public class ProcessManager implements Jsonizable { - protected List _processes = new LinkedList(); - - public ProcessManager() { - - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("processes"); writer.array(); - for (Process p : _processes) { - p.write(writer, options); - } - writer.endArray(); - - writer.endObject(); - } - - public HistoryEntry queueProcess(Process process) { - if (process.isImmediate() && _processes.size() == 0) { - try { - return process.performImmediate(); - } catch (Exception e) { - // TODO: Not sure what to do yet - e.printStackTrace(); - } - } else { - _processes.add(process); - - update(); - } - return null; - } - - public boolean queueProcess(HistoryProcess process) { - if (process.isImmediate() && _processes.size() == 0) { - try { - return process.performImmediate() != null; - } catch (Exception e) { - // TODO: Not sure what to do yet - e.printStackTrace(); - } - } else { - _processes.add(process); - - update(); - } - return false; - } - - public boolean hasPending() { - return _processes.size() > 0; - } - - public void onDoneProcess(Process p) { - _processes.remove(p); - update(); - } - - public void cancelAll() { - for (Process p : _processes) { - if (!p.isImmediate() && p.isRunning()) { - p.cancel(); - } - } - _processes.clear(); - } - - protected void update() { - while (_processes.size() > 0) { - Process p = _processes.get(0); - if (p.isImmediate()) { - try { - p.performImmediate(); - } catch (Exception e) { - // TODO: Not sure what to do yet - e.printStackTrace(); - } - _processes.remove(0); - } else if (p.isDone()) { - _processes.remove(0); - } else { - if (!p.isRunning()) { - p.startPerforming(this); - } - break; - } - } - } -} +package com.metaweb.gridworks.process; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.history.HistoryProcess; + +public class ProcessManager implements Jsonizable { + protected List _processes = new LinkedList(); + + public ProcessManager() { + + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("processes"); writer.array(); + for (Process p : _processes) { + p.write(writer, options); + } + writer.endArray(); + + writer.endObject(); + } + + public HistoryEntry queueProcess(Process process) { + if (process.isImmediate() && _processes.size() == 0) { + try { + return process.performImmediate(); + } catch (Exception e) { + // TODO: Not sure what to do yet + e.printStackTrace(); + } + } else { + _processes.add(process); + + update(); + } + return null; + } + + public boolean queueProcess(HistoryProcess process) { + if (process.isImmediate() && _processes.size() == 0) { + try { + return process.performImmediate() != null; + } catch (Exception e) { + // TODO: Not sure what to do yet + e.printStackTrace(); + } + } else { + _processes.add(process); + + update(); + } + return false; + } + + public boolean hasPending() { + return _processes.size() > 0; + } + + public void onDoneProcess(Process p) { + _processes.remove(p); + update(); + } + + public void cancelAll() { + for (Process p : _processes) { + if (!p.isImmediate() && p.isRunning()) { + p.cancel(); + } + } + _processes.clear(); + } + + protected void update() { + while (_processes.size() > 0) { + Process p = _processes.get(0); + if (p.isImmediate()) { + try { + p.performImmediate(); + } catch (Exception e) { + // TODO: Not sure what to do yet + e.printStackTrace(); + } + _processes.remove(0); + } else if (p.isDone()) { + _processes.remove(0); + } else { + if (!p.isRunning()) { + p.startPerforming(this); + } + break; + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/process/QuickHistoryEntryProcess.java b/src/main/java/com/metaweb/gridworks/process/QuickHistoryEntryProcess.java index 47cc65a45..67d548008 100644 --- a/src/main/java/com/metaweb/gridworks/process/QuickHistoryEntryProcess.java +++ b/src/main/java/com/metaweb/gridworks/process/QuickHistoryEntryProcess.java @@ -1,66 +1,66 @@ -package com.metaweb.gridworks.process; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.history.HistoryEntry; -import com.metaweb.gridworks.model.Project; - -abstract public class QuickHistoryEntryProcess extends Process { - final protected Project _project; - final protected String _briefDescription; - protected HistoryEntry _historyEntry; - boolean _done = false; - - public QuickHistoryEntryProcess(Project project, String briefDescription) { - _project = project; - _briefDescription = briefDescription; - } - - public void cancel() { - throw new RuntimeException("Not a long-running process"); - } - - public boolean isImmediate() { - return true; - } - - public boolean isRunning() { - throw new RuntimeException("Not a long-running process"); - } - - public HistoryEntry performImmediate() throws Exception { - if (_historyEntry == null) { - _historyEntry = createHistoryEntry(HistoryEntry.allocateID()); - } - _project.history.addEntry(_historyEntry); - _done = true; - - return _historyEntry; - } - - public void startPerforming(ProcessManager manager) { - throw new RuntimeException("Not a long-running process"); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("id"); writer.value(hashCode()); - writer.key("description"); writer.value(_historyEntry != null ? _historyEntry.description : _briefDescription); - writer.key("immediate"); writer.value(true); - writer.key("status"); writer.value(_done ? "done" : "pending"); - writer.endObject(); - } - - - @Override - public boolean isDone() { - return _done; - } - - abstract protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception; -} +package com.metaweb.gridworks.process; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.history.HistoryEntry; +import com.metaweb.gridworks.model.Project; + +abstract public class QuickHistoryEntryProcess extends Process { + final protected Project _project; + final protected String _briefDescription; + protected HistoryEntry _historyEntry; + boolean _done = false; + + public QuickHistoryEntryProcess(Project project, String briefDescription) { + _project = project; + _briefDescription = briefDescription; + } + + public void cancel() { + throw new RuntimeException("Not a long-running process"); + } + + public boolean isImmediate() { + return true; + } + + public boolean isRunning() { + throw new RuntimeException("Not a long-running process"); + } + + public HistoryEntry performImmediate() throws Exception { + if (_historyEntry == null) { + _historyEntry = createHistoryEntry(HistoryEntry.allocateID()); + } + _project.history.addEntry(_historyEntry); + _done = true; + + return _historyEntry; + } + + public void startPerforming(ProcessManager manager) { + throw new RuntimeException("Not a long-running process"); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("id"); writer.value(hashCode()); + writer.key("description"); writer.value(_historyEntry != null ? _historyEntry.description : _briefDescription); + writer.key("immediate"); writer.value(true); + writer.key("status"); writer.value(_done ? "done" : "pending"); + writer.endObject(); + } + + + @Override + public boolean isDone() { + return _done; + } + + abstract protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception; +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/AnonymousNode.java b/src/main/java/com/metaweb/gridworks/protograph/AnonymousNode.java index 4ff4e8d93..57a441d60 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/AnonymousNode.java +++ b/src/main/java/com/metaweb/gridworks/protograph/AnonymousNode.java @@ -1,45 +1,45 @@ -package com.metaweb.gridworks.protograph; - -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class AnonymousNode implements Node, NodeWithLinks { - final public FreebaseType type; - final public List links = new LinkedList(); - - public AnonymousNode(FreebaseType type) { - this.type = type; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("anonymous"); - writer.key("type"); type.write(writer, options); - if (links != null) { - writer.key("links"); writer.array(); - for (Link link : links) { - link.write(writer, options); - } - writer.endArray(); - } - writer.endObject(); - } - - public void addLink(Link link) { - links.add(link); - } - - public Link getLink(int index) { - return links.get(index); - } - - public int getLinkCount() { - return links.size(); - } -} +package com.metaweb.gridworks.protograph; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +public class AnonymousNode implements Node, NodeWithLinks { + final public FreebaseType type; + final public List links = new LinkedList(); + + public AnonymousNode(FreebaseType type) { + this.type = type; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("anonymous"); + writer.key("type"); type.write(writer, options); + if (links != null) { + writer.key("links"); writer.array(); + for (Link link : links) { + link.write(writer, options); + } + writer.endArray(); + } + writer.endObject(); + } + + public void addLink(Link link) { + links.add(link); + } + + public Link getLink(int index) { + return links.get(index); + } + + public int getLinkCount() { + return links.size(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/CellKeyNode.java b/src/main/java/com/metaweb/gridworks/protograph/CellKeyNode.java index 7db9191fc..037a180f6 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/CellKeyNode.java +++ b/src/main/java/com/metaweb/gridworks/protograph/CellKeyNode.java @@ -1,29 +1,29 @@ -package com.metaweb.gridworks.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class CellKeyNode extends CellNode { - final public FreebaseTopic namespace; - - public CellKeyNode( - String columnName, - FreebaseTopic namespace - ) { - super(columnName); - - this.namespace = namespace; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("cell-as-key"); - writer.key("columnName"); writer.value(columnName); - writer.key("namespace"); namespace.write(writer, options); - writer.endObject(); - } -} +package com.metaweb.gridworks.protograph; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +public class CellKeyNode extends CellNode { + final public FreebaseTopic namespace; + + public CellKeyNode( + String columnName, + FreebaseTopic namespace + ) { + super(columnName); + + this.namespace = namespace; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("cell-as-key"); + writer.key("columnName"); writer.value(columnName); + writer.key("namespace"); namespace.write(writer, options); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/CellNode.java b/src/main/java/com/metaweb/gridworks/protograph/CellNode.java index c662d1c77..89672c20b 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/CellNode.java +++ b/src/main/java/com/metaweb/gridworks/protograph/CellNode.java @@ -1,11 +1,11 @@ -package com.metaweb.gridworks.protograph; - -abstract public class CellNode implements Node { - final public String columnName; - - public CellNode( - String columnName - ) { - this.columnName = columnName; - } -} +package com.metaweb.gridworks.protograph; + +abstract public class CellNode implements Node { + final public String columnName; + + public CellNode( + String columnName + ) { + this.columnName = columnName; + } +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/CellTopicNode.java b/src/main/java/com/metaweb/gridworks/protograph/CellTopicNode.java index e68b58309..e0204ff6a 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/CellTopicNode.java +++ b/src/main/java/com/metaweb/gridworks/protograph/CellTopicNode.java @@ -1,58 +1,58 @@ -package com.metaweb.gridworks.protograph; - -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class CellTopicNode extends CellNode implements NodeWithLinks { - final public boolean createForNoReconMatch; - final public FreebaseType type; - final public List links = new LinkedList(); - - public CellTopicNode( - String columnName, - boolean createForNoReconMatch, - FreebaseType type - ) { - super(columnName); - - this.createForNoReconMatch = createForNoReconMatch; - this.type = type; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("cell-as-topic"); - writer.key("columnName"); writer.value(columnName); - writer.key("createForNoReconMatch"); writer.value(createForNoReconMatch); - if (createForNoReconMatch && type != null) { - writer.key("type"); type.write(writer, options); - } - if (links != null) { - writer.key("links"); writer.array(); - for (Link link : links) { - link.write(writer, options); - } - writer.endArray(); - } - - writer.endObject(); - } - - public void addLink(Link link) { - links.add(link); - } - - public Link getLink(int index) { - return links.get(index); - } - - public int getLinkCount() { - return links.size(); - } -} +package com.metaweb.gridworks.protograph; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +public class CellTopicNode extends CellNode implements NodeWithLinks { + final public boolean createForNoReconMatch; + final public FreebaseType type; + final public List links = new LinkedList(); + + public CellTopicNode( + String columnName, + boolean createForNoReconMatch, + FreebaseType type + ) { + super(columnName); + + this.createForNoReconMatch = createForNoReconMatch; + this.type = type; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("cell-as-topic"); + writer.key("columnName"); writer.value(columnName); + writer.key("createForNoReconMatch"); writer.value(createForNoReconMatch); + if (createForNoReconMatch && type != null) { + writer.key("type"); type.write(writer, options); + } + if (links != null) { + writer.key("links"); writer.array(); + for (Link link : links) { + link.write(writer, options); + } + writer.endArray(); + } + + writer.endObject(); + } + + public void addLink(Link link) { + links.add(link); + } + + public Link getLink(int index) { + return links.get(index); + } + + public int getLinkCount() { + return links.size(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/CellValueNode.java b/src/main/java/com/metaweb/gridworks/protograph/CellValueNode.java index 879dbcbb6..61a29aae7 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/CellValueNode.java +++ b/src/main/java/com/metaweb/gridworks/protograph/CellValueNode.java @@ -1,34 +1,34 @@ -package com.metaweb.gridworks.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class CellValueNode extends CellNode { - final public String valueType; - final public String lang; - - public CellValueNode( - String columnName, - String valueType, - String lang - ) { - super(columnName); - - this.valueType = valueType; - this.lang = lang; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("cell-as-value"); - writer.key("columnName"); writer.value(columnName); - writer.key("valueType"); writer.value(valueType); - writer.key("lang"); writer.value(lang); - writer.endObject(); - } - -} +package com.metaweb.gridworks.protograph; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +public class CellValueNode extends CellNode { + final public String valueType; + final public String lang; + + public CellValueNode( + String columnName, + String valueType, + String lang + ) { + super(columnName); + + this.valueType = valueType; + this.lang = lang; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("cell-as-value"); + writer.key("columnName"); writer.value(columnName); + writer.key("valueType"); writer.value(valueType); + writer.key("lang"); writer.value(lang); + writer.endObject(); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/FreebaseProperty.java b/src/main/java/com/metaweb/gridworks/protograph/FreebaseProperty.java index acc547d3f..d887db128 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/FreebaseProperty.java +++ b/src/main/java/com/metaweb/gridworks/protograph/FreebaseProperty.java @@ -1,9 +1,9 @@ -package com.metaweb.gridworks.protograph; - -public class FreebaseProperty extends FreebaseTopic { - //final protected FreebaseType _expectedType; - - public FreebaseProperty(String id, String name) { - super(id, name); - } -} +package com.metaweb.gridworks.protograph; + +public class FreebaseProperty extends FreebaseTopic { + //final protected FreebaseType _expectedType; + + public FreebaseProperty(String id, String name) { + super(id, name); + } +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/FreebaseTopic.java b/src/main/java/com/metaweb/gridworks/protograph/FreebaseTopic.java index 0083394a1..284b35c93 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/FreebaseTopic.java +++ b/src/main/java/com/metaweb/gridworks/protograph/FreebaseTopic.java @@ -1,28 +1,28 @@ -package com.metaweb.gridworks.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; - -public class FreebaseTopic implements Jsonizable { - final public String id; - final public String name; - - public FreebaseTopic(String id, String name) { - this.id = id; - this.name = name; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("id"); writer.value(id); - writer.key("name"); writer.value(name); - writer.endObject(); - } - -} +package com.metaweb.gridworks.protograph; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; + +public class FreebaseTopic implements Jsonizable { + final public String id; + final public String name; + + public FreebaseTopic(String id, String name) { + this.id = id; + this.name = name; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("id"); writer.value(id); + writer.key("name"); writer.value(name); + writer.endObject(); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/FreebaseTopicNode.java b/src/main/java/com/metaweb/gridworks/protograph/FreebaseTopicNode.java index 1c9159585..13deabb2c 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/FreebaseTopicNode.java +++ b/src/main/java/com/metaweb/gridworks/protograph/FreebaseTopicNode.java @@ -1,46 +1,46 @@ -package com.metaweb.gridworks.protograph; - -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class FreebaseTopicNode implements Node, NodeWithLinks { - final public FreebaseTopic topic; - final public List links = new LinkedList(); - - public FreebaseTopicNode(FreebaseTopic topic) { - this.topic = topic; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("topic"); - writer.key("topic"); topic.write(writer, options); - if (links != null) { - writer.key("links"); writer.array(); - for (Link link : links) { - link.write(writer, options); - } - writer.endArray(); - } - - writer.endObject(); - } - - public void addLink(Link link) { - links.add(link); - } - - public Link getLink(int index) { - return links.get(index); - } - - public int getLinkCount() { - return links.size(); - } -} +package com.metaweb.gridworks.protograph; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +public class FreebaseTopicNode implements Node, NodeWithLinks { + final public FreebaseTopic topic; + final public List links = new LinkedList(); + + public FreebaseTopicNode(FreebaseTopic topic) { + this.topic = topic; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("topic"); + writer.key("topic"); topic.write(writer, options); + if (links != null) { + writer.key("links"); writer.array(); + for (Link link : links) { + link.write(writer, options); + } + writer.endArray(); + } + + writer.endObject(); + } + + public void addLink(Link link) { + links.add(link); + } + + public Link getLink(int index) { + return links.get(index); + } + + public int getLinkCount() { + return links.size(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/FreebaseType.java b/src/main/java/com/metaweb/gridworks/protograph/FreebaseType.java index 2ccc3911a..0920e3064 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/FreebaseType.java +++ b/src/main/java/com/metaweb/gridworks/protograph/FreebaseType.java @@ -1,36 +1,36 @@ -package com.metaweb.gridworks.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; - -public class FreebaseType extends FreebaseTopic implements Jsonizable { - public FreebaseType(String id, String name) { - super(id, name); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("id"); writer.value(id); - writer.key("name"); writer.value(name); - writer.endObject(); - } - - static public FreebaseType load(JSONObject obj) throws Exception { - if (obj == null) { - return null; - } - - FreebaseType type = new FreebaseType( - obj.getString("id"), - obj.getString("name") - ); - return type; - } -} +package com.metaweb.gridworks.protograph; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; + +public class FreebaseType extends FreebaseTopic implements Jsonizable { + public FreebaseType(String id, String name) { + super(id, name); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("id"); writer.value(id); + writer.key("name"); writer.value(name); + writer.endObject(); + } + + static public FreebaseType load(JSONObject obj) throws Exception { + if (obj == null) { + return null; + } + + FreebaseType type = new FreebaseType( + obj.getString("id"), + obj.getString("name") + ); + return type; + } +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/Link.java b/src/main/java/com/metaweb/gridworks/protograph/Link.java index abd0c4e75..b162c998f 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/Link.java +++ b/src/main/java/com/metaweb/gridworks/protograph/Link.java @@ -1,36 +1,36 @@ -package com.metaweb.gridworks.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; - -public class Link implements Jsonizable { - final public FreebaseProperty property; - final public Node target; - - public Link(FreebaseProperty property, Node target) { - this.property = property; - this.target = target; - } - - public FreebaseProperty getProperty() { - return property; - } - - public Node getTarget() { - return target; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("property"); property.write(writer, options); - writer.key("target"); target.write(writer, options); - writer.endObject(); - } - -} +package com.metaweb.gridworks.protograph; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; + +public class Link implements Jsonizable { + final public FreebaseProperty property; + final public Node target; + + public Link(FreebaseProperty property, Node target) { + this.property = property; + this.target = target; + } + + public FreebaseProperty getProperty() { + return property; + } + + public Node getTarget() { + return target; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("property"); property.write(writer, options); + writer.key("target"); target.write(writer, options); + writer.endObject(); + } + +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/Node.java b/src/main/java/com/metaweb/gridworks/protograph/Node.java index 6290b0406..fc735865d 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/Node.java +++ b/src/main/java/com/metaweb/gridworks/protograph/Node.java @@ -1,6 +1,6 @@ -package com.metaweb.gridworks.protograph; - -import com.metaweb.gridworks.Jsonizable; - -public interface Node extends Jsonizable { -} +package com.metaweb.gridworks.protograph; + +import com.metaweb.gridworks.Jsonizable; + +public interface Node extends Jsonizable { +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/NodeWithLinks.java b/src/main/java/com/metaweb/gridworks/protograph/NodeWithLinks.java index a13dfc867..a489e07d2 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/NodeWithLinks.java +++ b/src/main/java/com/metaweb/gridworks/protograph/NodeWithLinks.java @@ -1,9 +1,9 @@ -package com.metaweb.gridworks.protograph; - -public interface NodeWithLinks { - public void addLink(Link link); - - public int getLinkCount(); - - public Link getLink(int index); -} +package com.metaweb.gridworks.protograph; + +public interface NodeWithLinks { + public void addLink(Link link); + + public int getLinkCount(); + + public Link getLink(int index); +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/Protograph.java b/src/main/java/com/metaweb/gridworks/protograph/Protograph.java index e98f47e33..d2e0db5df 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/Protograph.java +++ b/src/main/java/com/metaweb/gridworks/protograph/Protograph.java @@ -1,150 +1,150 @@ -package com.metaweb.gridworks.protograph; - -import java.io.Writer; -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.metaweb.gridworks.Jsonizable; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.util.ParsingUtilities; - -public class Protograph implements Jsonizable { - final protected List _rootNodes = new LinkedList(); - - public int getRootNodeCount() { - return _rootNodes.size(); - } - - public Node getRootNode(int index) { - return _rootNodes.get(index); - } - - static public Protograph reconstruct(JSONObject o) throws JSONException { - Protograph g = new Protograph(); - - JSONArray rootNodes = o.getJSONArray("rootNodes"); - int count = rootNodes.length(); - - for (int i = 0; i < count; i++) { - JSONObject o2 = rootNodes.getJSONObject(i); - Node node = reconstructNode(o2); - if (node != null) { - g._rootNodes.add(node); - } - } - - return g; - } - - static protected Node reconstructNode(JSONObject o) throws JSONException { - Node node = null; - - String nodeType = o.getString("nodeType"); - if (nodeType.startsWith("cell-as-")) { - String columnName = o.getString("columnName"); - - if ("cell-as-topic".equals(nodeType)) { - if (o.has("type")) { - node = new CellTopicNode( - columnName, - o.getBoolean("createForNoReconMatch"), - reconstructType(o.getJSONObject("type")) - ); - } - } else if ("cell-as-value".equals(nodeType)) { - node = new CellValueNode( - columnName, - o.getString("valueType"), - o.getString("lang") - ); - } else if ("cell-as-key".equals(nodeType)) { - node = new CellKeyNode( - columnName, - reconstructTopic(o.getJSONObject("namespace")) - ); - } - } else if ("topic".equals(nodeType)) { - node = new FreebaseTopicNode(reconstructTopic(o.getJSONObject("topic"))); - } else if ("value".equals(nodeType)) { - node = new ValueNode( - o.get("value"), - o.getString("valueType"), - o.getString("lang") - ); - } else if ("anonymous".equals(nodeType)) { - node = new AnonymousNode(reconstructType(o.getJSONObject("type"))); - } - - if (node != null && node instanceof NodeWithLinks && o.has("links")) { - NodeWithLinks node2 = (NodeWithLinks) node; - - JSONArray links = o.getJSONArray("links"); - int linkCount = links.length(); - - for (int j = 0; j < linkCount; j++) { - JSONObject oLink = links.getJSONObject(j); - - node2.addLink(new Link( - reconstructProperty(oLink.getJSONObject("property")), - reconstructNode(oLink.getJSONObject("target")) - )); - } - } - - return node; - } - - static protected FreebaseProperty reconstructProperty(JSONObject o) throws JSONException { - return new FreebaseProperty( - o.getString("id"), - o.getString("name") - ); - } - - static protected FreebaseType reconstructType(JSONObject o) throws JSONException { - return new FreebaseType( - o.getString("id"), - o.getString("name") - ); - } - - static protected FreebaseTopic reconstructTopic(JSONObject o) throws JSONException { - return new FreebaseTopic( - o.getString("id"), - o.getString("name") - ); - } - - public void write(JSONWriter writer, Properties options) throws JSONException { - writer.object(); - writer.key("rootNodes"); writer.array(); - - for (Node node : _rootNodes) { - node.write(writer, options); - } - - writer.endArray(); - writer.endObject(); - } - - public void save(Writer writer, Properties options) { - JSONWriter jsonWriter = new JSONWriter(writer); - try { - write(jsonWriter, options); - } catch (JSONException e) { - e.printStackTrace(); - } - } - - static public Protograph load(Project project, String s) throws Exception { - JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(s); - - return reconstruct(obj); - } -} +package com.metaweb.gridworks.protograph; + +import java.io.Writer; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.metaweb.gridworks.Jsonizable; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.util.ParsingUtilities; + +public class Protograph implements Jsonizable { + final protected List _rootNodes = new LinkedList(); + + public int getRootNodeCount() { + return _rootNodes.size(); + } + + public Node getRootNode(int index) { + return _rootNodes.get(index); + } + + static public Protograph reconstruct(JSONObject o) throws JSONException { + Protograph g = new Protograph(); + + JSONArray rootNodes = o.getJSONArray("rootNodes"); + int count = rootNodes.length(); + + for (int i = 0; i < count; i++) { + JSONObject o2 = rootNodes.getJSONObject(i); + Node node = reconstructNode(o2); + if (node != null) { + g._rootNodes.add(node); + } + } + + return g; + } + + static protected Node reconstructNode(JSONObject o) throws JSONException { + Node node = null; + + String nodeType = o.getString("nodeType"); + if (nodeType.startsWith("cell-as-")) { + String columnName = o.getString("columnName"); + + if ("cell-as-topic".equals(nodeType)) { + if (o.has("type")) { + node = new CellTopicNode( + columnName, + o.getBoolean("createForNoReconMatch"), + reconstructType(o.getJSONObject("type")) + ); + } + } else if ("cell-as-value".equals(nodeType)) { + node = new CellValueNode( + columnName, + o.getString("valueType"), + o.getString("lang") + ); + } else if ("cell-as-key".equals(nodeType)) { + node = new CellKeyNode( + columnName, + reconstructTopic(o.getJSONObject("namespace")) + ); + } + } else if ("topic".equals(nodeType)) { + node = new FreebaseTopicNode(reconstructTopic(o.getJSONObject("topic"))); + } else if ("value".equals(nodeType)) { + node = new ValueNode( + o.get("value"), + o.getString("valueType"), + o.getString("lang") + ); + } else if ("anonymous".equals(nodeType)) { + node = new AnonymousNode(reconstructType(o.getJSONObject("type"))); + } + + if (node != null && node instanceof NodeWithLinks && o.has("links")) { + NodeWithLinks node2 = (NodeWithLinks) node; + + JSONArray links = o.getJSONArray("links"); + int linkCount = links.length(); + + for (int j = 0; j < linkCount; j++) { + JSONObject oLink = links.getJSONObject(j); + + node2.addLink(new Link( + reconstructProperty(oLink.getJSONObject("property")), + reconstructNode(oLink.getJSONObject("target")) + )); + } + } + + return node; + } + + static protected FreebaseProperty reconstructProperty(JSONObject o) throws JSONException { + return new FreebaseProperty( + o.getString("id"), + o.getString("name") + ); + } + + static protected FreebaseType reconstructType(JSONObject o) throws JSONException { + return new FreebaseType( + o.getString("id"), + o.getString("name") + ); + } + + static protected FreebaseTopic reconstructTopic(JSONObject o) throws JSONException { + return new FreebaseTopic( + o.getString("id"), + o.getString("name") + ); + } + + public void write(JSONWriter writer, Properties options) throws JSONException { + writer.object(); + writer.key("rootNodes"); writer.array(); + + for (Node node : _rootNodes) { + node.write(writer, options); + } + + writer.endArray(); + writer.endObject(); + } + + public void save(Writer writer, Properties options) { + JSONWriter jsonWriter = new JSONWriter(writer); + try { + write(jsonWriter, options); + } catch (JSONException e) { + e.printStackTrace(); + } + } + + static public Protograph load(Project project, String s) throws Exception { + JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(s); + + return reconstruct(obj); + } +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/ValueNode.java b/src/main/java/com/metaweb/gridworks/protograph/ValueNode.java index a4850f430..ef9daff7a 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/ValueNode.java +++ b/src/main/java/com/metaweb/gridworks/protograph/ValueNode.java @@ -1,29 +1,29 @@ -package com.metaweb.gridworks.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class ValueNode implements Node { - final public Object value; - final public String valueType; - final public String lang; - - public ValueNode(Object value, String valueType, String lang) { - this.value = value; - this.valueType = valueType; - this.lang = lang; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("value"); - writer.key("value"); writer.value(value); - writer.key("valueType"); writer.value(valueType); - writer.key("lang"); writer.value(lang); - writer.endObject(); - } -} +package com.metaweb.gridworks.protograph; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +public class ValueNode implements Node { + final public Object value; + final public String valueType; + final public String lang; + + public ValueNode(Object value, String valueType, String lang) { + this.value = value; + this.valueType = valueType; + this.lang = lang; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("value"); + writer.key("value"); writer.value(value); + writer.key("valueType"); writer.value(valueType); + writer.key("lang"); writer.value(lang); + writer.endObject(); + } +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/transpose/MqlreadLikeTransposedNodeFactory.java b/src/main/java/com/metaweb/gridworks/protograph/transpose/MqlreadLikeTransposedNodeFactory.java index da817046e..b469995fa 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/transpose/MqlreadLikeTransposedNodeFactory.java +++ b/src/main/java/com/metaweb/gridworks/protograph/transpose/MqlreadLikeTransposedNodeFactory.java @@ -1,316 +1,316 @@ -package com.metaweb.gridworks.protograph.transpose; - -import java.util.LinkedList; -import java.util.List; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.protograph.AnonymousNode; -import com.metaweb.gridworks.protograph.CellKeyNode; -import com.metaweb.gridworks.protograph.CellNode; -import com.metaweb.gridworks.protograph.CellTopicNode; -import com.metaweb.gridworks.protograph.CellValueNode; -import com.metaweb.gridworks.protograph.FreebaseProperty; -import com.metaweb.gridworks.protograph.FreebaseTopicNode; -import com.metaweb.gridworks.protograph.ValueNode; - -public class MqlreadLikeTransposedNodeFactory implements TransposedNodeFactory { - protected List rootObjects = new LinkedList(); - - private static final String TYPE = "type"; - private static final String ID = "id"; - private static final String NAME = "name"; - private static final String CREATE = "create"; - private static final String VALUE = "value"; - private static final String CONNECT = "connect"; - private static final String LANG = "connect"; - - public JSONArray getJSON() { - return new JSONArray(rootObjects); - } - - abstract protected class JsonTransposedNode implements TransposedNode { - abstract public Object getJSON(); - } - - abstract protected class JsonObjectTransposedNode extends JsonTransposedNode { - abstract public JSONObject getJSONObject(); - - protected JSONObject obj; - - public Object getJSON() { - return getJSONObject(); - } - } - - protected class AnonymousTransposedNode extends JsonObjectTransposedNode { - JsonObjectTransposedNode parent; - FreebaseProperty property; - AnonymousNode node; - - protected AnonymousTransposedNode( - JsonObjectTransposedNode parent, - FreebaseProperty property, - AnonymousNode node - ) { - this.parent = parent; - this.property = property; - this.node = node; - } - - public JSONObject getJSONObject() { - if (obj == null) { - obj = new JSONObject(); - try { - obj.put(TYPE, this.node.type.id); - obj.put(ID, (String) null); - obj.put(CREATE, "unconditional"); - } catch (JSONException e) { - e.printStackTrace(); - } - - linkTransposedNodeJSON(obj, parent, property); - } - - return obj; - } - } - - protected class CellTopicTransposedNode extends JsonObjectTransposedNode { - protected CellTopicNode node; - protected Cell cell; - - public CellTopicTransposedNode(CellTopicNode node, Cell cell) { - this.node = node; - this.cell = cell; - } - - @Override - public JSONObject getJSONObject() { - if (obj == null) { - obj = new JSONObject(); - try { - if (cell.recon != null && - cell.recon.judgment == Recon.Judgment.Matched && - cell.recon.match != null) { - obj.put(ID, cell.recon.match.topicID); - } else { - obj.put(ID, (String) null); - obj.put(NAME, cell.value.toString()); - obj.put(TYPE, node.type.id); - obj.put(CREATE, "unless_exists"); - } - } catch (JSONException e) { - e.printStackTrace(); - } - } - return obj; - } - } - - protected class CellValueTransposedNode extends JsonTransposedNode { - protected JSONObject obj; - protected CellValueNode node; - protected Cell cell; - - public CellValueTransposedNode(CellValueNode node, Cell cell) { - this.node = node; - this.cell = cell; - } - - public Object getJSON() { - if (obj == null) { - obj = new JSONObject(); - try { - obj.put(VALUE, cell.value.toString()); - obj.put(TYPE, node.valueType); - if ("/type/text".equals(node.lang)) { - obj.put(LANG, node.lang); - } - - obj.put(CONNECT, "insert"); - } catch (JSONException e) { - e.printStackTrace(); - } - } - return obj; - } - } - - protected class CellKeyTransposedNode extends JsonTransposedNode { - protected JSONObject obj; - protected CellKeyNode node; - protected Cell cell; - - public CellKeyTransposedNode(CellKeyNode node, Cell cell) { - this.node = node; - this.cell = cell; - } - - public Object getJSON() { - if (obj == null) { - obj = new JSONObject(); - try { - obj.put(VALUE, cell.value.toString()); - - JSONObject nsObj = new JSONObject(); - nsObj.put(ID, node.namespace.id); - - obj.put("namespace", nsObj); - obj.put(CONNECT, "insert"); - } catch (JSONException e) { - e.printStackTrace(); - } - } - return obj; - } - } - - protected class TopicTransposedNode extends JsonObjectTransposedNode { - protected FreebaseTopicNode node; - - public TopicTransposedNode(FreebaseTopicNode node) { - this.node = node; - } - - @Override - public JSONObject getJSONObject() { - if (obj == null) { - obj = new JSONObject(); - try { - obj.put(ID, node.topic.id); - } catch (JSONException e) { - e.printStackTrace(); - } - } - return obj; - } - } - - protected class ValueTransposedNode extends JsonTransposedNode { - protected JSONObject obj; - protected ValueNode node; - - public ValueTransposedNode(ValueNode node) { - this.node = node; - } - - public Object getJSON() { - if (obj == null) { - obj = new JSONObject(); - try { - obj.put(VALUE, node.value); - obj.put(TYPE, node.valueType); - if ("/type/text".equals(node.lang)) { - obj.put(LANG, node.lang); - } - - obj.put(CONNECT, "insert"); - } catch (JSONException e) { - e.printStackTrace(); - } - } - return obj; - } - } - public TransposedNode transposeAnonymousNode( - TransposedNode parentNode, - FreebaseProperty property, - AnonymousNode node) { - - return new AnonymousTransposedNode( - parentNode instanceof JsonObjectTransposedNode ? (JsonObjectTransposedNode) parentNode : null, - property, - node - ); - } - - public TransposedNode transposeCellNode( - TransposedNode parentNode, - FreebaseProperty property, - CellNode node, - Cell cell) { - - JsonTransposedNode tnode = null; - if (node instanceof CellTopicNode) { - tnode = new CellTopicTransposedNode((CellTopicNode) node, cell); - } else if (node instanceof CellValueNode) { - tnode = new CellValueTransposedNode((CellValueNode) node, cell); - } else if (node instanceof CellKeyNode) { - tnode = new CellKeyTransposedNode((CellKeyNode) node, cell); - } - - if (tnode != null) { - processTransposedNode(tnode, parentNode, property); - } - return tnode; - } - - public TransposedNode transposeTopicNode( - TransposedNode parentNode, - FreebaseProperty property, - FreebaseTopicNode node) { - - JsonTransposedNode tnode = new TopicTransposedNode(node); - - processTransposedNode(tnode, parentNode, property); - - return tnode; - } - - public TransposedNode transposeValueNode( - TransposedNode parentNode, - FreebaseProperty property, - ValueNode node) { - - JsonTransposedNode tnode = new ValueTransposedNode(node); - - processTransposedNode(tnode, parentNode, property); - - return tnode; - } - - protected void processTransposedNode( - JsonTransposedNode tnode, - TransposedNode parentNode, - FreebaseProperty property - ) { - - if (!(tnode instanceof AnonymousTransposedNode)) { - linkTransposedNodeJSON(tnode.getJSON(), parentNode, property); - } - } - - protected void linkTransposedNodeJSON( - Object obj, - TransposedNode parentNode, - FreebaseProperty property - ) { - - if (parentNode == null) { - if (obj instanceof JSONObject) { - rootObjects.add((JSONObject) obj); - } - } else if (parentNode instanceof JsonTransposedNode) { - JSONObject parentObj = ((JsonObjectTransposedNode) parentNode).getJSONObject(); - - try { - JSONArray a = null; - if (parentObj.has(property.id)) { - a = parentObj.getJSONArray(property.id); - } else { - a = new JSONArray(); - parentObj.put(property.id, a); - } - - a.put(a.length(), obj); - } catch (JSONException e) { - e.printStackTrace(); - } - } - } -} +package com.metaweb.gridworks.protograph.transpose; + +import java.util.LinkedList; +import java.util.List; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.protograph.AnonymousNode; +import com.metaweb.gridworks.protograph.CellKeyNode; +import com.metaweb.gridworks.protograph.CellNode; +import com.metaweb.gridworks.protograph.CellTopicNode; +import com.metaweb.gridworks.protograph.CellValueNode; +import com.metaweb.gridworks.protograph.FreebaseProperty; +import com.metaweb.gridworks.protograph.FreebaseTopicNode; +import com.metaweb.gridworks.protograph.ValueNode; + +public class MqlreadLikeTransposedNodeFactory implements TransposedNodeFactory { + protected List rootObjects = new LinkedList(); + + private static final String TYPE = "type"; + private static final String ID = "id"; + private static final String NAME = "name"; + private static final String CREATE = "create"; + private static final String VALUE = "value"; + private static final String CONNECT = "connect"; + private static final String LANG = "connect"; + + public JSONArray getJSON() { + return new JSONArray(rootObjects); + } + + abstract protected class JsonTransposedNode implements TransposedNode { + abstract public Object getJSON(); + } + + abstract protected class JsonObjectTransposedNode extends JsonTransposedNode { + abstract public JSONObject getJSONObject(); + + protected JSONObject obj; + + public Object getJSON() { + return getJSONObject(); + } + } + + protected class AnonymousTransposedNode extends JsonObjectTransposedNode { + JsonObjectTransposedNode parent; + FreebaseProperty property; + AnonymousNode node; + + protected AnonymousTransposedNode( + JsonObjectTransposedNode parent, + FreebaseProperty property, + AnonymousNode node + ) { + this.parent = parent; + this.property = property; + this.node = node; + } + + public JSONObject getJSONObject() { + if (obj == null) { + obj = new JSONObject(); + try { + obj.put(TYPE, this.node.type.id); + obj.put(ID, (String) null); + obj.put(CREATE, "unconditional"); + } catch (JSONException e) { + e.printStackTrace(); + } + + linkTransposedNodeJSON(obj, parent, property); + } + + return obj; + } + } + + protected class CellTopicTransposedNode extends JsonObjectTransposedNode { + protected CellTopicNode node; + protected Cell cell; + + public CellTopicTransposedNode(CellTopicNode node, Cell cell) { + this.node = node; + this.cell = cell; + } + + @Override + public JSONObject getJSONObject() { + if (obj == null) { + obj = new JSONObject(); + try { + if (cell.recon != null && + cell.recon.judgment == Recon.Judgment.Matched && + cell.recon.match != null) { + obj.put(ID, cell.recon.match.topicID); + } else { + obj.put(ID, (String) null); + obj.put(NAME, cell.value.toString()); + obj.put(TYPE, node.type.id); + obj.put(CREATE, "unless_exists"); + } + } catch (JSONException e) { + e.printStackTrace(); + } + } + return obj; + } + } + + protected class CellValueTransposedNode extends JsonTransposedNode { + protected JSONObject obj; + protected CellValueNode node; + protected Cell cell; + + public CellValueTransposedNode(CellValueNode node, Cell cell) { + this.node = node; + this.cell = cell; + } + + public Object getJSON() { + if (obj == null) { + obj = new JSONObject(); + try { + obj.put(VALUE, cell.value.toString()); + obj.put(TYPE, node.valueType); + if ("/type/text".equals(node.lang)) { + obj.put(LANG, node.lang); + } + + obj.put(CONNECT, "insert"); + } catch (JSONException e) { + e.printStackTrace(); + } + } + return obj; + } + } + + protected class CellKeyTransposedNode extends JsonTransposedNode { + protected JSONObject obj; + protected CellKeyNode node; + protected Cell cell; + + public CellKeyTransposedNode(CellKeyNode node, Cell cell) { + this.node = node; + this.cell = cell; + } + + public Object getJSON() { + if (obj == null) { + obj = new JSONObject(); + try { + obj.put(VALUE, cell.value.toString()); + + JSONObject nsObj = new JSONObject(); + nsObj.put(ID, node.namespace.id); + + obj.put("namespace", nsObj); + obj.put(CONNECT, "insert"); + } catch (JSONException e) { + e.printStackTrace(); + } + } + return obj; + } + } + + protected class TopicTransposedNode extends JsonObjectTransposedNode { + protected FreebaseTopicNode node; + + public TopicTransposedNode(FreebaseTopicNode node) { + this.node = node; + } + + @Override + public JSONObject getJSONObject() { + if (obj == null) { + obj = new JSONObject(); + try { + obj.put(ID, node.topic.id); + } catch (JSONException e) { + e.printStackTrace(); + } + } + return obj; + } + } + + protected class ValueTransposedNode extends JsonTransposedNode { + protected JSONObject obj; + protected ValueNode node; + + public ValueTransposedNode(ValueNode node) { + this.node = node; + } + + public Object getJSON() { + if (obj == null) { + obj = new JSONObject(); + try { + obj.put(VALUE, node.value); + obj.put(TYPE, node.valueType); + if ("/type/text".equals(node.lang)) { + obj.put(LANG, node.lang); + } + + obj.put(CONNECT, "insert"); + } catch (JSONException e) { + e.printStackTrace(); + } + } + return obj; + } + } + public TransposedNode transposeAnonymousNode( + TransposedNode parentNode, + FreebaseProperty property, + AnonymousNode node) { + + return new AnonymousTransposedNode( + parentNode instanceof JsonObjectTransposedNode ? (JsonObjectTransposedNode) parentNode : null, + property, + node + ); + } + + public TransposedNode transposeCellNode( + TransposedNode parentNode, + FreebaseProperty property, + CellNode node, + Cell cell) { + + JsonTransposedNode tnode = null; + if (node instanceof CellTopicNode) { + tnode = new CellTopicTransposedNode((CellTopicNode) node, cell); + } else if (node instanceof CellValueNode) { + tnode = new CellValueTransposedNode((CellValueNode) node, cell); + } else if (node instanceof CellKeyNode) { + tnode = new CellKeyTransposedNode((CellKeyNode) node, cell); + } + + if (tnode != null) { + processTransposedNode(tnode, parentNode, property); + } + return tnode; + } + + public TransposedNode transposeTopicNode( + TransposedNode parentNode, + FreebaseProperty property, + FreebaseTopicNode node) { + + JsonTransposedNode tnode = new TopicTransposedNode(node); + + processTransposedNode(tnode, parentNode, property); + + return tnode; + } + + public TransposedNode transposeValueNode( + TransposedNode parentNode, + FreebaseProperty property, + ValueNode node) { + + JsonTransposedNode tnode = new ValueTransposedNode(node); + + processTransposedNode(tnode, parentNode, property); + + return tnode; + } + + protected void processTransposedNode( + JsonTransposedNode tnode, + TransposedNode parentNode, + FreebaseProperty property + ) { + + if (!(tnode instanceof AnonymousTransposedNode)) { + linkTransposedNodeJSON(tnode.getJSON(), parentNode, property); + } + } + + protected void linkTransposedNodeJSON( + Object obj, + TransposedNode parentNode, + FreebaseProperty property + ) { + + if (parentNode == null) { + if (obj instanceof JSONObject) { + rootObjects.add((JSONObject) obj); + } + } else if (parentNode instanceof JsonTransposedNode) { + JSONObject parentObj = ((JsonObjectTransposedNode) parentNode).getJSONObject(); + + try { + JSONArray a = null; + if (parentObj.has(property.id)) { + a = parentObj.getJSONArray(property.id); + } else { + a = new JSONArray(); + parentObj.put(property.id, a); + } + + a.put(a.length(), obj); + } catch (JSONException e) { + e.printStackTrace(); + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/transpose/TransposedNode.java b/src/main/java/com/metaweb/gridworks/protograph/transpose/TransposedNode.java index 32c7e527b..827dbc804 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/transpose/TransposedNode.java +++ b/src/main/java/com/metaweb/gridworks/protograph/transpose/TransposedNode.java @@ -1,4 +1,4 @@ -package com.metaweb.gridworks.protograph.transpose; - -public interface TransposedNode { -} +package com.metaweb.gridworks.protograph.transpose; + +public interface TransposedNode { +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/transpose/TransposedNodeFactory.java b/src/main/java/com/metaweb/gridworks/protograph/transpose/TransposedNodeFactory.java index 42ca343f6..6e3bfe30d 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/transpose/TransposedNodeFactory.java +++ b/src/main/java/com/metaweb/gridworks/protograph/transpose/TransposedNodeFactory.java @@ -1,35 +1,35 @@ -package com.metaweb.gridworks.protograph.transpose; - -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.protograph.AnonymousNode; -import com.metaweb.gridworks.protograph.CellNode; -import com.metaweb.gridworks.protograph.FreebaseProperty; -import com.metaweb.gridworks.protograph.FreebaseTopicNode; -import com.metaweb.gridworks.protograph.ValueNode; - -public interface TransposedNodeFactory { - public TransposedNode transposeAnonymousNode( - TransposedNode parentNode, - FreebaseProperty property, - AnonymousNode node - ); - - public TransposedNode transposeCellNode( - TransposedNode parentNode, - FreebaseProperty property, - CellNode node, - Cell cell - ); - - public TransposedNode transposeValueNode( - TransposedNode parentNode, - FreebaseProperty property, - ValueNode node - ); - - public TransposedNode transposeTopicNode( - TransposedNode parentNode, - FreebaseProperty property, - FreebaseTopicNode node - ); -} +package com.metaweb.gridworks.protograph.transpose; + +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.protograph.AnonymousNode; +import com.metaweb.gridworks.protograph.CellNode; +import com.metaweb.gridworks.protograph.FreebaseProperty; +import com.metaweb.gridworks.protograph.FreebaseTopicNode; +import com.metaweb.gridworks.protograph.ValueNode; + +public interface TransposedNodeFactory { + public TransposedNode transposeAnonymousNode( + TransposedNode parentNode, + FreebaseProperty property, + AnonymousNode node + ); + + public TransposedNode transposeCellNode( + TransposedNode parentNode, + FreebaseProperty property, + CellNode node, + Cell cell + ); + + public TransposedNode transposeValueNode( + TransposedNode parentNode, + FreebaseProperty property, + ValueNode node + ); + + public TransposedNode transposeTopicNode( + TransposedNode parentNode, + FreebaseProperty property, + FreebaseTopicNode node + ); +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/transpose/Transposer.java b/src/main/java/com/metaweb/gridworks/protograph/transpose/Transposer.java index c9afac828..e3a6f1c69 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/transpose/Transposer.java +++ b/src/main/java/com/metaweb/gridworks/protograph/transpose/Transposer.java @@ -1,170 +1,170 @@ -package com.metaweb.gridworks.protograph.transpose; - -import java.util.LinkedList; -import java.util.List; - -import com.metaweb.gridworks.expr.ExpressionUtils; -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Column; -import com.metaweb.gridworks.model.Project; -import com.metaweb.gridworks.model.Row; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.protograph.AnonymousNode; -import com.metaweb.gridworks.protograph.CellNode; -import com.metaweb.gridworks.protograph.CellTopicNode; -import com.metaweb.gridworks.protograph.FreebaseProperty; -import com.metaweb.gridworks.protograph.FreebaseTopicNode; -import com.metaweb.gridworks.protograph.Link; -import com.metaweb.gridworks.protograph.Node; -import com.metaweb.gridworks.protograph.NodeWithLinks; -import com.metaweb.gridworks.protograph.Protograph; -import com.metaweb.gridworks.protograph.ValueNode; - -public class Transposer { - static public void transpose( - Project project, - Protograph protograph, - Node rootNode, - TransposedNodeFactory nodeFactory - ) { - transpose(project, protograph, rootNode, nodeFactory, 20); - } - - static public void transpose( - Project project, - Protograph protograph, - Node rootNode, - TransposedNodeFactory nodeFactory, - int limit - ) { - Context rootContext = new Context(rootNode, null, null, limit); - - for (Row row : project.rows) { - descend(project, protograph, nodeFactory, row, rootNode, rootContext); - if (rootContext.limit > 0 && rootContext.count > rootContext.limit) { - break; - } - } - } - - static protected void descend( - Project project, - Protograph protograph, - TransposedNodeFactory nodeFactory, - Row row, - Node node, - Context context - ) { - TransposedNode tnode = null; - - TransposedNode parentNode = context.parent == null ? null : context.parent.transposedNode; - FreebaseProperty property = context.parent == null ? null : context.link.property; - - if (node instanceof CellNode) { - CellNode node2 = (CellNode) node; - Column column = project.columnModel.getColumnByName(node2.columnName); - Cell cell = row.getCell(column.getCellIndex()); - if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) { - if (node2 instanceof CellTopicNode && - !((CellTopicNode) node2).createForNoReconMatch && - (cell.recon == null || cell.recon.judgment == Judgment.None)) { - return; - } - - context.count++; - if (context.limit > 0 && context.count > context.limit) { - return; - } - - tnode = nodeFactory.transposeCellNode( - parentNode, - property, - node2, - cell - ); - } - } else { - if (node instanceof AnonymousNode) { - tnode = nodeFactory.transposeAnonymousNode( - parentNode, - property, - (AnonymousNode) node - ); - } else if (node instanceof FreebaseTopicNode) { - tnode = nodeFactory.transposeTopicNode( - parentNode, - property, - (FreebaseTopicNode) node - ); - } else if (node instanceof ValueNode) { - tnode = nodeFactory.transposeValueNode( - parentNode, - property, - (ValueNode) node - ); - } - } - - if (tnode != null) { - context.transposedNode = tnode; - context.nullifySubContextNodes(); - } /* - else, previous rows might have set the context transposed node already, - and we simply inherit that transposed node. - */ - - if (node instanceof NodeWithLinks && context.transposedNode != null) { - NodeWithLinks node2 = (NodeWithLinks) node; - - int linkCount = node2.getLinkCount(); - - for (int i = 0; i < linkCount; i++) { - descend( - project, - protograph, - nodeFactory, - row, - node2.getLink(i).getTarget(), - context.subContexts.get(i) - ); - } - } - } - - static class Context { - TransposedNode transposedNode; - List subContexts; - Context parent; - Link link; - int count; - int limit; - - Context(Node node, Context parent, Link link, int limit) { - this.parent = parent; - this.link = link; - this.limit = limit; - - if (node instanceof NodeWithLinks) { - NodeWithLinks node2 = (NodeWithLinks) node; - - int subContextCount = node2.getLinkCount(); - - subContexts = new LinkedList(); - for (int i = 0; i < subContextCount; i++) { - Link link2 = node2.getLink(i); - subContexts.add( - new Context(link2.getTarget(), this, link2, -1)); - } - } - } - - public void nullifySubContextNodes() { - if (subContexts != null) { - for (Context context : subContexts) { - context.transposedNode = null; - context.nullifySubContextNodes(); - } - } - } - } -} +package com.metaweb.gridworks.protograph.transpose; + +import java.util.LinkedList; +import java.util.List; + +import com.metaweb.gridworks.expr.ExpressionUtils; +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Column; +import com.metaweb.gridworks.model.Project; +import com.metaweb.gridworks.model.Row; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.protograph.AnonymousNode; +import com.metaweb.gridworks.protograph.CellNode; +import com.metaweb.gridworks.protograph.CellTopicNode; +import com.metaweb.gridworks.protograph.FreebaseProperty; +import com.metaweb.gridworks.protograph.FreebaseTopicNode; +import com.metaweb.gridworks.protograph.Link; +import com.metaweb.gridworks.protograph.Node; +import com.metaweb.gridworks.protograph.NodeWithLinks; +import com.metaweb.gridworks.protograph.Protograph; +import com.metaweb.gridworks.protograph.ValueNode; + +public class Transposer { + static public void transpose( + Project project, + Protograph protograph, + Node rootNode, + TransposedNodeFactory nodeFactory + ) { + transpose(project, protograph, rootNode, nodeFactory, 20); + } + + static public void transpose( + Project project, + Protograph protograph, + Node rootNode, + TransposedNodeFactory nodeFactory, + int limit + ) { + Context rootContext = new Context(rootNode, null, null, limit); + + for (Row row : project.rows) { + descend(project, protograph, nodeFactory, row, rootNode, rootContext); + if (rootContext.limit > 0 && rootContext.count > rootContext.limit) { + break; + } + } + } + + static protected void descend( + Project project, + Protograph protograph, + TransposedNodeFactory nodeFactory, + Row row, + Node node, + Context context + ) { + TransposedNode tnode = null; + + TransposedNode parentNode = context.parent == null ? null : context.parent.transposedNode; + FreebaseProperty property = context.parent == null ? null : context.link.property; + + if (node instanceof CellNode) { + CellNode node2 = (CellNode) node; + Column column = project.columnModel.getColumnByName(node2.columnName); + Cell cell = row.getCell(column.getCellIndex()); + if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) { + if (node2 instanceof CellTopicNode && + !((CellTopicNode) node2).createForNoReconMatch && + (cell.recon == null || cell.recon.judgment == Judgment.None)) { + return; + } + + context.count++; + if (context.limit > 0 && context.count > context.limit) { + return; + } + + tnode = nodeFactory.transposeCellNode( + parentNode, + property, + node2, + cell + ); + } + } else { + if (node instanceof AnonymousNode) { + tnode = nodeFactory.transposeAnonymousNode( + parentNode, + property, + (AnonymousNode) node + ); + } else if (node instanceof FreebaseTopicNode) { + tnode = nodeFactory.transposeTopicNode( + parentNode, + property, + (FreebaseTopicNode) node + ); + } else if (node instanceof ValueNode) { + tnode = nodeFactory.transposeValueNode( + parentNode, + property, + (ValueNode) node + ); + } + } + + if (tnode != null) { + context.transposedNode = tnode; + context.nullifySubContextNodes(); + } /* + else, previous rows might have set the context transposed node already, + and we simply inherit that transposed node. + */ + + if (node instanceof NodeWithLinks && context.transposedNode != null) { + NodeWithLinks node2 = (NodeWithLinks) node; + + int linkCount = node2.getLinkCount(); + + for (int i = 0; i < linkCount; i++) { + descend( + project, + protograph, + nodeFactory, + row, + node2.getLink(i).getTarget(), + context.subContexts.get(i) + ); + } + } + } + + static class Context { + TransposedNode transposedNode; + List subContexts; + Context parent; + Link link; + int count; + int limit; + + Context(Node node, Context parent, Link link, int limit) { + this.parent = parent; + this.link = link; + this.limit = limit; + + if (node instanceof NodeWithLinks) { + NodeWithLinks node2 = (NodeWithLinks) node; + + int subContextCount = node2.getLinkCount(); + + subContexts = new LinkedList(); + for (int i = 0; i < subContextCount; i++) { + Link link2 = node2.getLink(i); + subContexts.add( + new Context(link2.getTarget(), this, link2, -1)); + } + } + } + + public void nullifySubContextNodes() { + if (subContexts != null) { + for (Context context : subContexts) { + context.transposedNode = null; + context.nullifySubContextNodes(); + } + } + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/protograph/transpose/TripleLoaderTransposedNodeFactory.java b/src/main/java/com/metaweb/gridworks/protograph/transpose/TripleLoaderTransposedNodeFactory.java index d6cf574d4..8ff1b3ced 100644 --- a/src/main/java/com/metaweb/gridworks/protograph/transpose/TripleLoaderTransposedNodeFactory.java +++ b/src/main/java/com/metaweb/gridworks/protograph/transpose/TripleLoaderTransposedNodeFactory.java @@ -1,389 +1,389 @@ -package com.metaweb.gridworks.protograph.transpose; - -import java.io.IOException; -import java.io.Writer; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.json.JSONObject; - -import com.metaweb.gridworks.model.Cell; -import com.metaweb.gridworks.model.Recon; -import com.metaweb.gridworks.model.Recon.Judgment; -import com.metaweb.gridworks.protograph.AnonymousNode; -import com.metaweb.gridworks.protograph.CellKeyNode; -import com.metaweb.gridworks.protograph.CellNode; -import com.metaweb.gridworks.protograph.CellTopicNode; -import com.metaweb.gridworks.protograph.CellValueNode; -import com.metaweb.gridworks.protograph.FreebaseProperty; -import com.metaweb.gridworks.protograph.FreebaseTopicNode; -import com.metaweb.gridworks.protograph.ValueNode; - -public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory { - protected boolean start = true; - protected Writer writer; - protected WritingTransposedNode lastRootNode; - protected Map varPool = new HashMap(); - protected Map newTopicVars = new HashMap(); - protected Set serializedRecons = new HashSet(); - - public TripleLoaderTransposedNodeFactory(Writer writer) { - this.writer = writer; - } - - public void flush() { - if (lastRootNode != null) { - lastRootNode.write(null, null, null); - lastRootNode = null; - } - } - - protected void writeLine(String line) { - try { - if (start) { - start = false; - } else { - writer.write('\n'); - } - writer.write(line); - } catch (IOException e) { - // ignore - } - } - - protected void writeRecon(StringBuffer sb, Cell cell) { - Recon recon = cell.recon; - if (serializedRecons.contains(recon.id)) { - sb.append(Long.toString(recon.id)); - } else { - serializedRecons.add(recon.id); - - String s = cell.value instanceof String ? (String) cell.value : cell.value.toString(); - - sb.append("{ "); - sb.append("\"id\" : "); sb.append(Long.toString(recon.id)); - sb.append(", \"history_entry\" : "); sb.append(Long.toString(recon.judgmentHistoryEntry)); - sb.append(", \"text\" : "); sb.append(JSONObject.quote(s)); - sb.append(", \"service\" : "); sb.append(JSONObject.quote(recon.service)); - sb.append(", \"action\" : "); sb.append(JSONObject.quote(recon.judgmentAction)); - sb.append(", \"batch\" : "); sb.append(Integer.toString(recon.judgmentBatchSize)); - sb.append(", \"matchRank\" : "); sb.append(Integer.toString(recon.matchRank)); - sb.append(" }"); - } - } - - protected void writeLine(String subject, String predicate, Object object, Cell subjectCell, Cell objectCell) { - if (subject != null && object != null) { - String s = object instanceof String ? - JSONObject.quote((String) object) : object.toString(); - - StringBuffer sb = new StringBuffer(); - sb.append("{ \"s\" : \""); sb.append(subject); sb.append("\""); - sb.append(", \"p\" : \""); sb.append(predicate); sb.append("\""); - sb.append(", \"o\" : "); sb.append(s); - if (subjectCell != null || objectCell != null) { - sb.append(", \"meta\" : { "); - - if (subjectCell != null) { - sb.append("\"srecon\" : "); - writeRecon(sb, subjectCell); - } - if (objectCell != null) { - if (subjectCell != null) { - sb.append(", "); - } - sb.append("\"orecon\" : "); - writeRecon(sb, objectCell); - } - - sb.append(" }"); - } - sb.append(" }"); - - writeLine(sb.toString()); - } - } - - protected void writeLine(String subject, String predicate, Object object, String lang, Cell subjectCell) { - if (subject != null && object != null) { - String s = object instanceof String ? - JSONObject.quote((String) object) : object.toString(); - - StringBuffer sb = new StringBuffer(); - sb.append("{ \"s\" : \""); sb.append(subject); sb.append("\""); - sb.append(", \"p\" : \""); sb.append(predicate); sb.append("\""); - sb.append(", \"o\" : "); sb.append(s); - sb.append(", \"lang\" : "); sb.append(lang); - - if (subjectCell != null) { - sb.append(", \"meta\" : { "); - sb.append("\"srecon\" : "); - writeRecon(sb, subjectCell); - sb.append(" }"); - } - sb.append(" }"); - - writeLine(sb.toString()); - } - } - - protected interface WritingTransposedNode extends TransposedNode { - public String write(String subject, String predicate, Cell subjectCell); - } - - abstract protected class TransposedNodeWithChildren implements WritingTransposedNode { - public List properties = new LinkedList(); - public List children = new LinkedList(); - - protected void writeChildren(String subject, Cell subjectCell) { - for (int i = 0; i < children.size(); i++) { - WritingTransposedNode child = children.get(i); - String predicate = properties.get(i).id; - - child.write(subject, predicate, subjectCell); - } - } - } - - protected class AnonymousTransposedNode extends TransposedNodeWithChildren { - - //protected AnonymousTransposedNode(AnonymousNode node) { } - - public String write(String subject, String predicate, Cell subjectCell) { - if (children.size() == 0 || subject == null) { - return null; - } - - StringBuffer sb = new StringBuffer(); - sb.append("{ "); - - boolean first = true; - for (int i = 0; i < children.size(); i++) { - String s = children.get(i).write(null, null, null); - if (s != null) { - if (first) { - first = false; - } else { - sb.append(", "); - } - sb.append("\"" + properties.get(i).id + "\": "); - sb.append(JSONObject.quote(s)); - } - } - sb.append(" }"); - - writeLine(subject, predicate, sb, subjectCell, null); - - return null; - } - } - - protected class CellTopicTransposedNode extends TransposedNodeWithChildren { - protected CellTopicNode node; - protected Cell cell; - - public CellTopicTransposedNode(CellTopicNode node, Cell cell) { - this.node = node; - this.cell = cell; - } - - public String write(String subject, String predicate, Cell subjectCell) { - String id = null; - Cell objectCell = null; - - if (cell.recon != null && - cell.recon.judgment == Recon.Judgment.Matched && - cell.recon.match != null) { - - objectCell = cell; - id = cell.recon.match.topicID; - } else if (node.createForNoReconMatch || - (cell.recon != null && cell.recon.judgment == Judgment.New)) { - if (cell.recon != null && newTopicVars.containsKey(cell.recon.id)) { - id = newTopicVars.get(cell.recon.id); - } else { - long var = 0; - if (varPool.containsKey(node.columnName)) { - var = varPool.get(node.columnName); - } - varPool.put(node.columnName, var + 1); - - id = "$" + node.columnName.replaceAll("\\W+", "_") + "_" + var; - - writeLine(id, "type", node.type.id, (Cell) null, (Cell) null); - writeLine(id, "name", cell.value, (Cell) null, (Cell) null); - - if (cell.recon != null) { - newTopicVars.put(cell.recon.id, id); - } - } - } else { - return null; - } - - if (subject != null) { - writeLine(subject, predicate, id, subjectCell, objectCell); - } - - writeChildren(id, objectCell); - - return id; - } - } - - protected class CellValueTransposedNode implements WritingTransposedNode { - protected JSONObject obj; - protected CellValueNode node; - protected Cell cell; - - public CellValueTransposedNode(CellValueNode node, Cell cell) { - this.node = node; - this.cell = cell; - } - - public String write(String subject, String predicate, Cell subjectCell) { - if (subject != null) { - if ("/type/text".equals(node.lang)) { - writeLine(subject, predicate, cell.value, node.lang, subjectCell); - } else { - writeLine(subject, predicate, cell.value, subjectCell, null); - } - } - - return cell.value.toString(); - } - } - - protected class CellKeyTransposedNode implements WritingTransposedNode { - protected CellKeyNode node; - protected Cell cell; - - public CellKeyTransposedNode(CellKeyNode node, Cell cell) { - this.node = node; - this.cell = cell; - } - - public String write(String subject, String predicate, Cell subjectCell) { - writeLine(subject, "key", node.namespace.id + "/" + cell.value, subjectCell, null); - - return null; - } - } - - protected class TopicTransposedNode extends TransposedNodeWithChildren { - protected FreebaseTopicNode node; - - public TopicTransposedNode(FreebaseTopicNode node) { - this.node = node; - } - - public String write(String subject, String predicate, Cell subjectCell) { - writeLine(subject, predicate, node.topic.id, subjectCell, null); - writeChildren(node.topic.id, null); - - return node.topic.id; - } - } - - protected class ValueTransposedNode implements WritingTransposedNode { - protected ValueNode node; - - public ValueTransposedNode(ValueNode node) { - this.node = node; - } - - public String write(String subject, String predicate, Cell subjectCell) { - if ("/type/text".equals(node.lang)) { - writeLine(subject, predicate, node.value, node.lang, subjectCell); - } else { - writeLine(subject, predicate, node.value, subjectCell, null); - } - - return node.value.toString(); - } - } - - public TransposedNode transposeAnonymousNode( - TransposedNode parentNode, - FreebaseProperty property, - AnonymousNode node) { - - WritingTransposedNode tnode = new AnonymousTransposedNode(); - - processTransposedNode(tnode, parentNode, property); - - return tnode; - } - - public TransposedNode transposeCellNode( - TransposedNode parentNode, - FreebaseProperty property, - CellNode node, - Cell cell) { - - WritingTransposedNode tnode = null; - if (node instanceof CellTopicNode) { - tnode = new CellTopicTransposedNode((CellTopicNode) node, cell); - } else if (node instanceof CellValueNode) { - tnode = new CellValueTransposedNode((CellValueNode) node, cell); - } else if (node instanceof CellKeyNode) { - tnode = new CellKeyTransposedNode((CellKeyNode) node, cell); - } - - if (tnode != null) { - processTransposedNode(tnode, parentNode, property); - } - return tnode; - } - - public TransposedNode transposeTopicNode( - TransposedNode parentNode, - FreebaseProperty property, - FreebaseTopicNode node) { - - WritingTransposedNode tnode = new TopicTransposedNode(node); - - processTransposedNode(tnode, parentNode, property); - - return tnode; - } - - public TransposedNode transposeValueNode( - TransposedNode parentNode, - FreebaseProperty property, - ValueNode node) { - - WritingTransposedNode tnode = new ValueTransposedNode(node); - - processTransposedNode(tnode, parentNode, property); - - return tnode; - } - - protected void processTransposedNode( - WritingTransposedNode tnode, - TransposedNode parentNode, - FreebaseProperty property - ) { - if (parentNode != null) { - if (parentNode instanceof TransposedNodeWithChildren) { - TransposedNodeWithChildren parentNode2 = (TransposedNodeWithChildren) parentNode; - parentNode2.children.add(tnode); - parentNode2.properties.add(property); - } - } else { - addRootNode(tnode); - } - } - - protected void addRootNode(WritingTransposedNode tnode) { - if (lastRootNode != null) { - lastRootNode.write(null, null, null); - } - lastRootNode = tnode; - } -} +package com.metaweb.gridworks.protograph.transpose; + +import java.io.IOException; +import java.io.Writer; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.json.JSONObject; + +import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.Recon; +import com.metaweb.gridworks.model.Recon.Judgment; +import com.metaweb.gridworks.protograph.AnonymousNode; +import com.metaweb.gridworks.protograph.CellKeyNode; +import com.metaweb.gridworks.protograph.CellNode; +import com.metaweb.gridworks.protograph.CellTopicNode; +import com.metaweb.gridworks.protograph.CellValueNode; +import com.metaweb.gridworks.protograph.FreebaseProperty; +import com.metaweb.gridworks.protograph.FreebaseTopicNode; +import com.metaweb.gridworks.protograph.ValueNode; + +public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory { + protected boolean start = true; + protected Writer writer; + protected WritingTransposedNode lastRootNode; + protected Map varPool = new HashMap(); + protected Map newTopicVars = new HashMap(); + protected Set serializedRecons = new HashSet(); + + public TripleLoaderTransposedNodeFactory(Writer writer) { + this.writer = writer; + } + + public void flush() { + if (lastRootNode != null) { + lastRootNode.write(null, null, null); + lastRootNode = null; + } + } + + protected void writeLine(String line) { + try { + if (start) { + start = false; + } else { + writer.write('\n'); + } + writer.write(line); + } catch (IOException e) { + // ignore + } + } + + protected void writeRecon(StringBuffer sb, Cell cell) { + Recon recon = cell.recon; + if (serializedRecons.contains(recon.id)) { + sb.append(Long.toString(recon.id)); + } else { + serializedRecons.add(recon.id); + + String s = cell.value instanceof String ? (String) cell.value : cell.value.toString(); + + sb.append("{ "); + sb.append("\"id\" : "); sb.append(Long.toString(recon.id)); + sb.append(", \"history_entry\" : "); sb.append(Long.toString(recon.judgmentHistoryEntry)); + sb.append(", \"text\" : "); sb.append(JSONObject.quote(s)); + sb.append(", \"service\" : "); sb.append(JSONObject.quote(recon.service)); + sb.append(", \"action\" : "); sb.append(JSONObject.quote(recon.judgmentAction)); + sb.append(", \"batch\" : "); sb.append(Integer.toString(recon.judgmentBatchSize)); + sb.append(", \"matchRank\" : "); sb.append(Integer.toString(recon.matchRank)); + sb.append(" }"); + } + } + + protected void writeLine(String subject, String predicate, Object object, Cell subjectCell, Cell objectCell) { + if (subject != null && object != null) { + String s = object instanceof String ? + JSONObject.quote((String) object) : object.toString(); + + StringBuffer sb = new StringBuffer(); + sb.append("{ \"s\" : \""); sb.append(subject); sb.append("\""); + sb.append(", \"p\" : \""); sb.append(predicate); sb.append("\""); + sb.append(", \"o\" : "); sb.append(s); + if (subjectCell != null || objectCell != null) { + sb.append(", \"meta\" : { "); + + if (subjectCell != null) { + sb.append("\"srecon\" : "); + writeRecon(sb, subjectCell); + } + if (objectCell != null) { + if (subjectCell != null) { + sb.append(", "); + } + sb.append("\"orecon\" : "); + writeRecon(sb, objectCell); + } + + sb.append(" }"); + } + sb.append(" }"); + + writeLine(sb.toString()); + } + } + + protected void writeLine(String subject, String predicate, Object object, String lang, Cell subjectCell) { + if (subject != null && object != null) { + String s = object instanceof String ? + JSONObject.quote((String) object) : object.toString(); + + StringBuffer sb = new StringBuffer(); + sb.append("{ \"s\" : \""); sb.append(subject); sb.append("\""); + sb.append(", \"p\" : \""); sb.append(predicate); sb.append("\""); + sb.append(", \"o\" : "); sb.append(s); + sb.append(", \"lang\" : "); sb.append(lang); + + if (subjectCell != null) { + sb.append(", \"meta\" : { "); + sb.append("\"srecon\" : "); + writeRecon(sb, subjectCell); + sb.append(" }"); + } + sb.append(" }"); + + writeLine(sb.toString()); + } + } + + protected interface WritingTransposedNode extends TransposedNode { + public String write(String subject, String predicate, Cell subjectCell); + } + + abstract protected class TransposedNodeWithChildren implements WritingTransposedNode { + public List properties = new LinkedList(); + public List children = new LinkedList(); + + protected void writeChildren(String subject, Cell subjectCell) { + for (int i = 0; i < children.size(); i++) { + WritingTransposedNode child = children.get(i); + String predicate = properties.get(i).id; + + child.write(subject, predicate, subjectCell); + } + } + } + + protected class AnonymousTransposedNode extends TransposedNodeWithChildren { + + //protected AnonymousTransposedNode(AnonymousNode node) { } + + public String write(String subject, String predicate, Cell subjectCell) { + if (children.size() == 0 || subject == null) { + return null; + } + + StringBuffer sb = new StringBuffer(); + sb.append("{ "); + + boolean first = true; + for (int i = 0; i < children.size(); i++) { + String s = children.get(i).write(null, null, null); + if (s != null) { + if (first) { + first = false; + } else { + sb.append(", "); + } + sb.append("\"" + properties.get(i).id + "\": "); + sb.append(JSONObject.quote(s)); + } + } + sb.append(" }"); + + writeLine(subject, predicate, sb, subjectCell, null); + + return null; + } + } + + protected class CellTopicTransposedNode extends TransposedNodeWithChildren { + protected CellTopicNode node; + protected Cell cell; + + public CellTopicTransposedNode(CellTopicNode node, Cell cell) { + this.node = node; + this.cell = cell; + } + + public String write(String subject, String predicate, Cell subjectCell) { + String id = null; + Cell objectCell = null; + + if (cell.recon != null && + cell.recon.judgment == Recon.Judgment.Matched && + cell.recon.match != null) { + + objectCell = cell; + id = cell.recon.match.topicID; + } else if (node.createForNoReconMatch || + (cell.recon != null && cell.recon.judgment == Judgment.New)) { + if (cell.recon != null && newTopicVars.containsKey(cell.recon.id)) { + id = newTopicVars.get(cell.recon.id); + } else { + long var = 0; + if (varPool.containsKey(node.columnName)) { + var = varPool.get(node.columnName); + } + varPool.put(node.columnName, var + 1); + + id = "$" + node.columnName.replaceAll("\\W+", "_") + "_" + var; + + writeLine(id, "type", node.type.id, (Cell) null, (Cell) null); + writeLine(id, "name", cell.value, (Cell) null, (Cell) null); + + if (cell.recon != null) { + newTopicVars.put(cell.recon.id, id); + } + } + } else { + return null; + } + + if (subject != null) { + writeLine(subject, predicate, id, subjectCell, objectCell); + } + + writeChildren(id, objectCell); + + return id; + } + } + + protected class CellValueTransposedNode implements WritingTransposedNode { + protected JSONObject obj; + protected CellValueNode node; + protected Cell cell; + + public CellValueTransposedNode(CellValueNode node, Cell cell) { + this.node = node; + this.cell = cell; + } + + public String write(String subject, String predicate, Cell subjectCell) { + if (subject != null) { + if ("/type/text".equals(node.lang)) { + writeLine(subject, predicate, cell.value, node.lang, subjectCell); + } else { + writeLine(subject, predicate, cell.value, subjectCell, null); + } + } + + return cell.value.toString(); + } + } + + protected class CellKeyTransposedNode implements WritingTransposedNode { + protected CellKeyNode node; + protected Cell cell; + + public CellKeyTransposedNode(CellKeyNode node, Cell cell) { + this.node = node; + this.cell = cell; + } + + public String write(String subject, String predicate, Cell subjectCell) { + writeLine(subject, "key", node.namespace.id + "/" + cell.value, subjectCell, null); + + return null; + } + } + + protected class TopicTransposedNode extends TransposedNodeWithChildren { + protected FreebaseTopicNode node; + + public TopicTransposedNode(FreebaseTopicNode node) { + this.node = node; + } + + public String write(String subject, String predicate, Cell subjectCell) { + writeLine(subject, predicate, node.topic.id, subjectCell, null); + writeChildren(node.topic.id, null); + + return node.topic.id; + } + } + + protected class ValueTransposedNode implements WritingTransposedNode { + protected ValueNode node; + + public ValueTransposedNode(ValueNode node) { + this.node = node; + } + + public String write(String subject, String predicate, Cell subjectCell) { + if ("/type/text".equals(node.lang)) { + writeLine(subject, predicate, node.value, node.lang, subjectCell); + } else { + writeLine(subject, predicate, node.value, subjectCell, null); + } + + return node.value.toString(); + } + } + + public TransposedNode transposeAnonymousNode( + TransposedNode parentNode, + FreebaseProperty property, + AnonymousNode node) { + + WritingTransposedNode tnode = new AnonymousTransposedNode(); + + processTransposedNode(tnode, parentNode, property); + + return tnode; + } + + public TransposedNode transposeCellNode( + TransposedNode parentNode, + FreebaseProperty property, + CellNode node, + Cell cell) { + + WritingTransposedNode tnode = null; + if (node instanceof CellTopicNode) { + tnode = new CellTopicTransposedNode((CellTopicNode) node, cell); + } else if (node instanceof CellValueNode) { + tnode = new CellValueTransposedNode((CellValueNode) node, cell); + } else if (node instanceof CellKeyNode) { + tnode = new CellKeyTransposedNode((CellKeyNode) node, cell); + } + + if (tnode != null) { + processTransposedNode(tnode, parentNode, property); + } + return tnode; + } + + public TransposedNode transposeTopicNode( + TransposedNode parentNode, + FreebaseProperty property, + FreebaseTopicNode node) { + + WritingTransposedNode tnode = new TopicTransposedNode(node); + + processTransposedNode(tnode, parentNode, property); + + return tnode; + } + + public TransposedNode transposeValueNode( + TransposedNode parentNode, + FreebaseProperty property, + ValueNode node) { + + WritingTransposedNode tnode = new ValueTransposedNode(node); + + processTransposedNode(tnode, parentNode, property); + + return tnode; + } + + protected void processTransposedNode( + WritingTransposedNode tnode, + TransposedNode parentNode, + FreebaseProperty property + ) { + if (parentNode != null) { + if (parentNode instanceof TransposedNodeWithChildren) { + TransposedNodeWithChildren parentNode2 = (TransposedNodeWithChildren) parentNode; + parentNode2.children.add(tnode); + parentNode2.properties.add(property); + } + } else { + addRootNode(tnode); + } + } + + protected void addRootNode(WritingTransposedNode tnode) { + if (lastRootNode != null) { + lastRootNode.write(null, null, null); + } + lastRootNode = tnode; + } +} diff --git a/src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java b/src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java index 4c3bdac79..4a5b8a3c3 100644 --- a/src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java +++ b/src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java @@ -35,19 +35,19 @@ public class FreebaseDataExtensionJob { } static public class ColumnInfo { - final public List names; - final public List path; - final public FreebaseType expectedType; - - protected ColumnInfo(List names, List path, FreebaseType expectedType) { - this.names = names; - this.path = path; - this.expectedType = expectedType; - } + final public List names; + final public List path; + final public FreebaseType expectedType; + + protected ColumnInfo(List names, List path, FreebaseType expectedType) { + this.names = names; + this.path = path; + this.expectedType = expectedType; + } } - final public JSONObject extension; - final public int columnCount; + final public JSONObject extension; + final public int columnCount; final public List columns = new ArrayList(); public FreebaseDataExtensionJob(JSONObject obj) throws JSONException { @@ -57,9 +57,9 @@ public class FreebaseDataExtensionJob { } public Map extend( - Set guids, + Set guids, Map reconCandidateMap - ) throws Exception { + ) throws Exception { StringWriter writer = new StringWriter(); formulateQuery(guids, extension, writer); @@ -92,9 +92,9 @@ public class FreebaseDataExtensionJob { } protected FreebaseDataExtensionJob.DataExtension collectResult( - JSONObject obj, + JSONObject obj, Map reconCandidateMap - ) throws JSONException { + ) throws JSONException { List rows = new ArrayList(); collectResult(rows, extension.getJSONArray("properties"), obj, 0, 0, reconCandidateMap); @@ -125,22 +125,22 @@ public class FreebaseDataExtensionJob { JSONObject obj, Map reconCandidateMap ) throws JSONException { - String guid = obj.getString("guid"); - ReconCandidate rc; - if (reconCandidateMap.containsKey(guid)) { - rc = reconCandidateMap.get(guid); - } else { - rc = new ReconCandidate( + String guid = obj.getString("guid"); + ReconCandidate rc; + if (reconCandidateMap.containsKey(guid)) { + rc = reconCandidateMap.get(guid); + } else { + rc = new ReconCandidate( obj.getString("id"), obj.getString("guid"), obj.getString("name"), JSONUtilities.getStringArray(obj, "type"), 100 ); - - reconCandidateMap.put(guid, rc); - } - + + reconCandidateMap.put(guid, rc); + } + storeCell(rows, row, col, rc, reconCandidateMap); } @@ -372,18 +372,18 @@ public class FreebaseDataExtensionJob { } static protected int countColumns(JSONObject obj, List columns, List names, List path) throws JSONException { - String name = obj.getString("name"); - - List names2 = null; - List path2 = null; - if (columns != null) { - names2 = new ArrayList(names); - names2.add(name); - - path2 = new ArrayList(path); - path2.add(obj.getString("id")); - } - + String name = obj.getString("name"); + + List names2 = null; + List path2 = null; + if (columns != null) { + names2 = new ArrayList(names); + names2.add(name); + + path2 = new ArrayList(path); + path2.add(obj.getString("id")); + } + if (obj.has("properties") && !obj.isNull("properties")) { boolean included = (obj.has("included") && obj.getBoolean("included")); if (included && columns != null) { @@ -394,7 +394,7 @@ public class FreebaseDataExtensionJob { } return (included ? 1 : 0) + - countColumns(obj.getJSONArray("properties"), columns, names2, path2); + countColumns(obj.getJSONArray("properties"), columns, names2, path2); } else { if (columns != null) { JSONObject expected = obj.getJSONObject("expected"); diff --git a/src/main/java/com/metaweb/gridworks/util/IndentWriter.java b/src/main/java/com/metaweb/gridworks/util/IndentWriter.java index 471c1c6f4..455b2090f 100644 --- a/src/main/java/com/metaweb/gridworks/util/IndentWriter.java +++ b/src/main/java/com/metaweb/gridworks/util/IndentWriter.java @@ -1,77 +1,77 @@ -/* - * Created on Dec 1, 2005 - * Created by dfhuynh - */ -package com.metaweb.gridworks.util; - -import java.io.IOException; -import java.io.Writer; - -/** - * A utility for writing indented code. - * - * @author dfhuynh - */ -public class IndentWriter { - final static private int s_max = 20; - - static private String[] s_indents = new String[s_max]; - static { - for (int i = 0; i < s_max; i++) { - StringBuffer sb = new StringBuffer(s_max); - for (int j = 0; j < i; j++) { - sb.append('\t'); - } - s_indents[i] = sb.toString(); - } - } - - private Writer m_writer; - private int m_count = 0; - private boolean m_indent = true; - - public IndentWriter(Writer writer) { - m_writer = writer; - } - - public void close() throws IOException { - m_writer.close(); - } - - public void flush() throws IOException { - m_writer.flush(); - } - - public void print(Object o) throws IOException { - printIndent(); - m_writer.write(o.toString()); - m_indent = false; - } - - public void println() throws IOException { - printIndent(); - m_writer.write("\n"); - m_indent = true; - } - - public void println(Object o) throws IOException { - printIndent(); - m_writer.write(o.toString()); - m_writer.write("\n"); - m_indent = true; - } - - public void indent() { - m_count++; - } - - public void unindent() { - m_count--; - } - - private void printIndent() throws IOException { - if (m_indent) { - m_writer.write(s_indents[m_count]); - } - } -} +/* + * Created on Dec 1, 2005 + * Created by dfhuynh + */ +package com.metaweb.gridworks.util; + +import java.io.IOException; +import java.io.Writer; + +/** + * A utility for writing indented code. + * + * @author dfhuynh + */ +public class IndentWriter { + final static private int s_max = 20; + + static private String[] s_indents = new String[s_max]; + static { + for (int i = 0; i < s_max; i++) { + StringBuffer sb = new StringBuffer(s_max); + for (int j = 0; j < i; j++) { + sb.append('\t'); + } + s_indents[i] = sb.toString(); + } + } + + private Writer m_writer; + private int m_count = 0; + private boolean m_indent = true; + + public IndentWriter(Writer writer) { + m_writer = writer; + } + + public void close() throws IOException { + m_writer.close(); + } + + public void flush() throws IOException { + m_writer.flush(); + } + + public void print(Object o) throws IOException { + printIndent(); + m_writer.write(o.toString()); + m_indent = false; + } + + public void println() throws IOException { + printIndent(); + m_writer.write("\n"); + m_indent = true; + } + + public void println(Object o) throws IOException { + printIndent(); + m_writer.write(o.toString()); + m_writer.write("\n"); + m_indent = true; + } + + public void indent() { + m_count++; + } + + public void unindent() { + m_count--; + } + + private void printIndent() throws IOException { + if (m_indent) { + m_writer.write(s_indents[m_count]); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/util/JSObject.java b/src/main/java/com/metaweb/gridworks/util/JSObject.java index fe46512cc..3920877aa 100644 --- a/src/main/java/com/metaweb/gridworks/util/JSObject.java +++ b/src/main/java/com/metaweb/gridworks/util/JSObject.java @@ -1,133 +1,133 @@ -package com.metaweb.gridworks.util; - -import java.io.IOException; -import java.util.Collection; -import java.util.Enumeration; -import java.util.Iterator; -import java.util.Properties; - -import org.apache.commons.lang.StringEscapeUtils; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -/** - * A utility class for encapsulating a Javascript object that can - * then be pretty-printed out through an IndentWriter. - * - * @author dfhuynh - */ -public class JSObject extends Properties { - private static final long serialVersionUID = 5864375136126385719L; - - @SuppressWarnings("unchecked") - static public void writeJSObject(IndentWriter writer, JSObject jso) throws IOException, JSONException { - writer.println("{"); - writer.indent(); - { - Enumeration e = jso.propertyNames(); - while (e.hasMoreElements()) { - String name = (String) e.nextElement(); - Object value = jso.get(name); - - writer.print("'"); - writer.print(name + "' : "); - writeObject(writer, value); - - if (e.hasMoreElements()) { - writer.println(","); - } else { - writer.println(); - } - } - } - writer.unindent(); - writer.print("}"); - } - - @SuppressWarnings("unchecked") - static public void writeCollection(IndentWriter writer, Collection c) throws IOException, JSONException { - writer.println("["); - writer.indent(); - { - Iterator i = c.iterator(); - while (i.hasNext()) { - writeObject(writer, i.next()); - if (i.hasNext()) { - writer.println(","); - } else { - writer.println(); - } - } - } - writer.unindent(); - writer.print("]"); - } - - static public void writeJSONObject(IndentWriter writer, JSONObject no) throws IOException, JSONException { - writer.println("{"); - writer.indent(); - { - String[] names = JSONObject.getNames(no); - for (int i = 0; i < names.length; i++) { - String name = names[i]; - Object value = no.get(name); - - writer.print("'"); - writer.print(name + "' : "); - writeObject(writer, value); - - if (i < names.length - 1) { - writer.println(","); - } else { - writer.println(); - } - } - } - writer.unindent(); - writer.print("}"); - } - - static public void writeJSONArray(IndentWriter writer, JSONArray na) throws IOException, JSONException { - writer.println("["); - writer.indent(); - { - int count = na.length(); - for (int i = 0; i < count; i++) { - Object element = na.get(i); - - writeObject(writer, element); - if (i < count - 1) { - writer.println(","); - } else { - writer.println(); - } - } - } - writer.unindent(); - writer.print("]"); - } - - @SuppressWarnings("unchecked") - static public void writeObject(IndentWriter writer, Object o) throws IOException, JSONException { - if (o == null) { - writer.print("null"); - } else if (o instanceof Boolean) { - writer.print(((Boolean) o).booleanValue() ? "true" : "false"); - } else if (o instanceof Number) { - writer.print(((Number) o).toString()); - - } else if (o instanceof Collection) { - writeCollection(writer, (Collection) o); - } else if (o instanceof JSONArray) { - writeJSONArray(writer, (JSONArray) o); - } else if (o instanceof JSObject) { - writeJSObject(writer, (JSObject) o); - } else if (o instanceof JSONObject) { - writeJSONObject(writer, (JSONObject) o); - - } else { - writer.print("\"" + StringEscapeUtils.escapeJavaScript(o.toString()) + "\""); - } - } -} +package com.metaweb.gridworks.util; + +import java.io.IOException; +import java.util.Collection; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.Properties; + +import org.apache.commons.lang.StringEscapeUtils; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * A utility class for encapsulating a Javascript object that can + * then be pretty-printed out through an IndentWriter. + * + * @author dfhuynh + */ +public class JSObject extends Properties { + private static final long serialVersionUID = 5864375136126385719L; + + @SuppressWarnings("unchecked") + static public void writeJSObject(IndentWriter writer, JSObject jso) throws IOException, JSONException { + writer.println("{"); + writer.indent(); + { + Enumeration e = jso.propertyNames(); + while (e.hasMoreElements()) { + String name = (String) e.nextElement(); + Object value = jso.get(name); + + writer.print("'"); + writer.print(name + "' : "); + writeObject(writer, value); + + if (e.hasMoreElements()) { + writer.println(","); + } else { + writer.println(); + } + } + } + writer.unindent(); + writer.print("}"); + } + + @SuppressWarnings("unchecked") + static public void writeCollection(IndentWriter writer, Collection c) throws IOException, JSONException { + writer.println("["); + writer.indent(); + { + Iterator i = c.iterator(); + while (i.hasNext()) { + writeObject(writer, i.next()); + if (i.hasNext()) { + writer.println(","); + } else { + writer.println(); + } + } + } + writer.unindent(); + writer.print("]"); + } + + static public void writeJSONObject(IndentWriter writer, JSONObject no) throws IOException, JSONException { + writer.println("{"); + writer.indent(); + { + String[] names = JSONObject.getNames(no); + for (int i = 0; i < names.length; i++) { + String name = names[i]; + Object value = no.get(name); + + writer.print("'"); + writer.print(name + "' : "); + writeObject(writer, value); + + if (i < names.length - 1) { + writer.println(","); + } else { + writer.println(); + } + } + } + writer.unindent(); + writer.print("}"); + } + + static public void writeJSONArray(IndentWriter writer, JSONArray na) throws IOException, JSONException { + writer.println("["); + writer.indent(); + { + int count = na.length(); + for (int i = 0; i < count; i++) { + Object element = na.get(i); + + writeObject(writer, element); + if (i < count - 1) { + writer.println(","); + } else { + writer.println(); + } + } + } + writer.unindent(); + writer.print("]"); + } + + @SuppressWarnings("unchecked") + static public void writeObject(IndentWriter writer, Object o) throws IOException, JSONException { + if (o == null) { + writer.print("null"); + } else if (o instanceof Boolean) { + writer.print(((Boolean) o).booleanValue() ? "true" : "false"); + } else if (o instanceof Number) { + writer.print(((Number) o).toString()); + + } else if (o instanceof Collection) { + writeCollection(writer, (Collection) o); + } else if (o instanceof JSONArray) { + writeJSONArray(writer, (JSONArray) o); + } else if (o instanceof JSObject) { + writeJSObject(writer, (JSObject) o); + } else if (o instanceof JSONObject) { + writeJSONObject(writer, (JSONObject) o); + + } else { + writer.print("\"" + StringEscapeUtils.escapeJavaScript(o.toString()) + "\""); + } + } +} diff --git a/src/main/java/com/metaweb/gridworks/util/ParsingUtilities.java b/src/main/java/com/metaweb/gridworks/util/ParsingUtilities.java index 94662f37b..5891d4b16 100644 --- a/src/main/java/com/metaweb/gridworks/util/ParsingUtilities.java +++ b/src/main/java/com/metaweb/gridworks/util/ParsingUtilities.java @@ -1,120 +1,120 @@ -package com.metaweb.gridworks.util; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.UnsupportedEncodingException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Properties; - -import javax.servlet.http.HttpServletRequest; - -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.net.URLCodec; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONTokener; - -public class ParsingUtilities { - - static final public SimpleDateFormat s_sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - - static public Properties parseUrlParameters(HttpServletRequest request) { - Properties options = new Properties(); - - String query = request.getQueryString(); - if (query != null) { - if (query.startsWith("?")) { - query = query.substring(1); - } - - parseParameters(options,query); - } - return options; - } - - static public Properties parseParameters(Properties p, String str) { - if (str != null) { - String[] pairs = str.split("&"); - for (String pairString : pairs) { - int equal = pairString.indexOf('='); - String name = (equal >= 0) ? pairString.substring(0, equal) : ""; - String value = (equal >= 0) ? ParsingUtilities.decode(pairString.substring(equal + 1)) : ""; - p.put(name, value); - } - } - return p; - } - - static public Properties parseParameters(String str) { - return (str == null) ? null : parseParameters(new Properties(),str); - } - - static public String inputStreamToString(InputStream is) throws IOException { - Reader reader = new InputStreamReader(is, "UTF-8"); - try { - return readerToString(reader); - } finally { - reader.close(); - } - } - - static public String readerToString(Reader reader) throws IOException { - StringBuffer sb = new StringBuffer(); - - char[] chars = new char[8192]; - int c; - - while ((c = reader.read(chars)) > 0) { - sb.insert(sb.length(), chars, 0, c); - } - - return sb.toString(); - } - - static public JSONObject evaluateJsonStringToObject(String s) throws JSONException { - JSONTokener t = new JSONTokener(s); - JSONObject o = (JSONObject) t.nextValue(); - return o; - } - - static public JSONArray evaluateJsonStringToArray(String s) throws JSONException { - JSONTokener t = new JSONTokener(s); - JSONArray a = (JSONArray) t.nextValue(); - return a; - } - - private static final URLCodec codec = new URLCodec(); - static public String encode(String s) { - try { - return codec.encode(s, "UTF-8"); - } catch (UnsupportedEncodingException e) { - return s; // should not happen - } - } - static public String decode(String s) { - try { - return codec.decode(s, "UTF-8"); - } catch (UnsupportedEncodingException e) { - return s; // should not happen - } catch (DecoderException e) { - return s; // should not happen - } - } - - static public String dateToString(Date d) { - return s_sdf.format(d); - } - - static public Date stringToDate(String s) { - try { - return s_sdf.parse(s); - } catch (ParseException e) { - return null; - } - } -} +package com.metaweb.gridworks.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.UnsupportedEncodingException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Properties; + +import javax.servlet.http.HttpServletRequest; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.net.URLCodec; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; + +public class ParsingUtilities { + + static final public SimpleDateFormat s_sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + + static public Properties parseUrlParameters(HttpServletRequest request) { + Properties options = new Properties(); + + String query = request.getQueryString(); + if (query != null) { + if (query.startsWith("?")) { + query = query.substring(1); + } + + parseParameters(options,query); + } + return options; + } + + static public Properties parseParameters(Properties p, String str) { + if (str != null) { + String[] pairs = str.split("&"); + for (String pairString : pairs) { + int equal = pairString.indexOf('='); + String name = (equal >= 0) ? pairString.substring(0, equal) : ""; + String value = (equal >= 0) ? ParsingUtilities.decode(pairString.substring(equal + 1)) : ""; + p.put(name, value); + } + } + return p; + } + + static public Properties parseParameters(String str) { + return (str == null) ? null : parseParameters(new Properties(),str); + } + + static public String inputStreamToString(InputStream is) throws IOException { + Reader reader = new InputStreamReader(is, "UTF-8"); + try { + return readerToString(reader); + } finally { + reader.close(); + } + } + + static public String readerToString(Reader reader) throws IOException { + StringBuffer sb = new StringBuffer(); + + char[] chars = new char[8192]; + int c; + + while ((c = reader.read(chars)) > 0) { + sb.insert(sb.length(), chars, 0, c); + } + + return sb.toString(); + } + + static public JSONObject evaluateJsonStringToObject(String s) throws JSONException { + JSONTokener t = new JSONTokener(s); + JSONObject o = (JSONObject) t.nextValue(); + return o; + } + + static public JSONArray evaluateJsonStringToArray(String s) throws JSONException { + JSONTokener t = new JSONTokener(s); + JSONArray a = (JSONArray) t.nextValue(); + return a; + } + + private static final URLCodec codec = new URLCodec(); + static public String encode(String s) { + try { + return codec.encode(s, "UTF-8"); + } catch (UnsupportedEncodingException e) { + return s; // should not happen + } + } + static public String decode(String s) { + try { + return codec.decode(s, "UTF-8"); + } catch (UnsupportedEncodingException e) { + return s; // should not happen + } catch (DecoderException e) { + return s; // should not happen + } + } + + static public String dateToString(Date d) { + return s_sdf.format(d); + } + + static public Date stringToDate(String s) { + try { + return s_sdf.parse(s); + } catch (ParseException e) { + return null; + } + } +}