From e4e73d0b9201a80a0d31e258f2ea601340de1bfe Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Mon, 5 Nov 2018 16:35:51 +0000 Subject: [PATCH] Migrate importers to use Jackson classes --- .../src/com/google/refine/ProjectManager.java | 1 - .../refine/browsing/DecoratedValue.java | 1 - .../com/google/refine/browsing/Engine.java | 1 - .../google/refine/browsing/EngineConfig.java | 1 - .../refine/browsing/facets/FacetConfig.java | 1 - .../browsing/facets/FacetConfigResolver.java | 1 - .../refine/browsing/facets/ListFacet.java | 1 - .../browsing/facets/NominalFacetChoice.java | 1 - .../refine/browsing/facets/RangeFacet.java | 1 - .../browsing/facets/ScatterplotFacet.java | 1 - .../browsing/facets/TextSearchFacet.java | 1 - .../browsing/facets/TimeRangeFacet.java | 1 - .../refine/clustering/ClustererConfig.java | 1 - .../clustering/binning/BinningClusterer.java | 1 - .../refine/clustering/knn/kNNClusterer.java | 1 - .../com/google/refine/commands/Command.java | 1 - .../refine/commands/GetPreferenceCommand.java | 1 - .../refine/commands/HttpHeadersSupport.java | 1 - .../google/refine/commands/HttpUtilities.java | 1 - .../refine/commands/SetPreferenceCommand.java | 3 - .../commands/cell/EditOneCellCommand.java | 1 - .../refine/commands/cell/MassEditCommand.java | 1 - .../AddColumnByFetchingURLsCommand.java | 1 - .../column/GetColumnsInfoCommand.java | 1 - .../expr/GetExpressionHistoryCommand.java | 1 - .../GetExpressionLanguageInfoCommand.java | 1 - .../expr/GetStarredExpressionsCommand.java | 1 - .../expr/PreviewExpressionCommand.java | 1 - .../history/GetOperationsCommand.java | 1 - .../GetImportingConfigurationCommand.java | 1 - .../GetImportingJobStatusCommand.java | 1 - .../commands/lang/GetLanguagesCommand.java | 1 - .../project/CreateProjectCommand.java | 24 +-- .../commands/project/GetModelsCommand.java | 1 - .../recon/GuessTypesOfColumnCommand.java | 2 - .../recon/PreviewExtendDataCommand.java | 1 - .../recon/ReconClearOneCellCommand.java | 1 - .../ReconUseValuesAsIdentifiersCommand.java | 5 - .../refine/commands/row/GetRowsCommand.java | 1 - .../GetAllProjectMetadataCommand.java | 1 - .../workspace/GetAllProjectTagsCommand.java | 1 - .../google/refine/exporters/CsvExporter.java | 1 - .../refine/exporters/HtmlTableExporter.java | 2 - .../google/refine/exporters/OdsExporter.java | 1 - .../refine/exporters/TemplatingExporter.java | 1 - .../google/refine/exporters/XlsExporter.java | 1 - .../refine/exporters/sql/SqlExporter.java | 2 - .../com/google/refine/expr/MetaParser.java | 1 - .../refine/expr/functions/strings/Trim.java | 1 - main/src/com/google/refine/grel/Control.java | 1 - .../com/google/refine/history/History.java | 1 - .../google/refine/history/HistoryEntry.java | 1 - .../google/refine/history/HistoryProcess.java | 1 - .../refine/importers/ExcelImporter.java | 27 +-- .../refine/importers/FixedWidthImporter.java | 18 +- .../refine/importers/ImporterUtilities.java | 7 +- .../refine/importers/ImportingParserBase.java | 25 +-- .../google/refine/importers/JsonImporter.java | 51 +++--- .../refine/importers/LineBasedImporter.java | 10 +- .../google/refine/importers/MarcImporter.java | 8 +- .../google/refine/importers/OdsImporter.java | 27 +-- .../refine/importers/RdfTripleImporter.java | 4 +- .../importers/SeparatorBasedImporter.java | 27 +-- .../importers/TabularImportingParserBase.java | 13 +- .../refine/importers/WikitextImporter.java | 11 +- .../google/refine/importers/XmlImporter.java | 49 +++--- .../tree/TreeImportingParserBase.java | 24 +-- .../importing/DefaultImportingController.java | 37 ++-- .../google/refine/importing/ImportingJob.java | 40 ++--- .../refine/importing/ImportingManager.java | 1 - .../refine/importing/ImportingParser.java | 13 +- .../refine/importing/ImportingUtilities.java | 160 +++++++++--------- .../google/refine/io/FileProjectManager.java | 1 - .../refine/model/AbstractOperation.java | 1 - main/src/com/google/refine/model/Cell.java | 1 - main/src/com/google/refine/model/Column.java | 1 - .../com/google/refine/model/ColumnGroup.java | 1 - main/src/com/google/refine/model/Recon.java | 1 - .../google/refine/model/ReconCandidate.java | 1 - .../com/google/refine/model/ReconStats.java | 1 - .../com/google/refine/model/ReconType.java | 1 - .../com/google/refine/model/RecordModel.java | 1 - main/src/com/google/refine/model/Row.java | 1 - .../refine/model/changes/MassChange.java | 1 - .../model/metadata/ProjectMetadata.java | 6 +- .../refine/model/recon/ReconConfig.java | 1 - .../recon/ReconciledDataExtensionJob.java | 1 - .../model/recon/StandardReconConfig.java | 1 - .../EngineDependentMassCellOperation.java | 1 - .../operations/EngineDependentOperation.java | 1 - .../refine/operations/OperationRegistry.java | 9 - .../refine/operations/OperationResolver.java | 1 - .../operations/cell/BlankDownOperation.java | 1 - .../operations/cell/FillDownOperation.java | 1 - .../cell/KeyValueColumnizeOperation.java | 1 - .../operations/cell/MassEditOperation.java | 1 - .../cell/MultiValuedCellJoinOperation.java | 1 - .../cell/MultiValuedCellSplitOperation.java | 1 - .../cell/TextTransformOperation.java | 1 - .../TransposeColumnsIntoRowsOperation.java | 1 - .../TransposeRowsIntoColumnsOperation.java | 1 - ...ColumnAdditionByFetchingURLsOperation.java | 2 - .../column/ColumnAdditionOperation.java | 1 - .../column/ColumnMoveOperation.java | 1 - .../column/ColumnRemovalOperation.java | 1 - .../column/ColumnRenameOperation.java | 1 - .../column/ColumnReorderOperation.java | 1 - .../column/ColumnSplitOperation.java | 1 - .../operations/recon/ExtendDataOperation.java | 1 - .../ReconClearSimilarCellsOperation.java | 1 - .../ReconCopyAcrossColumnsOperation.java | 1 - .../recon/ReconDiscardJudgmentsOperation.java | 1 - .../ReconJudgeSimilarCellsOperation.java | 1 - .../recon/ReconMarkNewTopicsOperation.java | 1 - .../ReconMatchBestCandidatesOperation.java | 1 - .../ReconMatchSpecificTopicOperation.java | 1 - .../operations/recon/ReconOperation.java | 1 - .../ReconUseValuesAsIdentifiersOperation.java | 1 - .../operations/row/DenormalizeOperation.java | 1 - .../operations/row/RowFlagOperation.java | 1 - .../operations/row/RowRemovalOperation.java | 1 - .../operations/row/RowReorderOperation.java | 1 - .../operations/row/RowStarOperation.java | 1 - .../refine/process/LongRunningProcess.java | 1 - .../com/google/refine/process/Process.java | 1 - .../google/refine/process/ProcessManager.java | 1 - .../process/QuickHistoryEntryProcess.java | 1 - .../com/google/refine/sorting/Criterion.java | 9 +- .../google/refine/sorting/SortingConfig.java | 1 - .../refine/sorting/StringCriterion.java | 1 - .../com/google/refine/util/JSONUtilities.java | 136 ++++++++++++--- .../google/refine/util/ParsingUtilities.java | 27 +++ main/src/com/google/refine/util/Pool.java | 1 - .../refine/util/SerializationFilters.java | 1 - .../com/google/refine/tests/RefineTest.java | 42 ++++- .../refine/tests/browsing/EngineTests.java | 1 + .../browsing/facets/EngineConfigTests.java | 2 +- .../tests/browsing/facets/ListFacetTests.java | 1 - .../browsing/facets/RangeFacetTests.java | 1 - .../facets/ScatterplotFacetTests.java | 1 - .../browsing/facets/TextSearchFacetTests.java | 1 - .../browsing/facets/TimeRangeFacetTests.java | 1 - .../ExpressionNominalValueGrouperTests.java | 8 +- .../expr/ExpressionCommandTestBase.java | 1 - ...GetExpressionLanguageInfoCommandTests.java | 1 - .../expr/PreviewExpressionCommandTests.java | 6 +- .../ToggleStarredExpressionCommandTests.java | 3 +- .../project/ValidateOperationTests.java | 2 +- .../commands/row/GetRowsCommandTest.java | 14 +- .../util/CancelProcessesCommandTests.java | 2 +- .../tests/exporters/sql/SqlExporterTests.java | 1 - .../expr/functions/ContainsFunctionTests.java | 15 +- .../strings/ToFromConversionTests.java | 2 +- .../refine/tests/history/HistoryTests.java | 2 +- .../tests/importers/ExcelImporterTests.java | 22 ++- .../importers/FixedWidthImporterTests.java | 29 +--- .../refine/tests/importers/ImporterTest.java | 15 +- .../tests/importers/JsonImporterTests.java | 27 +-- .../tests/importers/TsvCsvImporterTests.java | 31 +--- .../tests/importers/XmlImporterTests.java | 27 +-- .../importing/ImportingUtilitiesTests.java | 20 +-- .../google/refine/tests/model/CacheTests.java | 1 - .../refine/tests/model/ColumnTests.java | 5 +- .../tests/model/changes/MassChangeTests.java | 2 +- .../cell/KeyValueColumnizeTests.java | 6 +- .../operations/cell/MassOperationTests.java | 1 - .../QuickHistoryEntryProcessTests.java | 3 +- .../PatternSyntaxExceptionParserTests.java | 6 +- .../google/refine/tests/util/TestUtils.java | 1 - 169 files changed, 572 insertions(+), 613 deletions(-) diff --git a/main/src/com/google/refine/ProjectManager.java b/main/src/com/google/refine/ProjectManager.java index 2e90d4624..3738209d0 100644 --- a/main/src/com/google/refine/ProjectManager.java +++ b/main/src/com/google/refine/ProjectManager.java @@ -56,7 +56,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.HistoryEntryManager; import com.google.refine.model.Project; import com.google.refine.model.metadata.IMetadata; diff --git a/main/src/com/google/refine/browsing/DecoratedValue.java b/main/src/com/google/refine/browsing/DecoratedValue.java index 67aa8a09d..909bf3d5b 100644 --- a/main/src/com/google/refine/browsing/DecoratedValue.java +++ b/main/src/com/google/refine/browsing/DecoratedValue.java @@ -37,7 +37,6 @@ import java.time.OffsetDateTime; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.util.StringUtils; /** diff --git a/main/src/com/google/refine/browsing/Engine.java b/main/src/com/google/refine/browsing/Engine.java index dd0249e3a..827555d6d 100644 --- a/main/src/com/google/refine/browsing/Engine.java +++ b/main/src/com/google/refine/browsing/Engine.java @@ -43,7 +43,6 @@ import org.json.JSONObject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.facets.Facet; import com.google.refine.browsing.util.ConjunctiveFilteredRecords; import com.google.refine.browsing.util.ConjunctiveFilteredRows; diff --git a/main/src/com/google/refine/browsing/EngineConfig.java b/main/src/com/google/refine/browsing/EngineConfig.java index 3e9a4c4e4..0898fbd89 100644 --- a/main/src/com/google/refine/browsing/EngineConfig.java +++ b/main/src/com/google/refine/browsing/EngineConfig.java @@ -8,7 +8,6 @@ import org.json.JSONObject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine.Mode; import com.google.refine.browsing.facets.FacetConfig; import com.google.refine.util.ParsingUtilities; diff --git a/main/src/com/google/refine/browsing/facets/FacetConfig.java b/main/src/com/google/refine/browsing/facets/FacetConfig.java index 07ffc9156..e745e2f7a 100644 --- a/main/src/com/google/refine/browsing/facets/FacetConfig.java +++ b/main/src/com/google/refine/browsing/facets/FacetConfig.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; - import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/browsing/facets/FacetConfigResolver.java b/main/src/com/google/refine/browsing/facets/FacetConfigResolver.java index 0cfac814c..e05fde24b 100644 --- a/main/src/com/google/refine/browsing/facets/FacetConfigResolver.java +++ b/main/src/com/google/refine/browsing/facets/FacetConfigResolver.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.DatabindContext; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.jsontype.impl.TypeIdResolverBase; import com.fasterxml.jackson.databind.type.TypeFactory; - import com.google.refine.model.recon.ReconConfig; public class FacetConfigResolver extends TypeIdResolverBase { diff --git a/main/src/com/google/refine/browsing/facets/ListFacet.java b/main/src/com/google/refine/browsing/facets/ListFacet.java index f1f2ebff9..dc0d1eea0 100644 --- a/main/src/com/google/refine/browsing/facets/ListFacet.java +++ b/main/src/com/google/refine/browsing/facets/ListFacet.java @@ -42,7 +42,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.ProjectManager; import com.google.refine.browsing.DecoratedValue; import com.google.refine.browsing.FilteredRecords; diff --git a/main/src/com/google/refine/browsing/facets/NominalFacetChoice.java b/main/src/com/google/refine/browsing/facets/NominalFacetChoice.java index f29a6ae96..bec3dd180 100644 --- a/main/src/com/google/refine/browsing/facets/NominalFacetChoice.java +++ b/main/src/com/google/refine/browsing/facets/NominalFacetChoice.java @@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.browsing.facets; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.DecoratedValue; /** diff --git a/main/src/com/google/refine/browsing/facets/RangeFacet.java b/main/src/com/google/refine/browsing/facets/RangeFacet.java index 4e978849a..f66bdcde6 100644 --- a/main/src/com/google/refine/browsing/facets/RangeFacet.java +++ b/main/src/com/google/refine/browsing/facets/RangeFacet.java @@ -38,7 +38,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.FilteredRecords; import com.google.refine.browsing.FilteredRows; import com.google.refine.browsing.RecordFilter; diff --git a/main/src/com/google/refine/browsing/facets/ScatterplotFacet.java b/main/src/com/google/refine/browsing/facets/ScatterplotFacet.java index f2f3d3243..1aa721ef0 100644 --- a/main/src/com/google/refine/browsing/facets/ScatterplotFacet.java +++ b/main/src/com/google/refine/browsing/facets/ScatterplotFacet.java @@ -51,7 +51,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.FilteredRecords; import com.google.refine.browsing.FilteredRows; import com.google.refine.browsing.RecordFilter; diff --git a/main/src/com/google/refine/browsing/facets/TextSearchFacet.java b/main/src/com/google/refine/browsing/facets/TextSearchFacet.java index 4a5ea6733..12e159565 100644 --- a/main/src/com/google/refine/browsing/facets/TextSearchFacet.java +++ b/main/src/com/google/refine/browsing/facets/TextSearchFacet.java @@ -38,7 +38,6 @@ import java.util.regex.Pattern; import org.json.JSONException; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.FilteredRecords; import com.google.refine.browsing.FilteredRows; import com.google.refine.browsing.RecordFilter; diff --git a/main/src/com/google/refine/browsing/facets/TimeRangeFacet.java b/main/src/com/google/refine/browsing/facets/TimeRangeFacet.java index 63ecfcc39..9d6898115 100644 --- a/main/src/com/google/refine/browsing/facets/TimeRangeFacet.java +++ b/main/src/com/google/refine/browsing/facets/TimeRangeFacet.java @@ -37,7 +37,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.FilteredRecords; import com.google.refine.browsing.FilteredRows; import com.google.refine.browsing.RecordFilter; diff --git a/main/src/com/google/refine/clustering/ClustererConfig.java b/main/src/com/google/refine/clustering/ClustererConfig.java index 5ac10369f..c63346036 100644 --- a/main/src/com/google/refine/clustering/ClustererConfig.java +++ b/main/src/com/google/refine/clustering/ClustererConfig.java @@ -3,7 +3,6 @@ package com.google.refine.clustering; import org.json.JSONObject; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.model.Project; /** diff --git a/main/src/com/google/refine/clustering/binning/BinningClusterer.java b/main/src/com/google/refine/clustering/binning/BinningClusterer.java index b09cd6dcd..f01c116b3 100644 --- a/main/src/com/google/refine/clustering/binning/BinningClusterer.java +++ b/main/src/com/google/refine/clustering/binning/BinningClusterer.java @@ -53,7 +53,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.FilteredRows; import com.google.refine.browsing.RowVisitor; diff --git a/main/src/com/google/refine/clustering/knn/kNNClusterer.java b/main/src/com/google/refine/clustering/knn/kNNClusterer.java index da0a502d8..13ed16434 100644 --- a/main/src/com/google/refine/clustering/knn/kNNClusterer.java +++ b/main/src/com/google/refine/clustering/knn/kNNClusterer.java @@ -50,7 +50,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.FilteredRows; import com.google.refine.browsing.RowVisitor; diff --git a/main/src/com/google/refine/commands/Command.java b/main/src/com/google/refine/commands/Command.java index f60dc15cb..a2af0c768 100644 --- a/main/src/com/google/refine/commands/Command.java +++ b/main/src/com/google/refine/commands/Command.java @@ -51,7 +51,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonGenerator; - import com.google.refine.ProjectManager; import com.google.refine.RefineServlet; import com.google.refine.browsing.Engine; diff --git a/main/src/com/google/refine/commands/GetPreferenceCommand.java b/main/src/com/google/refine/commands/GetPreferenceCommand.java index 2d133c041..8b87dd31e 100644 --- a/main/src/com/google/refine/commands/GetPreferenceCommand.java +++ b/main/src/com/google/refine/commands/GetPreferenceCommand.java @@ -40,7 +40,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.ProjectManager; import com.google.refine.preference.PreferenceStore; import com.google.refine.preference.TopList; diff --git a/main/src/com/google/refine/commands/HttpHeadersSupport.java b/main/src/com/google/refine/commands/HttpHeadersSupport.java index e7cd88b3e..73d850288 100644 --- a/main/src/com/google/refine/commands/HttpHeadersSupport.java +++ b/main/src/com/google/refine/commands/HttpHeadersSupport.java @@ -39,7 +39,6 @@ import java.util.Set; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.RefineServlet; abstract public class HttpHeadersSupport { diff --git a/main/src/com/google/refine/commands/HttpUtilities.java b/main/src/com/google/refine/commands/HttpUtilities.java index 09a8cdd27..6d7d9ddc5 100644 --- a/main/src/com/google/refine/commands/HttpUtilities.java +++ b/main/src/com/google/refine/commands/HttpUtilities.java @@ -17,7 +17,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.JsonGenerator; - import com.google.refine.RefineServlet; import com.google.refine.util.ParsingUtilities; diff --git a/main/src/com/google/refine/commands/SetPreferenceCommand.java b/main/src/com/google/refine/commands/SetPreferenceCommand.java index 877b8dcd7..7dcbf07ba 100644 --- a/main/src/com/google/refine/commands/SetPreferenceCommand.java +++ b/main/src/com/google/refine/commands/SetPreferenceCommand.java @@ -39,9 +39,6 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.json.JSONException; -import org.json.JSONTokener; - import com.fasterxml.jackson.databind.JsonNode; import com.google.refine.ProjectManager; import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/commands/cell/EditOneCellCommand.java b/main/src/com/google/refine/commands/cell/EditOneCellCommand.java index 41c7ab0b3..e299000ce 100644 --- a/main/src/com/google/refine/commands/cell/EditOneCellCommand.java +++ b/main/src/com/google/refine/commands/cell/EditOneCellCommand.java @@ -43,7 +43,6 @@ import javax.servlet.http.HttpServletResponse; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.commands.Command; import com.google.refine.history.Change; import com.google.refine.history.HistoryEntry; diff --git a/main/src/com/google/refine/commands/cell/MassEditCommand.java b/main/src/com/google/refine/commands/cell/MassEditCommand.java index 1b1558db2..eecfb796a 100644 --- a/main/src/com/google/refine/commands/cell/MassEditCommand.java +++ b/main/src/com/google/refine/commands/cell/MassEditCommand.java @@ -38,7 +38,6 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; import com.fasterxml.jackson.core.type.TypeReference; - import com.google.refine.browsing.EngineConfig; import com.google.refine.commands.EngineDependentCommand; import com.google.refine.model.AbstractOperation; diff --git a/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java b/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java index 13c7ea762..2d480b262 100644 --- a/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java +++ b/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java @@ -39,7 +39,6 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; import com.fasterxml.jackson.databind.ObjectMapper; - import com.google.refine.browsing.EngineConfig; import com.google.refine.commands.EngineDependentCommand; import com.google.refine.model.AbstractOperation; diff --git a/main/src/com/google/refine/commands/column/GetColumnsInfoCommand.java b/main/src/com/google/refine/commands/column/GetColumnsInfoCommand.java index e8683a124..378053a5c 100644 --- a/main/src/com/google/refine/commands/column/GetColumnsInfoCommand.java +++ b/main/src/com/google/refine/commands/column/GetColumnsInfoCommand.java @@ -40,7 +40,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import com.fasterxml.jackson.core.JsonGenerator; - import com.google.refine.browsing.util.ExpressionBasedRowEvaluable; import com.google.refine.browsing.util.NumericBinIndex; import com.google.refine.browsing.util.NumericBinRowIndex; diff --git a/main/src/com/google/refine/commands/expr/GetExpressionHistoryCommand.java b/main/src/com/google/refine/commands/expr/GetExpressionHistoryCommand.java index b4f229be9..75a88e406 100644 --- a/main/src/com/google/refine/commands/expr/GetExpressionHistoryCommand.java +++ b/main/src/com/google/refine/commands/expr/GetExpressionHistoryCommand.java @@ -45,7 +45,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.ProjectManager; import com.google.refine.commands.Command; import com.google.refine.preference.TopList; diff --git a/main/src/com/google/refine/commands/expr/GetExpressionLanguageInfoCommand.java b/main/src/com/google/refine/commands/expr/GetExpressionLanguageInfoCommand.java index a47f7b62b..3c4df3089 100644 --- a/main/src/com/google/refine/commands/expr/GetExpressionLanguageInfoCommand.java +++ b/main/src/com/google/refine/commands/expr/GetExpressionLanguageInfoCommand.java @@ -41,7 +41,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.commands.Command; import com.google.refine.grel.Control; import com.google.refine.grel.ControlFunctionRegistry; diff --git a/main/src/com/google/refine/commands/expr/GetStarredExpressionsCommand.java b/main/src/com/google/refine/commands/expr/GetStarredExpressionsCommand.java index 60635c399..245f69390 100644 --- a/main/src/com/google/refine/commands/expr/GetStarredExpressionsCommand.java +++ b/main/src/com/google/refine/commands/expr/GetStarredExpressionsCommand.java @@ -9,7 +9,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.ProjectManager; import com.google.refine.commands.Command; import com.google.refine.preference.TopList; diff --git a/main/src/com/google/refine/commands/expr/PreviewExpressionCommand.java b/main/src/com/google/refine/commands/expr/PreviewExpressionCommand.java index e59ad8b5b..135bf92b5 100644 --- a/main/src/com/google/refine/commands/expr/PreviewExpressionCommand.java +++ b/main/src/com/google/refine/commands/expr/PreviewExpressionCommand.java @@ -52,7 +52,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; - import com.google.refine.commands.Command; import com.google.refine.expr.EvalError; import com.google.refine.expr.Evaluable; diff --git a/main/src/com/google/refine/commands/history/GetOperationsCommand.java b/main/src/com/google/refine/commands/history/GetOperationsCommand.java index 9b3c88829..6b24fe4f0 100644 --- a/main/src/com/google/refine/commands/history/GetOperationsCommand.java +++ b/main/src/com/google/refine/commands/history/GetOperationsCommand.java @@ -44,7 +44,6 @@ import javax.servlet.http.HttpServletResponse; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.commands.Command; import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; diff --git a/main/src/com/google/refine/commands/importing/GetImportingConfigurationCommand.java b/main/src/com/google/refine/commands/importing/GetImportingConfigurationCommand.java index 3dcc418fe..2899114d6 100644 --- a/main/src/com/google/refine/commands/importing/GetImportingConfigurationCommand.java +++ b/main/src/com/google/refine/commands/importing/GetImportingConfigurationCommand.java @@ -40,7 +40,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.commands.Command; import com.google.refine.importing.ImportingManager.ImportingConfiguration; diff --git a/main/src/com/google/refine/commands/importing/GetImportingJobStatusCommand.java b/main/src/com/google/refine/commands/importing/GetImportingJobStatusCommand.java index 4a06556b1..930b22032 100644 --- a/main/src/com/google/refine/commands/importing/GetImportingJobStatusCommand.java +++ b/main/src/com/google/refine/commands/importing/GetImportingJobStatusCommand.java @@ -43,7 +43,6 @@ import javax.servlet.http.HttpServletResponse; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.commands.Command; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingManager; diff --git a/main/src/com/google/refine/commands/lang/GetLanguagesCommand.java b/main/src/com/google/refine/commands/lang/GetLanguagesCommand.java index cd0ea55c9..560533162 100644 --- a/main/src/com/google/refine/commands/lang/GetLanguagesCommand.java +++ b/main/src/com/google/refine/commands/lang/GetLanguagesCommand.java @@ -48,7 +48,6 @@ import org.apache.commons.io.filefilter.WildcardFileFilter; import org.json.JSONObject; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.commands.Command; import edu.mit.simile.butterfly.ButterflyModule; diff --git a/main/src/com/google/refine/commands/project/CreateProjectCommand.java b/main/src/com/google/refine/commands/project/CreateProjectCommand.java index 0d5020fe1..ac34c5cc7 100644 --- a/main/src/com/google/refine/commands/project/CreateProjectCommand.java +++ b/main/src/com/google/refine/commands/project/CreateProjectCommand.java @@ -42,11 +42,11 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.ProjectManager; import com.google.refine.commands.Command; import com.google.refine.commands.HttpUtilities; @@ -69,7 +69,7 @@ public class CreateProjectCommand extends Command { try { Properties parameters = ParsingUtilities.parseUrlParameters(request); ImportingJob job = ImportingManager.createJob(); - JSONObject config = job.getOrCreateDefaultConfig(); + ObjectNode config = job.getOrCreateDefaultConfig(); ImportingUtilities.loadDataAndPrepareJob( request, response, parameters, job, config); @@ -93,9 +93,9 @@ public class CreateProjectCommand extends Command { "\\t".equals(parameters.getProperty("separator"))) { format = "text/line-based/*sv"; } else { - JSONArray rankedFormats = JSONUtilities.getArray(config, "rankedFormats"); - if (rankedFormats != null && rankedFormats.length() > 0) { - format = rankedFormats.getString(0); + ArrayNode rankedFormats = JSONUtilities.getArray(config, "rankedFormats"); + if (rankedFormats != null && rankedFormats.size() > 0) { + format = rankedFormats.get(0).asText(); } } @@ -105,10 +105,10 @@ public class CreateProjectCommand extends Command { } } - JSONObject optionObj = null; - String optionsString = parameters.getProperty("options"); + ObjectNode optionObj = null; + String optionsString = parameters.getParameter("options"); if (optionsString != null && !optionsString.isEmpty()) { - optionObj = ParsingUtilities.evaluateJsonStringToObject(optionsString); + optionObj = ParsingUtilities.evaluateJsonStringToObjectNode(optionsString); } else { Format formatRecord = ImportingManager.formatToRecord.get(format); optionObj = formatRecord.parser.createParserUIInitializationData( @@ -133,7 +133,7 @@ public class CreateProjectCommand extends Command { } } - static private void adjustLegacyOptions(String format, Properties parameters, JSONObject optionObj) { + static private void adjustLegacyOptions(String format, Properties parameters, ObjectNode optionObj) { if (",".equals(parameters.getProperty("separator"))) { JSONUtilities.safePut(optionObj, "separator", ","); } else if ("\\t".equals(parameters.getProperty("separator"))) { @@ -150,7 +150,7 @@ public class CreateProjectCommand extends Command { } static private void adjustLegacyIntegerOption( - String format, Properties parameters, JSONObject optionObj, String legacyName, String newName) { + String format, Properties parameters, ObjectNode optionObj, String legacyName, String newName) { String s = parameters.getProperty(legacyName); if (s != null && !s.isEmpty()) { @@ -165,7 +165,7 @@ public class CreateProjectCommand extends Command { static private void adjustLegacyBooleanOption( String format, Properties parameters, - JSONObject optionObj, + ObjectNode optionObj, String legacyName, String newName, boolean invert) { diff --git a/main/src/com/google/refine/commands/project/GetModelsCommand.java b/main/src/com/google/refine/commands/project/GetModelsCommand.java index 876f5fa76..f42436823 100644 --- a/main/src/com/google/refine/commands/project/GetModelsCommand.java +++ b/main/src/com/google/refine/commands/project/GetModelsCommand.java @@ -44,7 +44,6 @@ import javax.servlet.http.HttpServletResponse; import org.json.JSONException; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.commands.Command; import com.google.refine.commands.HttpHeadersSupport; import com.google.refine.commands.HttpHeadersSupport.HttpHeaderInfo; diff --git a/main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java b/main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java index 9759a80ae..2600e36f3 100644 --- a/main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java +++ b/main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java @@ -36,7 +36,6 @@ package com.google.refine.commands.recon; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.StringWriter; import java.net.HttpURLConnection; import java.net.URL; import java.util.ArrayList; @@ -59,7 +58,6 @@ import org.json.JSONObject; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.commands.Command; import com.google.refine.expr.ExpressionUtils; import com.google.refine.model.Column; diff --git a/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java b/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java index f42675fa0..a571aa4da 100644 --- a/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java +++ b/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java @@ -48,7 +48,6 @@ import javax.servlet.http.HttpServletResponse; import org.json.JSONArray; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.commands.Command; import com.google.refine.model.Cell; import com.google.refine.model.Column; diff --git a/main/src/com/google/refine/commands/recon/ReconClearOneCellCommand.java b/main/src/com/google/refine/commands/recon/ReconClearOneCellCommand.java index 128c4dbad..6234f872e 100644 --- a/main/src/com/google/refine/commands/recon/ReconClearOneCellCommand.java +++ b/main/src/com/google/refine/commands/recon/ReconClearOneCellCommand.java @@ -40,7 +40,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.commands.Command; import com.google.refine.expr.ExpressionUtils; import com.google.refine.history.Change; diff --git a/main/src/com/google/refine/commands/recon/ReconUseValuesAsIdentifiersCommand.java b/main/src/com/google/refine/commands/recon/ReconUseValuesAsIdentifiersCommand.java index cece2bddb..6d13ed369 100644 --- a/main/src/com/google/refine/commands/recon/ReconUseValuesAsIdentifiersCommand.java +++ b/main/src/com/google/refine/commands/recon/ReconUseValuesAsIdentifiersCommand.java @@ -2,15 +2,10 @@ package com.google.refine.commands.recon; import javax.servlet.http.HttpServletRequest; -import org.json.JSONObject; - import com.google.refine.browsing.EngineConfig; import com.google.refine.commands.EngineDependentCommand; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Project; -import com.google.refine.model.ReconCandidate; -import com.google.refine.model.recon.StandardReconConfig; -import com.google.refine.operations.recon.ReconMatchSpecificTopicOperation; import com.google.refine.operations.recon.ReconUseValuesAsIdentifiersOperation; public class ReconUseValuesAsIdentifiersCommand extends EngineDependentCommand { diff --git a/main/src/com/google/refine/commands/row/GetRowsCommand.java b/main/src/com/google/refine/commands/row/GetRowsCommand.java index 9e11f9be7..b69042350 100644 --- a/main/src/com/google/refine/commands/row/GetRowsCommand.java +++ b/main/src/com/google/refine/commands/row/GetRowsCommand.java @@ -49,7 +49,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonUnwrapped; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine.Mode; import com.google.refine.browsing.FilteredRecords; diff --git a/main/src/com/google/refine/commands/workspace/GetAllProjectMetadataCommand.java b/main/src/com/google/refine/commands/workspace/GetAllProjectMetadataCommand.java index 4968dffdd..034e42d4a 100644 --- a/main/src/com/google/refine/commands/workspace/GetAllProjectMetadataCommand.java +++ b/main/src/com/google/refine/commands/workspace/GetAllProjectMetadataCommand.java @@ -46,7 +46,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonRawValue; - import com.google.refine.ProjectManager; import com.google.refine.commands.Command; import com.google.refine.model.metadata.ProjectMetadata; diff --git a/main/src/com/google/refine/commands/workspace/GetAllProjectTagsCommand.java b/main/src/com/google/refine/commands/workspace/GetAllProjectTagsCommand.java index 49330e8bf..10920810e 100644 --- a/main/src/com/google/refine/commands/workspace/GetAllProjectTagsCommand.java +++ b/main/src/com/google/refine/commands/workspace/GetAllProjectTagsCommand.java @@ -37,7 +37,6 @@ import javax.servlet.http.HttpServletResponse; import org.json.JSONException; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.ProjectManager; import com.google.refine.commands.Command; diff --git a/main/src/com/google/refine/exporters/CsvExporter.java b/main/src/com/google/refine/exporters/CsvExporter.java index a25276703..03724ecb8 100644 --- a/main/src/com/google/refine/exporters/CsvExporter.java +++ b/main/src/com/google/refine/exporters/CsvExporter.java @@ -43,7 +43,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; - import com.google.refine.browsing.Engine; import com.google.refine.model.Project; import com.google.refine.util.ParsingUtilities; diff --git a/main/src/com/google/refine/exporters/HtmlTableExporter.java b/main/src/com/google/refine/exporters/HtmlTableExporter.java index 530a6e9d5..0f3f3ff0c 100644 --- a/main/src/com/google/refine/exporters/HtmlTableExporter.java +++ b/main/src/com/google/refine/exporters/HtmlTableExporter.java @@ -39,10 +39,8 @@ import java.util.List; import java.util.Properties; import org.apache.commons.lang3.StringEscapeUtils; -import org.json.JSONObject; import com.fasterxml.jackson.databind.JsonNode; - import com.google.refine.ProjectManager; import com.google.refine.browsing.Engine; import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/exporters/OdsExporter.java b/main/src/com/google/refine/exporters/OdsExporter.java index e397d0ded..64ef9d19b 100644 --- a/main/src/com/google/refine/exporters/OdsExporter.java +++ b/main/src/com/google/refine/exporters/OdsExporter.java @@ -45,7 +45,6 @@ import org.odftoolkit.odfdom.doc.table.OdfTableCell; import org.odftoolkit.odfdom.doc.table.OdfTableRow; import com.fasterxml.jackson.databind.JsonNode; - import com.google.refine.ProjectManager; import com.google.refine.browsing.Engine; import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/exporters/TemplatingExporter.java b/main/src/com/google/refine/exporters/TemplatingExporter.java index 05568fbf5..a18a62b57 100644 --- a/main/src/com/google/refine/exporters/TemplatingExporter.java +++ b/main/src/com/google/refine/exporters/TemplatingExporter.java @@ -41,7 +41,6 @@ import org.json.JSONException; import org.json.JSONObject; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine.Mode; import com.google.refine.browsing.FilteredRecords; diff --git a/main/src/com/google/refine/exporters/XlsExporter.java b/main/src/com/google/refine/exporters/XlsExporter.java index 3910f5cef..79ffdaecd 100644 --- a/main/src/com/google/refine/exporters/XlsExporter.java +++ b/main/src/com/google/refine/exporters/XlsExporter.java @@ -50,7 +50,6 @@ import org.apache.poi.ss.util.WorkbookUtil; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import com.fasterxml.jackson.databind.JsonNode; - import com.google.refine.ProjectManager; import com.google.refine.browsing.Engine; import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/exporters/sql/SqlExporter.java b/main/src/com/google/refine/exporters/sql/SqlExporter.java index dd00a2f00..1bc55fd35 100755 --- a/main/src/com/google/refine/exporters/sql/SqlExporter.java +++ b/main/src/com/google/refine/exporters/sql/SqlExporter.java @@ -35,12 +35,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.JsonNode; - import com.google.refine.ProjectManager; import com.google.refine.browsing.Engine; import com.google.refine.exporters.CustomizableTabularExporterUtilities; diff --git a/main/src/com/google/refine/expr/MetaParser.java b/main/src/com/google/refine/expr/MetaParser.java index 03ec48949..7473868ab 100644 --- a/main/src/com/google/refine/expr/MetaParser.java +++ b/main/src/com/google/refine/expr/MetaParser.java @@ -41,7 +41,6 @@ import java.util.Set; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.grel.Parser; import clojure.lang.IFn; diff --git a/main/src/com/google/refine/expr/functions/strings/Trim.java b/main/src/com/google/refine/expr/functions/strings/Trim.java index 68e6efd10..6d75a74ca 100644 --- a/main/src/com/google/refine/expr/functions/strings/Trim.java +++ b/main/src/com/google/refine/expr/functions/strings/Trim.java @@ -36,7 +36,6 @@ package com.google.refine.expr.functions.strings; import java.util.Properties; import com.google.common.base.CharMatcher; - import com.google.refine.expr.EvalError; import com.google.refine.grel.Function; diff --git a/main/src/com/google/refine/grel/Control.java b/main/src/com/google/refine/grel/Control.java index 392017e23..5ff09b19a 100644 --- a/main/src/com/google/refine/grel/Control.java +++ b/main/src/com/google/refine/grel/Control.java @@ -38,7 +38,6 @@ import java.util.Properties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.expr.Evaluable; /** diff --git a/main/src/com/google/refine/history/History.java b/main/src/com/google/refine/history/History.java index e667e3503..bd0181c7e 100644 --- a/main/src/com/google/refine/history/History.java +++ b/main/src/com/google/refine/history/History.java @@ -47,7 +47,6 @@ import java.util.List; import java.util.Properties; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.ProjectManager; import com.google.refine.RefineServlet; import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/history/HistoryEntry.java b/main/src/com/google/refine/history/HistoryEntry.java index 137bae283..108df9416 100644 --- a/main/src/com/google/refine/history/HistoryEntry.java +++ b/main/src/com/google/refine/history/HistoryEntry.java @@ -45,7 +45,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonView; - import com.google.refine.ProjectManager; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/history/HistoryProcess.java b/main/src/com/google/refine/history/HistoryProcess.java index 80a5ccbba..903ac250d 100644 --- a/main/src/com/google/refine/history/HistoryProcess.java +++ b/main/src/com/google/refine/history/HistoryProcess.java @@ -35,7 +35,6 @@ package com.google.refine.history; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.model.Project; import com.google.refine.process.Process; import com.google.refine.process.ProcessManager; diff --git a/main/src/com/google/refine/importers/ExcelImporter.java b/main/src/com/google/refine/importers/ExcelImporter.java index 78d912084..efdd8b159 100644 --- a/main/src/com/google/refine/importers/ExcelImporter.java +++ b/main/src/com/google/refine/importers/ExcelImporter.java @@ -54,12 +54,12 @@ import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook; -import org.json.JSONArray; import org.json.JSONException; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingUtilities; import com.google.refine.model.Cell; @@ -69,6 +69,7 @@ import com.google.refine.model.Recon.Judgment; import com.google.refine.model.ReconCandidate; import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; public class ExcelImporter extends TabularImportingParserBase { static final Logger logger = LoggerFactory.getLogger(ExcelImporter.class); @@ -78,15 +79,15 @@ public class ExcelImporter extends TabularImportingParserBase { } @Override - public JSONObject createParserUIInitializationData( - ImportingJob job, List fileRecords, String format) { - JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); + public ObjectNode createParserUIInitializationData( + ImportingJob job, List fileRecords, String format) { + ObjectNode options = super.createParserUIInitializationData(job, fileRecords, format); - JSONArray sheetRecords = new JSONArray(); + ArrayNode sheetRecords = ParsingUtilities.mapper.createArrayNode(); JSONUtilities.safePut(options, "sheetRecords", sheetRecords); try { for (int index = 0;index < fileRecords.size();index++) { - JSONObject fileRecord = fileRecords.get(index); + ObjectNode fileRecord = fileRecords.get(index); File file = ImportingUtilities.getFile(job, fileRecord); InputStream is = new FileInputStream(file); @@ -104,7 +105,7 @@ public class ExcelImporter extends TabularImportingParserBase { Sheet sheet = wb.getSheetAt(i); int rows = sheet.getLastRowNum() - sheet.getFirstRowNum() + 1; - JSONObject sheetRecord = new JSONObject(); + ObjectNode sheetRecord = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(sheetRecord, "name", file.getName() + "#" + sheet.getSheetName()); JSONUtilities.safePut(sheetRecord, "fileNameAndSheetIndex", file.getName() + "#" + i); JSONUtilities.safePut(sheetRecord, "rows", rows); @@ -138,7 +139,7 @@ public class ExcelImporter extends TabularImportingParserBase { String fileSource, InputStream inputStream, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { Workbook wb = null; @@ -182,14 +183,14 @@ public class ExcelImporter extends TabularImportingParserBase { return; } - JSONArray sheets = JSONUtilities.getArray(options, "sheets"); + ArrayNode sheets = (ArrayNode) options.get("sheets"); - for(int i=0;i fileRecords, String format) { - JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); - JSONArray columnWidths = new JSONArray(); + public ObjectNode createParserUIInitializationData( + ImportingJob job, List fileRecords, String format) { + ObjectNode options = super.createParserUIInitializationData(job, fileRecords, format); + ArrayNode columnWidths = ParsingUtilities.mapper.createArrayNode(); if (fileRecords.size() > 0) { - JSONObject firstFileRecord = fileRecords.get(0); + ObjectNode firstFileRecord = fileRecords.get(0); String encoding = ImportingUtilities.getEncoding(firstFileRecord); String location = JSONUtilities.getString(firstFileRecord, "location", null); if (location != null) { @@ -59,7 +59,7 @@ public class FixedWidthImporter extends TabularImportingParserBase { String fileSource, Reader reader, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { final int[] columnWidths = JSONUtilities.getIntArray(options, "columnWidths"); diff --git a/main/src/com/google/refine/importers/ImporterUtilities.java b/main/src/com/google/refine/importers/ImporterUtilities.java index 7d543a3fb..d66ebb75e 100644 --- a/main/src/com/google/refine/importers/ImporterUtilities.java +++ b/main/src/com/google/refine/importers/ImporterUtilities.java @@ -43,8 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import org.json.JSONObject; - +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingUtilities; import com.google.refine.model.Column; @@ -197,9 +196,9 @@ public class ImporterUtilities { } static public MultiFileReadingProgress createMultiFileReadingProgress( - final ImportingJob job, List fileRecords) { + final ImportingJob job, List fileRecords) { long totalSize = 0; - for (JSONObject fileRecord : fileRecords) { + for (ObjectNode fileRecord : fileRecords) { File file = ImportingUtilities.getFile(job, fileRecord); totalSize += file.length(); } diff --git a/main/src/com/google/refine/importers/ImportingParserBase.java b/main/src/com/google/refine/importers/ImportingParserBase.java index bcb272d61..c37f02430 100644 --- a/main/src/com/google/refine/importers/ImportingParserBase.java +++ b/main/src/com/google/refine/importers/ImportingParserBase.java @@ -40,10 +40,10 @@ import java.io.Reader; import java.util.List; import org.json.JSONException; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importers.ImporterUtilities.MultiFileReadingProgress; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingParser; @@ -53,6 +53,7 @@ import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; abstract public class ImportingParserBase implements ImportingParser { final static Logger logger = LoggerFactory.getLogger("ImportingParserBase"); @@ -67,9 +68,9 @@ abstract public class ImportingParserBase implements ImportingParser { } @Override - public JSONObject createParserUIInitializationData(ImportingJob job, - List fileRecords, String format) { - JSONObject options = new JSONObject(); + public ObjectNode createParserUIInitializationData(ImportingJob job, + List fileRecords, String format) { + ObjectNode options = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(options, "includeFileSources", fileRecords.size() > 1); return options; @@ -77,10 +78,10 @@ abstract public class ImportingParserBase implements ImportingParser { @Override public void parse(Project project, ProjectMetadata metadata, - final ImportingJob job, List fileRecords, String format, - int limit, JSONObject options, List exceptions) { + final ImportingJob job, List fileRecords, String format, + int limit, ObjectNode options, List exceptions) { MultiFileReadingProgress progress = ImporterUtilities.createMultiFileReadingProgress(job, fileRecords); - for (JSONObject fileRecord : fileRecords) { + for (ObjectNode fileRecord : fileRecords) { if (job.canceled) { break; } @@ -101,9 +102,9 @@ abstract public class ImportingParserBase implements ImportingParser { Project project, ProjectMetadata metadata, ImportingJob job, - JSONObject fileRecord, + ObjectNode fileRecord, int limit, - JSONObject options, + ObjectNode options, List exceptions, final MultiFileReadingProgress progress ) throws IOException { @@ -142,13 +143,13 @@ abstract public class ImportingParserBase implements ImportingParser { String fileSource, Reader reader, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { pushImportingOptions(metadata, fileSource, options); } - private void pushImportingOptions(ProjectMetadata metadata, String fileSource, JSONObject options) { + private void pushImportingOptions(ProjectMetadata metadata, String fileSource, ObjectNode options) { try { options.put("fileSource", fileSource); } catch (JSONException e) { @@ -165,7 +166,7 @@ abstract public class ImportingParserBase implements ImportingParser { String fileSource, InputStream inputStream, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { pushImportingOptions(metadata, fileSource, options); diff --git a/main/src/com/google/refine/importers/JsonImporter.java b/main/src/com/google/refine/importers/JsonImporter.java index 4c7661549..25390755e 100644 --- a/main/src/com/google/refine/importers/JsonImporter.java +++ b/main/src/com/google/refine/importers/JsonImporter.java @@ -39,8 +39,6 @@ import java.io.InputStream; import java.io.Serializable; import java.util.List; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,7 +47,13 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonParser.NumberType; import com.fasterxml.jackson.core.JsonToken; - +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.DoubleNode; +import com.fasterxml.jackson.databind.node.LongNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import com.google.refine.importers.tree.ImportColumnGroup; import com.google.refine.importers.tree.TreeImportingParserBase; import com.google.refine.importers.tree.TreeReader; @@ -59,6 +63,7 @@ import com.google.refine.importing.ImportingUtilities; import com.google.refine.model.Project; import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; public class JsonImporter extends TreeImportingParserBase { static final Logger logger = LoggerFactory.getLogger(JsonImporter.class); @@ -76,18 +81,18 @@ public class JsonImporter extends TreeImportingParserBase { final static private int PREVIEW_PARSING_LIMIT = 1000; @Override - public JSONObject createParserUIInitializationData( - ImportingJob job, List fileRecords, String format) { - JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); + public ObjectNode createParserUIInitializationData(ImportingJob job, + List fileRecords, String format) { + ObjectNode options = super.createParserUIInitializationData(job, fileRecords, format); if (fileRecords.size() > 0) { try { - JSONObject firstFileRecord = fileRecords.get(0); + ObjectNode firstFileRecord = fileRecords.get(0); File file = ImportingUtilities.getFile(job, firstFileRecord); JsonFactory factory = new JsonFactory(); JsonParser parser = factory.createJsonParser(file); PreviewParsingState state = new PreviewParsingState(); - Object rootValue = parseForPreview(parser, state); + JsonNode rootValue = parseForPreview(parser, state); if (rootValue != null) { JSONUtilities.safePut(options, "dom", rootValue); } @@ -99,7 +104,7 @@ public class JsonImporter extends TreeImportingParserBase { return options; } - final static private Object parseForPreview(JsonParser parser, PreviewParsingState state, JsonToken token) + final static private JsonNode parseForPreview(JsonParser parser, PreviewParsingState state, JsonToken token) throws JsonParseException, IOException { if (token != null) { switch (token) { @@ -108,15 +113,15 @@ public class JsonImporter extends TreeImportingParserBase { case START_OBJECT: return parseObjectForPreview(parser, state); case VALUE_STRING: - return parser.getText(); + return new TextNode(parser.getText()); case VALUE_NUMBER_INT: - return Long.valueOf(parser.getLongValue()); + return new LongNode(parser.getLongValue()); case VALUE_NUMBER_FLOAT: - return Double.valueOf(parser.getDoubleValue()); + return new DoubleNode(parser.getDoubleValue()); case VALUE_TRUE: - return Boolean.TRUE; + return BooleanNode.getTrue(); case VALUE_FALSE: - return Boolean.FALSE; + return BooleanNode.getFalse(); case VALUE_NULL: return null; case END_ARRAY: @@ -131,7 +136,7 @@ public class JsonImporter extends TreeImportingParserBase { return null; } - final static private Object parseForPreview(JsonParser parser, PreviewParsingState state) { + final static private JsonNode parseForPreview(JsonParser parser, PreviewParsingState state) { try { JsonToken token = parser.nextToken(); state.tokenCount++; @@ -141,8 +146,8 @@ public class JsonImporter extends TreeImportingParserBase { } } - final static private JSONObject parseObjectForPreview(JsonParser parser, PreviewParsingState state) { - JSONObject result = new JSONObject(); + final static private ObjectNode parseObjectForPreview(JsonParser parser, PreviewParsingState state) { + ObjectNode result = ParsingUtilities.mapper.createObjectNode(); loop:while (state.tokenCount < PREVIEW_PARSING_LIMIT) { try { JsonToken token = parser.nextToken(); @@ -154,7 +159,7 @@ public class JsonImporter extends TreeImportingParserBase { switch (token) { case FIELD_NAME: String fieldName = parser.getText(); - Object fieldValue = parseForPreview(parser, state); + JsonNode fieldValue = parseForPreview(parser, state); JSONUtilities.safePut(result, fieldName, fieldValue); break; case END_OBJECT: @@ -169,8 +174,8 @@ public class JsonImporter extends TreeImportingParserBase { return result; } - final static private JSONArray parseArrayForPreview(JsonParser parser, PreviewParsingState state) { - JSONArray result = new JSONArray(); + final static private ArrayNode parseArrayForPreview(JsonParser parser, PreviewParsingState state) { + ArrayNode result = ParsingUtilities.mapper.createArrayNode(); loop:while (state.tokenCount < PREVIEW_PARSING_LIMIT) { try { JsonToken token = parser.nextToken(); @@ -183,8 +188,8 @@ public class JsonImporter extends TreeImportingParserBase { case END_ARRAY: break loop; default: - Object element = parseForPreview(parser, state, token); - JSONUtilities.append(result, element); + JsonNode element = parseForPreview(parser, state, token); + result.add(element); } } catch (IOException e) { break; @@ -196,7 +201,7 @@ public class JsonImporter extends TreeImportingParserBase { @Override public void parseOneFile(Project project, ProjectMetadata metadata, ImportingJob job, String fileSource, InputStream is, - ImportColumnGroup rootColumnGroup, int limit, JSONObject options, List exceptions) { + ImportColumnGroup rootColumnGroup, int limit, ObjectNode options, List exceptions) { parseOneFile(project, metadata, job, fileSource, new JSONTreeReader(is), rootColumnGroup, limit, options, exceptions); diff --git a/main/src/com/google/refine/importers/LineBasedImporter.java b/main/src/com/google/refine/importers/LineBasedImporter.java index eb1ba6c08..f8829fed5 100644 --- a/main/src/com/google/refine/importers/LineBasedImporter.java +++ b/main/src/com/google/refine/importers/LineBasedImporter.java @@ -6,10 +6,10 @@ import java.io.Reader; import java.util.ArrayList; import java.util.List; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importing.ImportingJob; import com.google.refine.model.Project; import com.google.refine.model.metadata.ProjectMetadata; @@ -23,9 +23,9 @@ public class LineBasedImporter extends TabularImportingParserBase { } @Override - public JSONObject createParserUIInitializationData( - ImportingJob job, List fileRecords, String format) { - JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); + public ObjectNode createParserUIInitializationData( + ImportingJob job, List fileRecords, String format) { + ObjectNode options = super.createParserUIInitializationData(job, fileRecords, format); JSONUtilities.safePut(options, "linesPerRow", 1); JSONUtilities.safePut(options, "headerLines", 0); @@ -42,7 +42,7 @@ public class LineBasedImporter extends TabularImportingParserBase { String fileSource, Reader reader, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { final int linesPerRow = JSONUtilities.getInt(options, "linesPerRow", 1); diff --git a/main/src/com/google/refine/importers/MarcImporter.java b/main/src/com/google/refine/importers/MarcImporter.java index 4c1c35717..acb52c5ce 100644 --- a/main/src/com/google/refine/importers/MarcImporter.java +++ b/main/src/com/google/refine/importers/MarcImporter.java @@ -40,12 +40,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import org.json.JSONObject; import org.marc4j.MarcPermissiveStreamReader; import org.marc4j.MarcWriter; import org.marc4j.MarcXmlWriter; import org.marc4j.marc.Record; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingUtilities; import com.google.refine.util.JSONUtilities; @@ -57,9 +57,9 @@ public class MarcImporter extends XmlImporter { } @Override - public JSONObject createParserUIInitializationData(ImportingJob job, java.util.List fileRecords, String format) { + public ObjectNode createParserUIInitializationData(ImportingJob job, java.util.List fileRecords, String format) { if (fileRecords.size() > 0) { - JSONObject firstFileRecord = fileRecords.get(0); + ObjectNode firstFileRecord = fileRecords.get(0); File file = ImportingUtilities.getFile(job, firstFileRecord); File tempFile = new File(file.getAbsolutePath()+".xml"); @@ -97,7 +97,7 @@ public class MarcImporter extends XmlImporter { logger.error("Failed to create temporary XML file from MARC file", e); } } - JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); + ObjectNode options = super.createParserUIInitializationData(job, fileRecords, format); return options; }; diff --git a/main/src/com/google/refine/importers/OdsImporter.java b/main/src/com/google/refine/importers/OdsImporter.java index a03f3ff66..74e13a2d3 100644 --- a/main/src/com/google/refine/importers/OdsImporter.java +++ b/main/src/com/google/refine/importers/OdsImporter.java @@ -45,9 +45,7 @@ import java.util.List; import java.util.Map; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.json.JSONArray; import org.json.JSONException; -import org.json.JSONObject; import org.odftoolkit.odfdom.doc.OdfDocument; import org.odftoolkit.odfdom.doc.table.OdfTable; import org.odftoolkit.odfdom.doc.table.OdfTableCell; @@ -55,6 +53,8 @@ import org.odftoolkit.odfdom.doc.table.OdfTableRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingUtilities; import com.google.refine.model.Cell; @@ -64,6 +64,7 @@ import com.google.refine.model.Recon.Judgment; import com.google.refine.model.ReconCandidate; import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; public class OdsImporter extends TabularImportingParserBase { @@ -75,16 +76,16 @@ public class OdsImporter extends TabularImportingParserBase { @Override - public JSONObject createParserUIInitializationData( - ImportingJob job, List fileRecords, String format) { - JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); + public ObjectNode createParserUIInitializationData( + ImportingJob job, List fileRecords, String format) { + ObjectNode options = super.createParserUIInitializationData(job, fileRecords, format); - JSONArray sheetRecords = new JSONArray(); + ArrayNode sheetRecords = ParsingUtilities.mapper.createArrayNode(); JSONUtilities.safePut(options, "sheetRecords", sheetRecords); OdfDocument odfDoc = null; try { for (int index = 0;index < fileRecords.size();index++) { - JSONObject fileRecord = fileRecords.get(index); + ObjectNode fileRecord = fileRecords.get(index); File file = ImportingUtilities.getFile(job, fileRecord); InputStream is = new FileInputStream(file); odfDoc = OdfDocument.loadDocument(is); @@ -95,7 +96,7 @@ public class OdsImporter extends TabularImportingParserBase { OdfTable sheet = tables.get(i); int rows = sheet.getRowCount(); - JSONObject sheetRecord = new JSONObject(); + ObjectNode sheetRecord = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(sheetRecord, "name", file.getName() + "#" + sheet.getTableName()); JSONUtilities.safePut(sheetRecord, "fileNameAndSheetIndex", file.getName() + "#" + i); JSONUtilities.safePut(sheetRecord, "rows", rows); @@ -129,7 +130,7 @@ public class OdsImporter extends TabularImportingParserBase { String fileSource, InputStream inputStream, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { OdfDocument odfDoc; @@ -142,13 +143,13 @@ public class OdsImporter extends TabularImportingParserBase { List tables = odfDoc.getTableList(); - JSONArray sheets = JSONUtilities.getArray(options, "sheets"); - for(int i=0;i exceptions) { + InputStream input, int limit, ObjectNode options, List exceptions) { // create an empty model Model model = ModelFactory.createDefaultModel(); diff --git a/main/src/com/google/refine/importers/SeparatorBasedImporter.java b/main/src/com/google/refine/importers/SeparatorBasedImporter.java index 80920aaae..cd086e47f 100644 --- a/main/src/com/google/refine/importers/SeparatorBasedImporter.java +++ b/main/src/com/google/refine/importers/SeparatorBasedImporter.java @@ -50,8 +50,8 @@ import java.util.List; import java.util.Map; import org.apache.commons.lang3.StringEscapeUtils; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingUtilities; import com.google.refine.model.Project; @@ -66,9 +66,9 @@ public class SeparatorBasedImporter extends TabularImportingParserBase { } @Override - public JSONObject createParserUIInitializationData(ImportingJob job, - List fileRecords, String format) { - JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); + public ObjectNode createParserUIInitializationData(ImportingJob job, + List fileRecords, String format) { + ObjectNode options = super.createParserUIInitializationData(job, fileRecords, format); String separator = guessSeparator(job, fileRecords); JSONUtilities.safePut(options, "separator", separator != null ? separator : "\\t"); @@ -88,7 +88,7 @@ public class SeparatorBasedImporter extends TabularImportingParserBase { String fileSource, Reader reader, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { String sep = JSONUtilities.getString(options, "separator", "\\t"); @@ -119,6 +119,7 @@ public class SeparatorBasedImporter extends TabularImportingParserBase { } } } + final List columnNames = retrievedColumnNames; Character quote = CSVParser.DEFAULT_QUOTE_CHARACTER; @@ -144,14 +145,14 @@ public class SeparatorBasedImporter extends TabularImportingParserBase { if (columnNames != null && !usedColumnNames) { usedColumnNames = true; return columnNames; - } else { - String line = lnReader.readLine(); - if (line == null) { - return null; } else { - return getCells(line, parser, lnReader); + String line = lnReader.readLine(); + if (line == null) { + return null; + } else { + return getCells(line, parser, lnReader); + } } - } } }; @@ -172,9 +173,9 @@ public class SeparatorBasedImporter extends TabularImportingParserBase { return cells; } - static public String guessSeparator(ImportingJob job, List fileRecords) { + static public String guessSeparator(ImportingJob job, List fileRecords) { for (int i = 0; i < 5 && i < fileRecords.size(); i++) { - JSONObject fileRecord = fileRecords.get(i); + ObjectNode fileRecord = fileRecords.get(i); String encoding = ImportingUtilities.getEncoding(fileRecord); String location = JSONUtilities.getString(fileRecord, "location", null); diff --git a/main/src/com/google/refine/importers/TabularImportingParserBase.java b/main/src/com/google/refine/importers/TabularImportingParserBase.java index efdcc0fe2..3018c67e5 100644 --- a/main/src/com/google/refine/importers/TabularImportingParserBase.java +++ b/main/src/com/google/refine/importers/TabularImportingParserBase.java @@ -39,8 +39,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; -import org.json.JSONObject; - +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.expr.ExpressionUtils; import com.google.refine.importing.ImportingJob; import com.google.refine.model.Cell; @@ -56,9 +55,9 @@ abstract public class TabularImportingParserBase extends ImportingParserBase { } @Override - public JSONObject createParserUIInitializationData(ImportingJob job, - List fileRecords, String format) { - JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); + public ObjectNode createParserUIInitializationData(ImportingJob job, + List fileRecords, String format) { + ObjectNode options = super.createParserUIInitializationData(job, fileRecords, format); JSONUtilities.safePut(options, "ignoreLines", -1); // number of blank lines at the beginning to ignore JSONUtilities.safePut(options, "headerLines", 1); // number of header lines @@ -85,7 +84,7 @@ abstract public class TabularImportingParserBase extends ImportingParserBase { TableDataReader reader, String fileSource, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { int ignoreLines = JSONUtilities.getInt(options, "ignoreLines", -1); @@ -203,7 +202,7 @@ abstract public class TabularImportingParserBase extends ImportingParserBase { } public void parseOneFile(Project project, ProjectMetadata metadata, ImportingJob job, String fileSource, - Reader dataReader, int limit, JSONObject options, List exceptions) { + Reader dataReader, int limit, ObjectNode options, List exceptions) { super.parseOneFile(project, metadata, job, fileSource, dataReader, limit, options, exceptions); } } diff --git a/main/src/com/google/refine/importers/WikitextImporter.java b/main/src/com/google/refine/importers/WikitextImporter.java index 8bc509759..429492f38 100644 --- a/main/src/com/google/refine/importers/WikitextImporter.java +++ b/main/src/com/google/refine/importers/WikitextImporter.java @@ -9,7 +9,6 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.json.JSONObject; import org.sweble.wikitext.parser.ParserConfig; import org.sweble.wikitext.parser.WikitextEncodingValidator; import org.sweble.wikitext.parser.WikitextParser; @@ -51,8 +50,8 @@ import org.sweble.wikitext.parser.parser.PreprocessorToParserTransformer; import org.sweble.wikitext.parser.preprocessor.PreprocessedWikitext; import org.sweble.wikitext.parser.utils.SimpleParserConfig; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.io.CharStreams; - import com.google.refine.importing.ImportingJob; import com.google.refine.model.Cell; import com.google.refine.model.Column; @@ -77,9 +76,9 @@ public class WikitextImporter extends TabularImportingParserBase { } @Override - public JSONObject createParserUIInitializationData( - ImportingJob job, List fileRecords, String format) { - JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); + public ObjectNode createParserUIInitializationData( + ImportingJob job, List fileRecords, String format) { + ObjectNode options = super.createParserUIInitializationData(job, fileRecords, format); JSONUtilities.safePut(options, "guessCellValueTypes", false); JSONUtilities.safePut(options, "blankSpanningCells", true); @@ -668,7 +667,7 @@ public class WikitextImporter extends TabularImportingParserBase { String fileSource, Reader reader, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { // Set-up a simple wiki configuration diff --git a/main/src/com/google/refine/importers/XmlImporter.java b/main/src/com/google/refine/importers/XmlImporter.java index 7a3643091..0e67fbc89 100644 --- a/main/src/com/google/refine/importers/XmlImporter.java +++ b/main/src/com/google/refine/importers/XmlImporter.java @@ -46,11 +46,11 @@ import javax.xml.stream.XMLStreamConstants; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importers.tree.ImportColumnGroup; import com.google.refine.importers.tree.TreeImportingParserBase; import com.google.refine.importers.tree.TreeReader; @@ -60,6 +60,7 @@ import com.google.refine.importing.ImportingUtilities; import com.google.refine.model.Project; import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; public class XmlImporter extends TreeImportingParserBase { static final Logger logger = LoggerFactory.getLogger(XmlImporter.class); @@ -75,12 +76,12 @@ public class XmlImporter extends TreeImportingParserBase { final static private int PREVIEW_PARSING_LIMIT = 1000; @Override - public JSONObject createParserUIInitializationData( - ImportingJob job, List fileRecords, String format) { - JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); + public ObjectNode createParserUIInitializationData( + ImportingJob job, List fileRecords, String format) { + ObjectNode options = super.createParserUIInitializationData(job, fileRecords, format); try { if (fileRecords.size() > 0) { - JSONObject firstFileRecord = fileRecords.get(0); + ObjectNode firstFileRecord = fileRecords.get(0); File file = ImportingUtilities.getFile(job, firstFileRecord); InputStream is = new FileInputStream(file); @@ -92,7 +93,7 @@ public class XmlImporter extends TreeImportingParserBase { int tokenType = parser.next(); state.tokenCount++; if (tokenType == XMLStreamConstants.START_ELEMENT) { - JSONObject rootElement = descendElement(parser, state); + ObjectNode rootElement = descendElement(parser, state); if (rootElement != null) { JSONUtilities.safePut(options, "dom", rootElement); break; @@ -114,8 +115,8 @@ public class XmlImporter extends TreeImportingParserBase { return options; } - final static private JSONObject descendElement(XMLStreamReader parser, PreviewParsingState state) { - JSONObject result = new JSONObject(); + final static private ObjectNode descendElement(XMLStreamReader parser, PreviewParsingState state) { + ObjectNode result = ParsingUtilities.mapper.createObjectNode(); { String name = parser.getLocalName(); JSONUtilities.safePut(result, "n", name); @@ -132,12 +133,11 @@ public class XmlImporter extends TreeImportingParserBase { int namespaceCount = parser.getNamespaceCount(); if (namespaceCount > 0) { - JSONArray namespaces = new JSONArray(); - JSONUtilities.safePut(result, "ns", namespaces); + ArrayNode namespaces = result.putArray("ns"); for (int i = 0; i < namespaceCount; i++) { - JSONObject namespace = new JSONObject(); - JSONUtilities.append(namespaces, namespace); + ObjectNode namespace = ParsingUtilities.mapper.createObjectNode(); + namespaces.add(namespace); JSONUtilities.safePut(namespace, "p", parser.getNamespacePrefix(i)); JSONUtilities.safePut(namespace, "uri", parser.getNamespaceURI(i)); } @@ -145,12 +145,11 @@ public class XmlImporter extends TreeImportingParserBase { int attributeCount = parser.getAttributeCount(); if (attributeCount > 0) { - JSONArray attributes = new JSONArray(); - JSONUtilities.safePut(result, "a", attributes); + ArrayNode attributes = result.putArray("a"); for (int i = 0; i < attributeCount; i++) { - JSONObject attribute = new JSONObject(); - JSONUtilities.append(attributes, attribute); + ObjectNode attribute = ParsingUtilities.mapper.createObjectNode(); + attributes.add(attribute); JSONUtilities.safePut(attribute, "n", parser.getAttributeLocalName(i)); JSONUtilities.safePut(attribute, "v", parser.getAttributeValue(i)); String prefix = parser.getAttributePrefix(i); @@ -160,7 +159,7 @@ public class XmlImporter extends TreeImportingParserBase { } } - JSONArray children = new JSONArray(); + ArrayNode children = ParsingUtilities.mapper.createArrayNode(); try { while (parser.hasNext() && state.tokenCount < PREVIEW_PARSING_LIMIT) { int tokenType = parser.next(); @@ -168,16 +167,16 @@ public class XmlImporter extends TreeImportingParserBase { if (tokenType == XMLStreamConstants.END_ELEMENT) { break; } else if (tokenType == XMLStreamConstants.START_ELEMENT) { - JSONObject childElement = descendElement(parser, state); + ObjectNode childElement = descendElement(parser, state); if (childElement != null) { - JSONUtilities.append(children, childElement); + children.add(childElement); } } else if (tokenType == XMLStreamConstants.CHARACTERS || tokenType == XMLStreamConstants.CDATA || tokenType == XMLStreamConstants.SPACE) { - JSONObject childElement = new JSONObject(); + ObjectNode childElement = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(childElement, "t", parser.getText()); - JSONUtilities.append(children, childElement); + children.add(childElement); } else { // ignore everything else } @@ -186,8 +185,8 @@ public class XmlImporter extends TreeImportingParserBase { logger.error("Error generating parser UI initialization data for XML file", e); } - if (children.length() > 0) { - JSONUtilities.safePut(result, "c", children); + if (children.size() > 0) { + result.put("c", children); } return result; } @@ -195,7 +194,7 @@ public class XmlImporter extends TreeImportingParserBase { @Override public void parseOneFile(Project project, ProjectMetadata metadata, ImportingJob job, String fileSource, InputStream inputStream, - ImportColumnGroup rootColumnGroup, int limit, JSONObject options, + ImportColumnGroup rootColumnGroup, int limit, ObjectNode options, List exceptions) { try { diff --git a/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java b/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java index 79b83a14a..27d207034 100644 --- a/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java +++ b/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java @@ -40,8 +40,8 @@ import java.io.Reader; import java.util.List; import org.apache.commons.lang3.NotImplementedException; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importers.ImporterUtilities; import com.google.refine.importers.ImporterUtilities.MultiFileReadingProgress; import com.google.refine.importers.ImportingParserBase; @@ -62,9 +62,9 @@ abstract public class TreeImportingParserBase extends ImportingParserBase { } @Override - public JSONObject createParserUIInitializationData(ImportingJob job, - List fileRecords, String format) { - JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); + public ObjectNode createParserUIInitializationData(ImportingJob job, + List fileRecords, String format) { + ObjectNode options = super.createParserUIInitializationData(job, fileRecords, format); JSONUtilities.safePut(options, "trimStrings", false); JSONUtilities.safePut(options, "guessCellValueTypes", false); @@ -75,13 +75,13 @@ abstract public class TreeImportingParserBase extends ImportingParserBase { @Override public void parse(Project project, ProjectMetadata metadata, - ImportingJob job, List fileRecords, String format, - int limit, JSONObject options, List exceptions) { + ImportingJob job, List fileRecords, String format, + int limit, ObjectNode options, List exceptions) { MultiFileReadingProgress progress = ImporterUtilities.createMultiFileReadingProgress(job, fileRecords); ImportColumnGroup rootColumnGroup = new ImportColumnGroup(); - for (JSONObject fileRecord : fileRecords) { + for (ObjectNode fileRecord : fileRecords) { try { parseOneFile(project, metadata, job, fileRecord, rootColumnGroup, limit, options, exceptions, progress); } catch (IOException e) { @@ -102,10 +102,10 @@ abstract public class TreeImportingParserBase extends ImportingParserBase { Project project, ProjectMetadata metadata, ImportingJob job, - JSONObject fileRecord, + ObjectNode fileRecord, ImportColumnGroup rootColumnGroup, int limit, - JSONObject options, + ObjectNode options, List exceptions, final MultiFileReadingProgress progress ) throws IOException { @@ -151,7 +151,7 @@ abstract public class TreeImportingParserBase extends ImportingParserBase { Reader reader, ImportColumnGroup rootColumnGroup, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { throw new NotImplementedException("project ID:" + project.id); @@ -171,7 +171,7 @@ abstract public class TreeImportingParserBase extends ImportingParserBase { InputStream inputStream, ImportColumnGroup rootColumnGroup, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { // throw new NotImplementedException(); @@ -190,7 +190,7 @@ abstract public class TreeImportingParserBase extends ImportingParserBase { TreeReader treeParser, ImportColumnGroup rootColumnGroup, int limit, - JSONObject options, + ObjectNode options, List exceptions ) { String[] recordPath = JSONUtilities.getStringArray(options, "recordPath"); diff --git a/main/src/com/google/refine/importing/DefaultImportingController.java b/main/src/com/google/refine/importing/DefaultImportingController.java index 65fb583ad..96c063735 100644 --- a/main/src/com/google/refine/importing/DefaultImportingController.java +++ b/main/src/com/google/refine/importing/DefaultImportingController.java @@ -45,13 +45,12 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.json.JSONArray; import org.json.JSONException; -import org.json.JSONObject; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonGenerator; - +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.RefineServlet; import com.google.refine.commands.HttpUtilities; import com.google.refine.importing.ImportingManager.Format; @@ -112,8 +111,8 @@ public class DefaultImportingController implements ImportingController { job.updating = true; try { - JSONObject config = job.getOrCreateDefaultConfig(); - if (!("new".equals(config.getString("state")))) { + ObjectNode config = job.getOrCreateDefaultConfig(); + if (!("new".equals(JSONUtilities.getString(config, "state", null)))) { HttpUtilities.respond(response, "error", "Job already started; cannot load more data"); return; } @@ -140,13 +139,13 @@ public class DefaultImportingController implements ImportingController { job.updating = true; try { - JSONObject config = job.getOrCreateDefaultConfig(); - if (!("ready".equals(config.getString("state")))) { + ObjectNode config = job.getOrCreateDefaultConfig(); + if (!("ready".equals(JSONUtilities.getString(config, "state", null)))) { HttpUtilities.respond(response, "error", "Job not ready"); return; } - JSONArray fileSelectionArray = ParsingUtilities.evaluateJsonStringToArray( + ArrayNode fileSelectionArray = ParsingUtilities.evaluateJsonStringToArrayNode( request.getParameter("fileSelection")); ImportingUtilities.updateJobWithNewFileSelection(job, fileSelectionArray); @@ -172,14 +171,14 @@ public class DefaultImportingController implements ImportingController { job.updating = true; try { - JSONObject config = job.getOrCreateDefaultConfig(); - if (!("ready".equals(config.getString("state")))) { + ObjectNode config = job.getOrCreateDefaultConfig(); + if (!("ready".equals(JSONUtilities.getString(config, "state", null)))) { HttpUtilities.respond(response, "error", "Job not ready"); return; } String format = request.getParameter("format"); - JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject( + ObjectNode optionObj = ParsingUtilities.evaluateJsonStringToObjectNode( request.getParameter("options")); List exceptions = new LinkedList(); @@ -230,9 +229,9 @@ public class DefaultImportingController implements ImportingController { String format = request.getParameter("format"); Format formatRecord = ImportingManager.formatToRecord.get(format); if (formatRecord != null && formatRecord.parser != null) { - JSONObject options = formatRecord.parser.createParserUIInitializationData( + ObjectNode options = formatRecord.parser.createParserUIInitializationData( job, job.getSelectedFileRecords(), format); - JSONObject result = new JSONObject(); + ObjectNode result = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(result, "status", "ok"); JSONUtilities.safePut(result, "options", options); @@ -255,14 +254,14 @@ public class DefaultImportingController implements ImportingController { job.updating = true; job.touch(); try { - JSONObject config = job.getOrCreateDefaultConfig(); - if (!("ready".equals(config.getString("state")))) { + ObjectNode config = job.getOrCreateDefaultConfig(); + if (!("ready".equals(JSONUtilities.getString(config, "state", null)))) { HttpUtilities.respond(response, "error", "Job not ready"); return; } String format = request.getParameter("format"); - JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject( + ObjectNode optionObj = ParsingUtilities.evaluateJsonStringToObjectNode( request.getParameter("options")); List exceptions = new LinkedList(); @@ -317,13 +316,13 @@ public class DefaultImportingController implements ImportingController { } } - static public JSONArray convertErrorsToJsonArray(List exceptions) { - JSONArray a = new JSONArray(); + static public ArrayNode convertErrorsToJsonArray(List exceptions) { + ArrayNode a = ParsingUtilities.mapper.createArrayNode(); for (Exception e : exceptions) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); - JSONObject o = new JSONObject(); + ObjectNode o = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(o, "message", e.getLocalizedMessage()); JSONUtilities.safePut(o, "stack", sw.toString()); JSONUtilities.append(a, o); diff --git a/main/src/com/google/refine/importing/ImportingJob.java b/main/src/com/google/refine/importing/ImportingJob.java index a19af621a..1da792315 100644 --- a/main/src/com/google/refine/importing/ImportingJob.java +++ b/main/src/com/google/refine/importing/ImportingJob.java @@ -39,24 +39,24 @@ import java.util.ArrayList; import java.util.List; import org.apache.commons.io.FileUtils; -import org.json.JSONArray; -import org.json.JSONObject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonRawValue; - +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.ProjectManager; import com.google.refine.model.Project; import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; public class ImportingJob { final public long id; final public File dir; // Temporary directory where the data about this job is stored - private JSONObject config; + private ObjectNode config; public Project project; public ProjectMetadata metadata; @@ -71,7 +71,7 @@ public class ImportingJob { this.id = id; this.dir = dir; - JSONObject cfg = new JSONObject(); + ObjectNode cfg = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(cfg, "state", "new"); JSONUtilities.safePut(cfg, "hasData", false); this.config = cfg; @@ -88,7 +88,7 @@ public class ImportingJob { } @JsonIgnore - public JSONObject getOrCreateDefaultConfig() { + public ObjectNode getOrCreateDefaultConfig() { return config; } @@ -114,9 +114,9 @@ public class ImportingJob { public void setProgress(int percent, String message) { synchronized (config) { - JSONObject progress = JSONUtilities.getObject(config, "progress"); + ObjectNode progress = JSONUtilities.getObject(config, "progress"); if (progress == null) { - progress = new JSONObject(); + progress = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(config, "progress", progress); } JSONUtilities.safePut(progress, "message", message); @@ -126,13 +126,13 @@ public class ImportingJob { } } - public void setFileSelection(JSONArray fileSelectionArray) { + public void setFileSelection(ArrayNode fileSelectionArray) { synchronized (config) { JSONUtilities.safePut(config, "fileSelection", fileSelectionArray); } } - public void setRankedFormats(JSONArray rankedFormats) { + public void setRankedFormats(ArrayNode rankedFormats) { synchronized (config) { JSONUtilities.safePut(config, "rankedFormats", rankedFormats); } @@ -140,7 +140,7 @@ public class ImportingJob { @JsonIgnore - public JSONObject getRetrievalRecord() { + public ObjectNode getRetrievalRecord() { synchronized(config) { return JSONUtilities.getObject(config,"retrievalRecord"); } @@ -149,25 +149,25 @@ public class ImportingJob { /** * TO check if the file record is a metadata file entry * @param fileRecordObject - * @return JSONObject + * @return ObjectNode */ - public boolean isMetadataFileRecord(JSONObject fileRecordObject) { + public boolean isMetadataFileRecord(ObjectNode fileRecordObject) { return fileRecordObject.has("metaDataFormat"); } @JsonIgnore - public List getSelectedFileRecords() { - List results = new ArrayList(); + public List getSelectedFileRecords() { + List results = new ArrayList(); - JSONObject retrievalRecord = JSONUtilities.getObject(config,"retrievalRecord"); + ObjectNode retrievalRecord = JSONUtilities.getObject(config,"retrievalRecord"); if (retrievalRecord != null) { - JSONArray fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files"); + ArrayNode fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files"); if (fileRecordArray != null) { - JSONArray fileSelectionArray = JSONUtilities.getArray(config,"fileSelection"); + ArrayNode fileSelectionArray = JSONUtilities.getArray(config,"fileSelection"); if (fileSelectionArray != null) { - for (int i = 0; i < fileSelectionArray.length(); i++) { + for (int i = 0; i < fileSelectionArray.size(); i++) { int index = JSONUtilities.getIntElement(fileSelectionArray, i, -1); - if (index >= 0 && index < fileRecordArray.length()) { + if (index >= 0 && index < fileRecordArray.size()) { results.add(JSONUtilities.getObjectElement(fileRecordArray, index)); } } diff --git a/main/src/com/google/refine/importing/ImportingManager.java b/main/src/com/google/refine/importing/ImportingManager.java index ec29caacd..02425cdc7 100644 --- a/main/src/com/google/refine/importing/ImportingManager.java +++ b/main/src/com/google/refine/importing/ImportingManager.java @@ -54,7 +54,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.RefineServlet; import edu.mit.simile.butterfly.ButterflyModule; diff --git a/main/src/com/google/refine/importing/ImportingParser.java b/main/src/com/google/refine/importing/ImportingParser.java index faafe4b3c..8d0b97295 100644 --- a/main/src/com/google/refine/importing/ImportingParser.java +++ b/main/src/com/google/refine/importing/ImportingParser.java @@ -35,8 +35,7 @@ package com.google.refine.importing; import java.util.List; -import org.json.JSONObject; - +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.model.Project; import com.google.refine.model.metadata.ProjectMetadata; @@ -49,11 +48,11 @@ public interface ImportingParser { * @param job * @param fileRecords * @param format - * @return JSONObject options + * @return ObjectNode options */ - public JSONObject createParserUIInitializationData( + public ObjectNode createParserUIInitializationData( ImportingJob job, - List fileRecords, + List fileRecords, String format ); @@ -77,10 +76,10 @@ public interface ImportingParser { Project project, ProjectMetadata metadata, ImportingJob job, - List fileRecords, + List fileRecords, String format, int limit, - JSONObject options, + ObjectNode options, List exceptions ); } diff --git a/main/src/com/google/refine/importing/ImportingUtilities.java b/main/src/com/google/refine/importing/ImportingUtilities.java index 375778019..067d1168b 100644 --- a/main/src/com/google/refine/importing/ImportingUtilities.java +++ b/main/src/com/google/refine/importing/ImportingUtilities.java @@ -83,11 +83,12 @@ import org.apache.http.util.EntityUtils; import org.apache.tools.bzip2.CBZip2InputStream; import org.apache.tools.tar.TarEntry; import org.apache.tools.tar.TarInputStream; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.ProjectManager; import com.google.refine.RefineServlet; import com.google.refine.importing.ImportingManager.Format; @@ -105,6 +106,7 @@ import com.google.refine.model.metadata.PackageExtension; import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.preference.PreferenceStore; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; import io.frictionlessdata.datapackage.Package; import io.frictionlessdata.tableschema.Field; @@ -129,13 +131,13 @@ public class ImportingUtilities { HttpServletResponse response, Properties parameters, final ImportingJob job, - JSONObject config) throws IOException, ServletException { + ObjectNode config) throws IOException, ServletException { - JSONObject retrievalRecord = new JSONObject(); + ObjectNode retrievalRecord = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(config, "retrievalRecord", retrievalRecord); JSONUtilities.safePut(config, "state", "loading-raw-data"); - final JSONObject progress = new JSONObject(); + final ObjectNode progress = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(config, "progress", progress); try { ImportingUtilities.retrieveContentFromPostRequest( @@ -164,13 +166,13 @@ public class ImportingUtilities { return; } - JSONArray fileSelectionIndexes = new JSONArray(); + ArrayNode fileSelectionIndexes = ParsingUtilities.mapper.createArrayNode(); JSONUtilities.safePut(config, "fileSelection", fileSelectionIndexes); String bestFormat = ImportingUtilities.autoSelectFiles(job, retrievalRecord, fileSelectionIndexes); bestFormat = ImportingUtilities.guessBetterFormat(job, bestFormat); - JSONArray rankedFormats = new JSONArray(); + ArrayNode rankedFormats = ParsingUtilities.mapper.createArrayNode(); ImportingUtilities.rankFormats(job, bestFormat, rankedFormats); JSONUtilities.safePut(config, "rankedFormats", rankedFormats); @@ -179,13 +181,13 @@ public class ImportingUtilities { config.remove("progress"); } - static public void updateJobWithNewFileSelection(ImportingJob job, JSONArray fileSelectionArray) { + static public void updateJobWithNewFileSelection(ImportingJob job, ArrayNode fileSelectionArray) { job.setFileSelection(fileSelectionArray); String bestFormat = ImportingUtilities.getCommonFormatForSelectedFiles(job, fileSelectionArray); bestFormat = ImportingUtilities.guessBetterFormat(job, bestFormat); - JSONArray rankedFormats = new JSONArray(); + ArrayNode rankedFormats = ParsingUtilities.mapper.createArrayNode(); ImportingUtilities.rankFormats(job, bestFormat, rankedFormats); job.setRankedFormats(rankedFormats); } @@ -194,10 +196,10 @@ public class ImportingUtilities { HttpServletRequest request, Properties parameters, File rawDataDir, - JSONObject retrievalRecord, + ObjectNode retrievalRecord, final Progress progress ) throws Exception { - JSONArray fileRecords = new JSONArray(); + ArrayNode fileRecords = ParsingUtilities.mapper.createArrayNode(); JSONUtilities.safePut(retrievalRecord, "files", fileRecords); JSONUtilities.safePut(retrievalRecord, "downloadCount", 0); JSONUtilities.safePut(retrievalRecord, "archiveCount", 0); @@ -264,7 +266,7 @@ public class ImportingUtilities { File file = allocateFile(rawDataDir, "clipboard.txt"); - JSONObject fileRecord = new JSONObject(); + ObjectNode fileRecord = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(fileRecord, "origin", "clipboard"); JSONUtilities.safePut(fileRecord, "declaredEncoding", encoding); JSONUtilities.safePut(fileRecord, "declaredMimeType", (String) null); @@ -310,7 +312,7 @@ public class ImportingUtilities { File file = allocateFile(rawDataDir, fileName); - JSONObject fileRecord = new JSONObject(); + ObjectNode fileRecord = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(fileRecord, "origin", "upload"); JSONUtilities.safePut(fileRecord, "declaredEncoding", request.getCharacterEncoding()); JSONUtilities.safePut(fileRecord, "declaredMimeType", fileItem.getContentType()); @@ -344,19 +346,19 @@ public class ImportingUtilities { JSONUtilities.safePut(retrievalRecord, "clipboardCount", clipboardCount); } - private static void processDataPackage(JSONObject retrievalRecord, JSONArray fileRecords) { + private static void processDataPackage(ObjectNode retrievalRecord, ArrayNode fileRecords) { int dataPackageJSONFileIndex = getDataPackageJSONFile(fileRecords); if (dataPackageJSONFileIndex >= 0) { - JSONObject dataPackageJSONFile = (JSONObject) fileRecords.get(dataPackageJSONFileIndex); + ObjectNode dataPackageJSONFile = (ObjectNode) fileRecords.get(dataPackageJSONFileIndex); JSONUtilities.safePut(dataPackageJSONFile, "metaDataFormat", MetadataFormat.DATAPACKAGE_METADATA.name()); JSONUtilities.safePut(retrievalRecord, METADATA_FILE_KEY, dataPackageJSONFile); fileRecords.remove(dataPackageJSONFileIndex); } } - private static int getDataPackageJSONFile(JSONArray fileRecords) { - for (int i = 0; i < fileRecords.length(); i++) { - JSONObject file = fileRecords.getJSONObject(i); + private static int getDataPackageJSONFile(ArrayNode fileRecords) { + for (int i = 0; i < fileRecords.size(); i++) { + ObjectNode file = JSONUtilities.getObjectElement(fileRecords, i); if (file.has("archiveFileName") && file.has("fileName") && file.get("fileName").equals(DataPackageMetadata.DEFAULT_FILE_NAME)) { @@ -366,8 +368,8 @@ public class ImportingUtilities { return -1; } - private static void download(File rawDataDir, JSONObject retrievalRecord, final Progress progress, - JSONArray fileRecords, final SavingUpdate update, String urlString) + private static void download(File rawDataDir, ObjectNode retrievalRecord, final Progress progress, + ArrayNode fileRecords, final SavingUpdate update, String urlString) throws URISyntaxException, IOException, ClientProtocolException, Exception { download(rawDataDir, retrievalRecord, progress, fileRecords, update, urlString, null); } @@ -384,11 +386,11 @@ public class ImportingUtilities { * @throws ClientProtocolException * @throws Exception */ - private static void download(File rawDataDir, JSONObject retrievalRecord, final Progress progress, - JSONArray fileRecords, final SavingUpdate update, String urlString, String metaDataFormat) + private static void download(File rawDataDir, ObjectNode retrievalRecord, final Progress progress, + ArrayNode fileRecords, final SavingUpdate update, String urlString, String metaDataFormat) throws URISyntaxException, IOException, ClientProtocolException, Exception { URL url = new URL(urlString); - JSONObject fileRecord = new JSONObject(); + ObjectNode fileRecord = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(fileRecord, "origin", "download"); JSONUtilities.safePut(fileRecord, "url", urlString); @@ -482,7 +484,7 @@ public class ImportingUtilities { } private static boolean saveStream(InputStream stream, URL url, File rawDataDir, final Progress progress, - final SavingUpdate update, JSONObject fileRecord, JSONArray fileRecords, long length) + final SavingUpdate update, ObjectNode fileRecord, ArrayNode fileRecords, long length) throws IOException, Exception { String localname = url.getPath(); if (localname.isEmpty() || localname.endsWith("/")) { @@ -538,17 +540,17 @@ public class ImportingUtilities { return file; } - static public Reader getFileReader(ImportingJob job, JSONObject fileRecord, String commonEncoding) + static public Reader getFileReader(ImportingJob job, ObjectNode fileRecord, String commonEncoding) throws FileNotFoundException { return getFileReader(getFile(job, JSONUtilities.getString(fileRecord, "location", "")), fileRecord, commonEncoding); } - static public Reader getFileReader(File file, JSONObject fileRecord, String commonEncoding) throws FileNotFoundException { + static public Reader getFileReader(File file, ObjectNode fileRecord, String commonEncoding) throws FileNotFoundException { return getReaderFromStream(new FileInputStream(file), fileRecord, commonEncoding); } - static public Reader getReaderFromStream(InputStream inputStream, JSONObject fileRecord, String commonEncoding) { + static public Reader getReaderFromStream(InputStream inputStream, ObjectNode fileRecord, String commonEncoding) { String encoding = getEncoding(fileRecord); if (encoding == null) { encoding = commonEncoding; @@ -563,7 +565,7 @@ public class ImportingUtilities { return new InputStreamReader(inputStream); } - static public File getFile(ImportingJob job, JSONObject fileRecord) { + static public File getFile(ImportingJob job, ObjectNode fileRecord) { return getFile(job, JSONUtilities.getString(fileRecord, "location", "")); } @@ -571,7 +573,7 @@ public class ImportingUtilities { return new File(job.getRawDataDir(), location); } - static public String getFileSource(JSONObject fileRecord) { + static public String getFileSource(ObjectNode fileRecord) { return JSONUtilities.getString( fileRecord, "url", @@ -608,7 +610,7 @@ public class ImportingUtilities { } static public boolean postProcessRetrievedFile( - File rawDataDir, File file, JSONObject fileRecord, JSONArray fileRecords, final Progress progress) { + File rawDataDir, File file, ObjectNode fileRecord, ArrayNode fileRecords, final Progress progress) { String mimeType = JSONUtilities.getString(fileRecord, "declaredMimeType", null); String contentEncoding = JSONUtilities.getString(fileRecord, "declaredEncoding", null); @@ -654,7 +656,7 @@ public class ImportingUtilities { return false; } - static public void postProcessSingleRetrievedFile(File file, JSONObject fileRecord) { + static public void postProcessSingleRetrievedFile(File file, ObjectNode fileRecord) { if (!fileRecord.has("format")) { JSONUtilities.safePut(fileRecord, "format", ImportingManager.getFormat( @@ -693,8 +695,8 @@ public class ImportingUtilities { static public boolean explodeArchive( File rawDataDir, InputStream archiveIS, - JSONObject archiveFileRecord, - JSONArray fileRecords, + ObjectNode archiveFileRecord, + ArrayNode fileRecords, final Progress progress ) { if (archiveIS instanceof TarInputStream) { @@ -708,7 +710,7 @@ public class ImportingUtilities { progress.setProgress("Extracting " + fileName2, -1); - JSONObject fileRecord2 = new JSONObject(); + ObjectNode fileRecord2 = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(fileRecord2, "origin", JSONUtilities.getString(archiveFileRecord, "origin", null)); JSONUtilities.safePut(fileRecord2, "declaredEncoding", (String) null); JSONUtilities.safePut(fileRecord2, "declaredMimeType", (String) null); @@ -738,7 +740,7 @@ public class ImportingUtilities { progress.setProgress("Extracting " + fileName2, -1); - JSONObject fileRecord2 = new JSONObject(); + ObjectNode fileRecord2 = ParsingUtilities.mapper.createObjectNode(); JSONUtilities.safePut(fileRecord2, "origin", JSONUtilities.getString(archiveFileRecord, "origin", null)); JSONUtilities.safePut(fileRecord2, "declaredEncoding", (String) null); JSONUtilities.safePut(fileRecord2, "declaredMimeType", (String) null); @@ -792,7 +794,7 @@ public class ImportingUtilities { static public File uncompressFile( File rawDataDir, InputStream uncompressedIS, - JSONObject fileRecord, + ObjectNode fileRecord, final Progress progress ) throws IOException { String fileName = JSONUtilities.getString(fileRecord, "location", "unknown"); @@ -823,10 +825,10 @@ public class ImportingUtilities { return NumberFormat.getIntegerInstance().format(bytes); } - static public String getEncoding(JSONObject fileRecord) { - String encoding = JSONUtilities.getString(fileRecord, "encoding", null); + static public String getEncoding(ObjectNode firstFileRecord) { + String encoding = JSONUtilities.getString(firstFileRecord, "encoding", null); if (encoding == null || encoding.isEmpty()) { - encoding = JSONUtilities.getString(fileRecord, "declaredEncoding", null); + encoding = JSONUtilities.getString(firstFileRecord, "declaredEncoding", null); } return encoding; } @@ -840,14 +842,14 @@ public class ImportingUtilities { * @param fileSelectionIndexes JSON array of selected file indices matching best format * @return best (highest frequency) format */ - static public String autoSelectFiles(ImportingJob job, JSONObject retrievalRecord, JSONArray fileSelectionIndexes) { + static public String autoSelectFiles(ImportingJob job, ObjectNode retrievalRecord, ArrayNode fileSelectionIndexes) { final Map formatToCount = new HashMap(); List formats = new ArrayList(); - JSONArray fileRecords = JSONUtilities.getArray(retrievalRecord, "files"); - int count = fileRecords.length(); + ArrayNode fileRecords = JSONUtilities.getArray(retrievalRecord, "files"); + int count = fileRecords.size(); for (int i = 0; i < count; i++) { - JSONObject fileRecord = JSONUtilities.getObjectElement(fileRecords, i); + ObjectNode fileRecord = JSONUtilities.getObjectElement(fileRecords, i); String format = JSONUtilities.getString(fileRecord, "format", null); if (format != null) { if (formatToCount.containsKey(format)) { @@ -875,7 +877,7 @@ public class ImportingUtilities { } else { // Otherwise, select files matching the best format for (int i = 0; i < count; i++) { - JSONObject fileRecord = JSONUtilities.getObjectElement(fileRecords, i); + ObjectNode fileRecord = JSONUtilities.getObjectElement(fileRecords, i); String format = JSONUtilities.getString(fileRecord, "format", null); if (format != null && format.equals(bestFormat)) { JSONUtilities.append(fileSelectionIndexes, i); @@ -884,7 +886,7 @@ public class ImportingUtilities { // If nothing matches the best format but we have some files, // then select them all - if (fileSelectionIndexes.length() == 0 && count > 0) { + if (fileSelectionIndexes.size() == 0 && count > 0) { for (int i = 0; i < count; i++) { JSONUtilities.append(fileSelectionIndexes, i); } @@ -893,18 +895,18 @@ public class ImportingUtilities { return bestFormat; } - static public String getCommonFormatForSelectedFiles(ImportingJob job, JSONArray fileSelectionIndexes) { - JSONObject retrievalRecord = job.getRetrievalRecord(); + static public String getCommonFormatForSelectedFiles(ImportingJob job, ArrayNode fileSelectionIndexes) { + ObjectNode retrievalRecord = job.getRetrievalRecord(); final Map formatToCount = new HashMap(); List formats = new ArrayList(); - JSONArray fileRecords = JSONUtilities.getArray(retrievalRecord, "files"); - int count = fileSelectionIndexes.length(); + ArrayNode fileRecords = JSONUtilities.getArray(retrievalRecord, "files"); + int count = fileSelectionIndexes.size(); for (int i = 0; i < count; i++) { int index = JSONUtilities.getIntElement(fileSelectionIndexes, i, -1); - if (index >= 0 && index < fileRecords.length()) { - JSONObject fileRecord = JSONUtilities.getObjectElement(fileRecords, index); + if (index >= 0 && index < fileRecords.size()) { + ObjectNode fileRecord = JSONUtilities.getObjectElement(fileRecords, index); String format = JSONUtilities.getString(fileRecord, "format", null); if (format != null) { if (formatToCount.containsKey(format)) { @@ -927,18 +929,18 @@ public class ImportingUtilities { } static String guessBetterFormat(ImportingJob job, String bestFormat) { - JSONObject retrievalRecord = job.getRetrievalRecord(); + ObjectNode retrievalRecord = job.getRetrievalRecord(); return retrievalRecord != null ? guessBetterFormat(job, retrievalRecord, bestFormat) : bestFormat; } - static String guessBetterFormat(ImportingJob job, JSONObject retrievalRecord, String bestFormat) { - JSONArray fileRecords = JSONUtilities.getArray(retrievalRecord, "files"); + static String guessBetterFormat(ImportingJob job, ObjectNode retrievalRecord, String bestFormat) { + ArrayNode fileRecords = JSONUtilities.getArray(retrievalRecord, "files"); return fileRecords != null ? guessBetterFormat(job, fileRecords, bestFormat) : bestFormat; } - static String guessBetterFormat(ImportingJob job, JSONArray fileRecords, String bestFormat) { - if (bestFormat != null && fileRecords != null && fileRecords.length() > 0) { - JSONObject firstFileRecord = JSONUtilities.getObjectElement(fileRecords, 0); + static String guessBetterFormat(ImportingJob job, ArrayNode fileRecords, String bestFormat) { + if (bestFormat != null && fileRecords != null && fileRecords.size() > 0) { + ObjectNode firstFileRecord = JSONUtilities.getObjectElement(fileRecords, 0); String encoding = getEncoding(firstFileRecord); String location = JSONUtilities.getString(firstFileRecord, "location", null); @@ -969,7 +971,7 @@ public class ImportingUtilities { return bestFormat; } - static void rankFormats(ImportingJob job, final String bestFormat, JSONArray rankedFormats) { + static void rankFormats(ImportingJob job, final String bestFormat, ArrayNode rankedFormats) { final Map formatToSegments = new HashMap(); boolean download = bestFormat == null ? true : @@ -1023,12 +1025,12 @@ public class ImportingUtilities { } for (String format : formats) { - JSONUtilities.append(rankedFormats, format); + rankedFormats.add(format); } } - static public void previewParse(ImportingJob job, String format, JSONObject optionObj, List exceptions) { + static public void previewParse(ImportingJob job, String format, ObjectNode optionObj, List exceptions) { Format record = ImportingManager.formatToRecord.get(format); if (record == null || record.parser == null) { // TODO: what to do? @@ -1054,7 +1056,7 @@ public class ImportingUtilities { static public long createProject( final ImportingJob job, final String format, - final JSONObject optionObj, + final ObjectNode optionObj, final List exceptions, boolean synchronous) { final Format record = ImportingManager.formatToRecord.get(format); @@ -1084,7 +1086,7 @@ public class ImportingUtilities { static private void createProjectSynchronously( final ImportingJob job, final String format, - final JSONObject optionObj, + final ObjectNode optionObj, final List exceptions, final Format record, final Project project @@ -1105,15 +1107,15 @@ public class ImportingUtilities { if (exceptions.size() == 0) { project.update(); // update all internal models, indexes, caches, etc. - boolean hasMetadataFileRecord = ((JSONObject)job.getRetrievalRecord()).has(METADATA_FILE_KEY); + boolean hasMetadataFileRecord = ((ObjectNode)job.getRetrievalRecord()).has(METADATA_FILE_KEY); if (hasMetadataFileRecord) { - JSONObject metadataFileRecord = (JSONObject) job.getRetrievalRecord().get(METADATA_FILE_KEY); + ObjectNode metadataFileRecord = (ObjectNode) job.getRetrievalRecord().get(METADATA_FILE_KEY); - String metadataFormat = (String)metadataFileRecord.get("metaDataFormat"); + String metadataFormat = JSONUtilities.getString(metadataFileRecord, "metaDataFormat", null); IMetadata metadata = MetadataFactory.buildMetadata(MetadataFormat.valueOf(metadataFormat)); - String relativePath = metadataFileRecord.getString("location"); + String relativePath = JSONUtilities.getString(metadataFileRecord, "location", null); File metadataFile = new File(job.getRawDataDir(), relativePath); metadata.loadFromFile(metadataFile); @@ -1159,10 +1161,10 @@ public class ImportingUtilities { } try { - JSONObject fieldsJSON = TypeInferrer.getInstance().infer(listCells, + ObjectNode fieldsJSON = JSONUtilities.jsonObjectToObjectNode(TypeInferrer.getInstance().infer(listCells, project.columnModel.getColumnNames().toArray(new String[0]), - 100); - populateColumnTypes(project.columnModel, fieldsJSON.getJSONArray(Schema.JSON_KEY_FIELDS)); + 100)); + populateColumnTypes(project.columnModel, JSONUtilities.getArray(fieldsJSON, Schema.JSON_KEY_FIELDS)); } catch (TypeInferringException e) { logger.error("infer column type exception.", ExceptionUtils.getStackTrace(e)); } @@ -1171,7 +1173,7 @@ public class ImportingUtilities { private static void populateDataPackageMetadata(Project project, ProjectMetadata pmd, DataPackageMetadata metadata) { // project metadata - JSONObject pkg = metadata.getPackage().getJson(); + ObjectNode pkg = JSONUtilities.jsonObjectToObjectNode(metadata.getPackage().getJson()); pmd.setName(getDataPackageProperty(pkg, Package.JSON_KEY_NAME)); pmd.setDescription(getDataPackageProperty(pkg, PackageExtension.JSON_KEY_DESCRIPTION)); @@ -1182,18 +1184,18 @@ public class ImportingUtilities { pmd.setVersion(getDataPackageProperty(pkg, PackageExtension.JSON_KEY_VERSION)); if (pkg.has(PackageExtension.JSON_KEY_KEYWORKS)) { - String[] tags = pkg.getJSONArray(PackageExtension.JSON_KEY_KEYWORKS).toList().toArray(new String[0]); + String[] tags = JSONUtilities.getStringArray(pkg, PackageExtension.JSON_KEY_KEYWORKS); pmd.appendTags(tags); } // column model - JSONObject schema = metadata.getPackage().getResources().get(0).getSchema(); + ObjectNode schema = JSONUtilities.jsonObjectToObjectNode(metadata.getPackage().getResources().get(0).getSchema()); if (schema != null) { - populateColumnTypes(project.columnModel, schema.getJSONArray(Schema.JSON_KEY_FIELDS)); + populateColumnTypes(project.columnModel, JSONUtilities.getArray(schema, Schema.JSON_KEY_FIELDS)); } } - private static String getDataPackageProperty(JSONObject pkg, String key) { + private static String getDataPackageProperty(ObjectNode pkg, String key) { return JSONUtilities.getString(pkg, key, StringUtils.EMPTY); } /** @@ -1201,12 +1203,12 @@ public class ImportingUtilities { * @param columnModel * @param fieldsJSON */ - private static void populateColumnTypes(ColumnModel columnModel, JSONArray fieldsJSON) { + private static void populateColumnTypes(ColumnModel columnModel, ArrayNode fieldsJSON) { int cellIndex = 0; - Iterator iter = fieldsJSON.iterator(); + Iterator iter = fieldsJSON.iterator(); while(iter.hasNext()){ - JSONObject fieldJsonObj = (JSONObject)iter.next(); - Field field = new Field(fieldJsonObj); + ObjectNode fieldJsonObj = (ObjectNode)iter.next(); + Field field = new Field(JSONUtilities.objectNodeToJsonNode(fieldJsonObj)); Column column = columnModel.getColumnByCellIndex(cellIndex); column.setType(field.getType()); @@ -1224,7 +1226,7 @@ public class ImportingUtilities { * @param optionObj * @return */ - static public ProjectMetadata createProjectMetadata(JSONObject optionObj) { + static public ProjectMetadata createProjectMetadata(ObjectNode optionObj) { ProjectMetadata pm = new ProjectMetadata(); PreferenceStore ps = ProjectManager.singleton.getPreferenceStore(); diff --git a/main/src/com/google/refine/io/FileProjectManager.java b/main/src/com/google/refine/io/FileProjectManager.java index fc93e0ca6..46875e02a 100644 --- a/main/src/com/google/refine/io/FileProjectManager.java +++ b/main/src/com/google/refine/io/FileProjectManager.java @@ -59,7 +59,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.ProjectManager; import com.google.refine.history.HistoryEntryManager; import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/model/AbstractOperation.java b/main/src/com/google/refine/model/AbstractOperation.java index 6ab8135c3..cec470d81 100644 --- a/main/src/com/google/refine/model/AbstractOperation.java +++ b/main/src/com/google/refine/model/AbstractOperation.java @@ -38,7 +38,6 @@ import java.util.Properties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.annotation.JsonTypeIdResolver; - import com.google.refine.history.HistoryEntry; import com.google.refine.operations.OperationRegistry; import com.google.refine.operations.OperationResolver; diff --git a/main/src/com/google/refine/model/Cell.java b/main/src/com/google/refine/model/Cell.java index 61394c809..d6767ff50 100644 --- a/main/src/com/google/refine/model/Cell.java +++ b/main/src/com/google/refine/model/Cell.java @@ -49,7 +49,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.InjectableValues; - import com.google.refine.expr.EvalError; import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.HasFields; diff --git a/main/src/com/google/refine/model/Column.java b/main/src/com/google/refine/model/Column.java index 29f0d52e0..f92d64598 100644 --- a/main/src/com/google/refine/model/Column.java +++ b/main/src/com/google/refine/model/Column.java @@ -46,7 +46,6 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; - import com.google.refine.InterProjectModel; import com.google.refine.model.recon.ReconConfig; import com.google.refine.util.ParsingUtilities; diff --git a/main/src/com/google/refine/model/ColumnGroup.java b/main/src/com/google/refine/model/ColumnGroup.java index c6059d4e9..45e18b152 100644 --- a/main/src/com/google/refine/model/ColumnGroup.java +++ b/main/src/com/google/refine/model/ColumnGroup.java @@ -43,7 +43,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonView; - import com.google.refine.util.JsonViews; import com.google.refine.util.ParsingUtilities; diff --git a/main/src/com/google/refine/model/Recon.java b/main/src/com/google/refine/model/Recon.java index caa1373e6..bf549e06b 100644 --- a/main/src/com/google/refine/model/Recon.java +++ b/main/src/com/google/refine/model/Recon.java @@ -46,7 +46,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonView; - import com.google.refine.expr.HasFields; import com.google.refine.util.JsonViews; import com.google.refine.util.ParsingUtilities; diff --git a/main/src/com/google/refine/model/ReconCandidate.java b/main/src/com/google/refine/model/ReconCandidate.java index 6e2a1e3da..65d8c698a 100644 --- a/main/src/com/google/refine/model/ReconCandidate.java +++ b/main/src/com/google/refine/model/ReconCandidate.java @@ -41,7 +41,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; - import com.google.refine.expr.HasFields; import com.google.refine.util.ParsingUtilities; diff --git a/main/src/com/google/refine/model/ReconStats.java b/main/src/com/google/refine/model/ReconStats.java index 89650c791..00bcbcd1a 100644 --- a/main/src/com/google/refine/model/ReconStats.java +++ b/main/src/com/google/refine/model/ReconStats.java @@ -38,7 +38,6 @@ import java.io.Writer; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.expr.ExpressionUtils; import com.google.refine.model.Recon.Judgment; import com.google.refine.util.ParsingUtilities; diff --git a/main/src/com/google/refine/model/ReconType.java b/main/src/com/google/refine/model/ReconType.java index 65a6d34e9..2634bd6f0 100644 --- a/main/src/com/google/refine/model/ReconType.java +++ b/main/src/com/google/refine/model/ReconType.java @@ -37,7 +37,6 @@ import java.io.IOException; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.util.ParsingUtilities; diff --git a/main/src/com/google/refine/model/RecordModel.java b/main/src/com/google/refine/model/RecordModel.java index fe9ccae13..843922241 100644 --- a/main/src/com/google/refine/model/RecordModel.java +++ b/main/src/com/google/refine/model/RecordModel.java @@ -44,7 +44,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.expr.ExpressionUtils; public class RecordModel { diff --git a/main/src/com/google/refine/model/Row.java b/main/src/com/google/refine/model/Row.java index 3e161f3f7..9e0339174 100644 --- a/main/src/com/google/refine/model/Row.java +++ b/main/src/com/google/refine/model/Row.java @@ -43,7 +43,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.InjectableValues; - import com.google.refine.expr.CellTuple; import com.google.refine.expr.HasFields; import com.google.refine.util.ParsingUtilities; diff --git a/main/src/com/google/refine/model/changes/MassChange.java b/main/src/com/google/refine/model/changes/MassChange.java index efb6feb81..1f9f9dab1 100644 --- a/main/src/com/google/refine/model/changes/MassChange.java +++ b/main/src/com/google/refine/model/changes/MassChange.java @@ -41,7 +41,6 @@ import java.util.List; import java.util.Properties; import com.google.common.collect.Lists; - import com.google.refine.history.Change; import com.google.refine.history.History; import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/model/metadata/ProjectMetadata.java b/main/src/com/google/refine/model/metadata/ProjectMetadata.java index 6ccd26fee..23dc2f253 100644 --- a/main/src/com/google/refine/model/metadata/ProjectMetadata.java +++ b/main/src/com/google/refine/model/metadata/ProjectMetadata.java @@ -63,7 +63,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonRawValue; import com.fasterxml.jackson.annotation.JsonUnwrapped; import com.fasterxml.jackson.annotation.JsonView; - +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.ProjectManager; import com.google.refine.preference.PreferenceStore; import com.google.refine.preference.TopList; @@ -362,8 +362,8 @@ public class ProjectMetadata extends AbstractMetadata { updateModified(); } - public void appendImportOptionMetadata(JSONObject obj) { - _importOptionMetadata.put(obj); + public void appendImportOptionMetadata(ObjectNode options) { + _importOptionMetadata.put(options); updateModified(); } diff --git a/main/src/com/google/refine/model/recon/ReconConfig.java b/main/src/com/google/refine/model/recon/ReconConfig.java index aa3459a89..bf2ebc063 100644 --- a/main/src/com/google/refine/model/recon/ReconConfig.java +++ b/main/src/com/google/refine/model/recon/ReconConfig.java @@ -46,7 +46,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.annotation.JsonTypeIdResolver; - import com.google.refine.model.Cell; import com.google.refine.model.Project; import com.google.refine.model.Recon; diff --git a/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java b/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java index 13a630e5c..4f0196110 100644 --- a/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java +++ b/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java @@ -60,7 +60,6 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonView; import com.fasterxml.jackson.databind.ObjectMapper; - import com.google.refine.expr.functions.ToDate; import com.google.refine.model.ReconCandidate; import com.google.refine.model.ReconType; diff --git a/main/src/com/google/refine/model/recon/StandardReconConfig.java b/main/src/com/google/refine/model/recon/StandardReconConfig.java index a8b568feb..78cda9334 100644 --- a/main/src/com/google/refine/model/recon/StandardReconConfig.java +++ b/main/src/com/google/refine/model/recon/StandardReconConfig.java @@ -60,7 +60,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; - import com.google.refine.expr.ExpressionUtils; import com.google.refine.model.Cell; import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/operations/EngineDependentMassCellOperation.java b/main/src/com/google/refine/operations/EngineDependentMassCellOperation.java index daf02378e..c829e7ccd 100644 --- a/main/src/com/google/refine/operations/EngineDependentMassCellOperation.java +++ b/main/src/com/google/refine/operations/EngineDependentMassCellOperation.java @@ -38,7 +38,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; diff --git a/main/src/com/google/refine/operations/EngineDependentOperation.java b/main/src/com/google/refine/operations/EngineDependentOperation.java index 3cf969164..b5cbca25c 100644 --- a/main/src/com/google/refine/operations/EngineDependentOperation.java +++ b/main/src/com/google/refine/operations/EngineDependentOperation.java @@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.operations; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.model.AbstractOperation; diff --git a/main/src/com/google/refine/operations/OperationRegistry.java b/main/src/com/google/refine/operations/OperationRegistry.java index 3a352917e..d2f6393a6 100644 --- a/main/src/com/google/refine/operations/OperationRegistry.java +++ b/main/src/com/google/refine/operations/OperationRegistry.java @@ -33,21 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.operations; -import java.io.IOException; -import java.lang.reflect.Method; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import org.json.JSONObject; - -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; - import com.google.refine.model.AbstractOperation; -import com.google.refine.model.Project; -import com.google.refine.util.ParsingUtilities; import edu.mit.simile.butterfly.ButterflyModule; diff --git a/main/src/com/google/refine/operations/OperationResolver.java b/main/src/com/google/refine/operations/OperationResolver.java index 5f8085e48..343e910b8 100644 --- a/main/src/com/google/refine/operations/OperationResolver.java +++ b/main/src/com/google/refine/operations/OperationResolver.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.DatabindContext; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.jsontype.impl.TypeIdResolverBase; import com.fasterxml.jackson.databind.type.TypeFactory; - import com.google.refine.model.AbstractOperation; public class OperationResolver extends TypeIdResolverBase { diff --git a/main/src/com/google/refine/operations/cell/BlankDownOperation.java b/main/src/com/google/refine/operations/cell/BlankDownOperation.java index f022f28af..9d4a7d96b 100644 --- a/main/src/com/google/refine/operations/cell/BlankDownOperation.java +++ b/main/src/com/google/refine/operations/cell/BlankDownOperation.java @@ -37,7 +37,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine.Mode; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; diff --git a/main/src/com/google/refine/operations/cell/FillDownOperation.java b/main/src/com/google/refine/operations/cell/FillDownOperation.java index b3a47ca60..1de411e10 100644 --- a/main/src/com/google/refine/operations/cell/FillDownOperation.java +++ b/main/src/com/google/refine/operations/cell/FillDownOperation.java @@ -37,7 +37,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine.Mode; import com.google.refine.browsing.EngineConfig; diff --git a/main/src/com/google/refine/operations/cell/KeyValueColumnizeOperation.java b/main/src/com/google/refine/operations/cell/KeyValueColumnizeOperation.java index bc04f38d9..96987a5bc 100644 --- a/main/src/com/google/refine/operations/cell/KeyValueColumnizeOperation.java +++ b/main/src/com/google/refine/operations/cell/KeyValueColumnizeOperation.java @@ -40,7 +40,6 @@ import java.util.Map; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.expr.ExpressionUtils; import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; diff --git a/main/src/com/google/refine/operations/cell/MassEditOperation.java b/main/src/com/google/refine/operations/cell/MassEditOperation.java index 593912b42..6d8a473e5 100644 --- a/main/src/com/google/refine/operations/cell/MassEditOperation.java +++ b/main/src/com/google/refine/operations/cell/MassEditOperation.java @@ -42,7 +42,6 @@ import java.util.Properties; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.expr.Evaluable; diff --git a/main/src/com/google/refine/operations/cell/MultiValuedCellJoinOperation.java b/main/src/com/google/refine/operations/cell/MultiValuedCellJoinOperation.java index 88ef3aeff..5bf16bc61 100644 --- a/main/src/com/google/refine/operations/cell/MultiValuedCellJoinOperation.java +++ b/main/src/com/google/refine/operations/cell/MultiValuedCellJoinOperation.java @@ -38,7 +38,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.expr.ExpressionUtils; import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; diff --git a/main/src/com/google/refine/operations/cell/MultiValuedCellSplitOperation.java b/main/src/com/google/refine/operations/cell/MultiValuedCellSplitOperation.java index fbb3a9006..658f5096d 100644 --- a/main/src/com/google/refine/operations/cell/MultiValuedCellSplitOperation.java +++ b/main/src/com/google/refine/operations/cell/MultiValuedCellSplitOperation.java @@ -43,7 +43,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Cell; diff --git a/main/src/com/google/refine/operations/cell/TextTransformOperation.java b/main/src/com/google/refine/operations/cell/TextTransformOperation.java index bdedef2cc..1a043f2a7 100644 --- a/main/src/com/google/refine/operations/cell/TextTransformOperation.java +++ b/main/src/com/google/refine/operations/cell/TextTransformOperation.java @@ -39,7 +39,6 @@ import java.util.Properties; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.expr.Evaluable; diff --git a/main/src/com/google/refine/operations/cell/TransposeColumnsIntoRowsOperation.java b/main/src/com/google/refine/operations/cell/TransposeColumnsIntoRowsOperation.java index c9e416d76..0697dc9da 100644 --- a/main/src/com/google/refine/operations/cell/TransposeColumnsIntoRowsOperation.java +++ b/main/src/com/google/refine/operations/cell/TransposeColumnsIntoRowsOperation.java @@ -41,7 +41,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Cell; diff --git a/main/src/com/google/refine/operations/cell/TransposeRowsIntoColumnsOperation.java b/main/src/com/google/refine/operations/cell/TransposeRowsIntoColumnsOperation.java index bcfa612a1..6feaa6197 100644 --- a/main/src/com/google/refine/operations/cell/TransposeRowsIntoColumnsOperation.java +++ b/main/src/com/google/refine/operations/cell/TransposeRowsIntoColumnsOperation.java @@ -38,7 +38,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Cell; diff --git a/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java b/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java index 4e920f435..4251235e5 100644 --- a/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java @@ -49,11 +49,9 @@ import org.json.JSONException; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; diff --git a/main/src/com/google/refine/operations/column/ColumnAdditionOperation.java b/main/src/com/google/refine/operations/column/ColumnAdditionOperation.java index 6e58835e5..377fa4ad8 100644 --- a/main/src/com/google/refine/operations/column/ColumnAdditionOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnAdditionOperation.java @@ -40,7 +40,6 @@ import java.util.Properties; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; diff --git a/main/src/com/google/refine/operations/column/ColumnMoveOperation.java b/main/src/com/google/refine/operations/column/ColumnMoveOperation.java index fd0842f53..bc19ffc06 100644 --- a/main/src/com/google/refine/operations/column/ColumnMoveOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnMoveOperation.java @@ -35,7 +35,6 @@ package com.google.refine.operations.column; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.Change; import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; diff --git a/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java b/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java index 7449345bc..0d38ebfd1 100644 --- a/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java @@ -35,7 +35,6 @@ package com.google.refine.operations.column; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.Change; import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; diff --git a/main/src/com/google/refine/operations/column/ColumnRenameOperation.java b/main/src/com/google/refine/operations/column/ColumnRenameOperation.java index 6359d6324..1b843fe6f 100644 --- a/main/src/com/google/refine/operations/column/ColumnRenameOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnRenameOperation.java @@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.operations.column; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.Change; import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; diff --git a/main/src/com/google/refine/operations/column/ColumnReorderOperation.java b/main/src/com/google/refine/operations/column/ColumnReorderOperation.java index 032a8f7b9..d45787f87 100644 --- a/main/src/com/google/refine/operations/column/ColumnReorderOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnReorderOperation.java @@ -37,7 +37,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/operations/column/ColumnSplitOperation.java b/main/src/com/google/refine/operations/column/ColumnSplitOperation.java index 6f55b7a47..77852088f 100644 --- a/main/src/com/google/refine/operations/column/ColumnSplitOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnSplitOperation.java @@ -44,7 +44,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; diff --git a/main/src/com/google/refine/operations/recon/ExtendDataOperation.java b/main/src/com/google/refine/operations/recon/ExtendDataOperation.java index adcd26861..670adda23 100644 --- a/main/src/com/google/refine/operations/recon/ExtendDataOperation.java +++ b/main/src/com/google/refine/operations/recon/ExtendDataOperation.java @@ -45,7 +45,6 @@ import org.json.JSONException; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; diff --git a/main/src/com/google/refine/operations/recon/ReconClearSimilarCellsOperation.java b/main/src/com/google/refine/operations/recon/ReconClearSimilarCellsOperation.java index 673e5e49d..3cccca2c2 100644 --- a/main/src/com/google/refine/operations/recon/ReconClearSimilarCellsOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconClearSimilarCellsOperation.java @@ -37,7 +37,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.history.Change; diff --git a/main/src/com/google/refine/operations/recon/ReconCopyAcrossColumnsOperation.java b/main/src/com/google/refine/operations/recon/ReconCopyAcrossColumnsOperation.java index 6b9f89aa9..3fb901a64 100644 --- a/main/src/com/google/refine/operations/recon/ReconCopyAcrossColumnsOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconCopyAcrossColumnsOperation.java @@ -44,7 +44,6 @@ import org.apache.commons.lang3.StringUtils; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; diff --git a/main/src/com/google/refine/operations/recon/ReconDiscardJudgmentsOperation.java b/main/src/com/google/refine/operations/recon/ReconDiscardJudgmentsOperation.java index c5e538e2e..c9ba826dc 100644 --- a/main/src/com/google/refine/operations/recon/ReconDiscardJudgmentsOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconDiscardJudgmentsOperation.java @@ -39,7 +39,6 @@ import java.util.Map; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.history.Change; diff --git a/main/src/com/google/refine/operations/recon/ReconJudgeSimilarCellsOperation.java b/main/src/com/google/refine/operations/recon/ReconJudgeSimilarCellsOperation.java index 5068334b4..bfa016462 100644 --- a/main/src/com/google/refine/operations/recon/ReconJudgeSimilarCellsOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconJudgeSimilarCellsOperation.java @@ -41,7 +41,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.expr.ExpressionUtils; diff --git a/main/src/com/google/refine/operations/recon/ReconMarkNewTopicsOperation.java b/main/src/com/google/refine/operations/recon/ReconMarkNewTopicsOperation.java index ac0eb1989..b6d7b42c3 100644 --- a/main/src/com/google/refine/operations/recon/ReconMarkNewTopicsOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconMarkNewTopicsOperation.java @@ -39,7 +39,6 @@ import java.util.Map; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.history.Change; diff --git a/main/src/com/google/refine/operations/recon/ReconMatchBestCandidatesOperation.java b/main/src/com/google/refine/operations/recon/ReconMatchBestCandidatesOperation.java index 17dd06184..b5f782d10 100644 --- a/main/src/com/google/refine/operations/recon/ReconMatchBestCandidatesOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconMatchBestCandidatesOperation.java @@ -39,7 +39,6 @@ import java.util.Map; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.history.Change; diff --git a/main/src/com/google/refine/operations/recon/ReconMatchSpecificTopicOperation.java b/main/src/com/google/refine/operations/recon/ReconMatchSpecificTopicOperation.java index 918a71d4d..83608fe92 100644 --- a/main/src/com/google/refine/operations/recon/ReconMatchSpecificTopicOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconMatchSpecificTopicOperation.java @@ -40,7 +40,6 @@ import java.util.Map; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.history.Change; diff --git a/main/src/com/google/refine/operations/recon/ReconOperation.java b/main/src/com/google/refine/operations/recon/ReconOperation.java index b9218f145..fbc0a3f3c 100644 --- a/main/src/com/google/refine/operations/recon/ReconOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconOperation.java @@ -46,7 +46,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; diff --git a/main/src/com/google/refine/operations/recon/ReconUseValuesAsIdentifiersOperation.java b/main/src/com/google/refine/operations/recon/ReconUseValuesAsIdentifiersOperation.java index d6115decc..ef6f51e05 100644 --- a/main/src/com/google/refine/operations/recon/ReconUseValuesAsIdentifiersOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconUseValuesAsIdentifiersOperation.java @@ -6,7 +6,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.expr.ExpressionUtils; diff --git a/main/src/com/google/refine/operations/row/DenormalizeOperation.java b/main/src/com/google/refine/operations/row/DenormalizeOperation.java index aaee43c51..c8888fc93 100644 --- a/main/src/com/google/refine/operations/row/DenormalizeOperation.java +++ b/main/src/com/google/refine/operations/row/DenormalizeOperation.java @@ -37,7 +37,6 @@ import java.util.ArrayList; import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; - import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Cell; diff --git a/main/src/com/google/refine/operations/row/RowFlagOperation.java b/main/src/com/google/refine/operations/row/RowFlagOperation.java index ddf142ec4..63501a273 100644 --- a/main/src/com/google/refine/operations/row/RowFlagOperation.java +++ b/main/src/com/google/refine/operations/row/RowFlagOperation.java @@ -38,7 +38,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; diff --git a/main/src/com/google/refine/operations/row/RowRemovalOperation.java b/main/src/com/google/refine/operations/row/RowRemovalOperation.java index bf536ce0d..f06d9e019 100644 --- a/main/src/com/google/refine/operations/row/RowRemovalOperation.java +++ b/main/src/com/google/refine/operations/row/RowRemovalOperation.java @@ -38,7 +38,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; diff --git a/main/src/com/google/refine/operations/row/RowReorderOperation.java b/main/src/com/google/refine/operations/row/RowReorderOperation.java index b90867846..ac5b23178 100644 --- a/main/src/com/google/refine/operations/row/RowReorderOperation.java +++ b/main/src/com/google/refine/operations/row/RowReorderOperation.java @@ -38,7 +38,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine.Mode; import com.google.refine.browsing.RecordVisitor; diff --git a/main/src/com/google/refine/operations/row/RowStarOperation.java b/main/src/com/google/refine/operations/row/RowStarOperation.java index 1d6caf472..81e2f27f5 100644 --- a/main/src/com/google/refine/operations/row/RowStarOperation.java +++ b/main/src/com/google/refine/operations/row/RowStarOperation.java @@ -38,7 +38,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; diff --git a/main/src/com/google/refine/process/LongRunningProcess.java b/main/src/com/google/refine/process/LongRunningProcess.java index b8e12c669..5d978676d 100644 --- a/main/src/com/google/refine/process/LongRunningProcess.java +++ b/main/src/com/google/refine/process/LongRunningProcess.java @@ -35,7 +35,6 @@ package com.google.refine.process; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.HistoryEntry; abstract public class LongRunningProcess extends Process { diff --git a/main/src/com/google/refine/process/Process.java b/main/src/com/google/refine/process/Process.java index 21af244fa..6633f68c6 100644 --- a/main/src/com/google/refine/process/Process.java +++ b/main/src/com/google/refine/process/Process.java @@ -35,7 +35,6 @@ package com.google.refine.process; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.HistoryEntry; public abstract class Process { diff --git a/main/src/com/google/refine/process/ProcessManager.java b/main/src/com/google/refine/process/ProcessManager.java index 3f5a48283..379de3364 100644 --- a/main/src/com/google/refine/process/ProcessManager.java +++ b/main/src/com/google/refine/process/ProcessManager.java @@ -42,7 +42,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.HistoryEntry; import com.google.refine.history.HistoryProcess; diff --git a/main/src/com/google/refine/process/QuickHistoryEntryProcess.java b/main/src/com/google/refine/process/QuickHistoryEntryProcess.java index 7f16f68e4..25f9cfa6a 100644 --- a/main/src/com/google/refine/process/QuickHistoryEntryProcess.java +++ b/main/src/com/google/refine/process/QuickHistoryEntryProcess.java @@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.process; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.history.HistoryEntry; import com.google.refine.model.Project; diff --git a/main/src/com/google/refine/sorting/Criterion.java b/main/src/com/google/refine/sorting/Criterion.java index a9d01ac26..7e8a56fc8 100644 --- a/main/src/com/google/refine/sorting/Criterion.java +++ b/main/src/com/google/refine/sorting/Criterion.java @@ -33,23 +33,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.sorting; -import java.io.IOException; - -import org.json.JSONException; -import org.json.JSONObject; - import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; - +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.refine.expr.ExpressionUtils; import com.google.refine.model.Column; import com.google.refine.model.Project; import com.google.refine.model.Record; import com.google.refine.model.Row; -import com.google.refine.util.ParsingUtilities; @JsonTypeInfo( use=JsonTypeInfo.Id.NAME, diff --git a/main/src/com/google/refine/sorting/SortingConfig.java b/main/src/com/google/refine/sorting/SortingConfig.java index a8e2e172b..cda207815 100644 --- a/main/src/com/google/refine/sorting/SortingConfig.java +++ b/main/src/com/google/refine/sorting/SortingConfig.java @@ -6,7 +6,6 @@ import org.json.JSONObject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.util.ParsingUtilities; diff --git a/main/src/com/google/refine/sorting/StringCriterion.java b/main/src/com/google/refine/sorting/StringCriterion.java index 78234f767..9ad91a4f7 100644 --- a/main/src/com/google/refine/sorting/StringCriterion.java +++ b/main/src/com/google/refine/sorting/StringCriterion.java @@ -38,7 +38,6 @@ import java.text.Collator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.expr.ExpressionUtils; public class StringCriterion extends Criterion { diff --git a/main/src/com/google/refine/util/JSONUtilities.java b/main/src/com/google/refine/util/JSONUtilities.java index 5d2ef8711..9f8433087 100644 --- a/main/src/com/google/refine/util/JSONUtilities.java +++ b/main/src/com/google/refine/util/JSONUtilities.java @@ -39,17 +39,16 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; -import java.util.Collection; import java.util.List; -import java.util.Map; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeType; - +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; @@ -63,6 +62,14 @@ public class JSONUtilities { } } + static public ObjectNode getObject(ObjectNode obj, String key) { + JsonNode node = obj.get(key); + if(node != null && node instanceof ObjectNode) { + return (ObjectNode)node; + } + return null; + } + static public String getString(JSONObject obj, String key, String def) { try { return obj.getString(key); @@ -72,7 +79,7 @@ public class JSONUtilities { } static public String getString(JsonNode obj, String key, String def) { - if (obj.get(key) != null) { + if (obj.has(key)) { return obj.get(key).textValue(); } else { return def; @@ -88,7 +95,7 @@ public class JSONUtilities { } static public int getInt(JsonNode obj, String key, int def) { - if (obj.get(key) != null) { + if (obj.has(key)) { return obj.get(key).asInt(def); } else { return def; @@ -104,7 +111,7 @@ public class JSONUtilities { } static public boolean getBoolean(JsonNode obj, String key, boolean def) { - if (obj.get(key) != null) { + if (obj.has(key)) { return obj.get(key).asBoolean(def); } else { return def; @@ -155,6 +162,14 @@ public class JSONUtilities { } } + static public ArrayNode getArray(ObjectNode obj, String key) { + JsonNode v = obj.get(key); + if( obj.has(key) && obj.get(key) instanceof ArrayNode) { + return (ArrayNode) obj.get(key); + } + return null; + } + static public List getArray(JsonNode obj, String key) { if (obj.has(key) && obj.get(key).getNodeType().equals(JsonNodeType.ARRAY)) { return Lists.newArrayList(obj.get(key).elements()); @@ -182,6 +197,20 @@ public class JSONUtilities { } } + static public int[] getIntArray(ObjectNode obj, String key) { + ArrayNode a = getArray(obj, key); + if (a == null) { + return new int[0]; + } + int[] r = new int[a.size()]; + int i = 0; + for(JsonNode n : a) { + r[i] = n.asInt(); + i++; + } + return r; + } + static public String[] getStringArray(JSONObject obj, String key) { try { JSONArray a = obj.getJSONArray(key); @@ -197,6 +226,20 @@ public class JSONUtilities { } } + static public String[] getStringArray(ObjectNode obj, String key) { + ArrayNode a = getArray(obj, key); + if (a == null) { + return new String[0]; + } + String[] r = new String[a.size()]; + + for (int i = 0; i < r.length; i++) { + r[i] = a.get(i).asText(); + } + + return r; + } + static public void getStringList(JSONObject obj, String key, List list) { try { JSONArray a = obj.getJSONArray(key); @@ -235,6 +278,14 @@ public class JSONUtilities { } } + static public ObjectNode getObjectElement(ArrayNode a, int i) { + JsonNode n = a.get(i); + if (n != null && n instanceof ObjectNode) { + return (ObjectNode) n; + } + return null; + } + static public int getIntElement(JSONArray a, int i, int def) { try { return a.getInt(i); @@ -243,13 +294,28 @@ public class JSONUtilities { } } - static public void append(JSONArray a, JSONObject element) { + static public int getIntElement(ArrayNode a, int i, int def) { + if (a.get(i) != null) { + return a.get(i).asInt(def); + } + return def; + } + + static public void append(JSONArray sheetRecords, JSONObject sheetRecord) { try { - a.put(a.length(), element); + sheetRecords.put(sheetRecords.length(), sheetRecord); } catch (JSONException e) { } } + static public void append(ArrayNode sheetRecords, ObjectNode sheetRecord) { + sheetRecords.add(sheetRecord); + } + + static public void append(ArrayNode array, int v) { + array.add(v); + } + static public void append(JSONArray a, Object element) { try { a.put(a.length(), element); @@ -292,9 +358,13 @@ public class JSONUtilities { } } - static public void safePut(JSONObject obj, String key, int value) { + static public void append(ArrayNode a, String element) { + a.add(element); + } + + static public void safePut(ObjectNode options, String key, JsonNode rootElement) { try { - obj.put(key, value); + options.put(key, rootElement); } catch (JSONException e) { // Ignore: the JSONObject is just too happy about throwing exceptions. } @@ -309,6 +379,15 @@ public class JSONUtilities { } } + static public void safeInc(ObjectNode obj, String key) { + int currentValue = getInt(obj, key, 0); + safePut(obj, key, currentValue + 1); + } + + static public void safePut(ObjectNode obj, String key, long value) { + obj.put(key, value); + } + static public void safePut(JSONObject obj, String key, long value) { try { obj.put(key, value); @@ -317,6 +396,10 @@ public class JSONUtilities { } } + static public void safePut(ObjectNode obj, String key, double value) { + obj.put(key, value); + } + static public void safePut(JSONObject obj, String key, double value) { try { obj.put(key, value); @@ -325,6 +408,10 @@ public class JSONUtilities { } } + static public void safePut(ObjectNode obj, String key, boolean value) { + obj.put(key, value); + } + static public void safePut(JSONObject obj, String key, boolean value) { try { obj.put(key, value); @@ -333,6 +420,10 @@ public class JSONUtilities { } } + static public void safePut(ObjectNode obj, String key, String value) { + obj.put(key, value); + } + static public void safePut(JSONObject obj, String key, String value) { try { obj.put(key, value); @@ -341,22 +432,6 @@ public class JSONUtilities { } } - static public void safePut(JSONObject obj, String key, Collection value) { - try { - obj.put(key, value); - } catch (JSONException e) { - // Ignore: the JSONObject is just too happy about throwing exceptions. - } - } - - static public void safePut(JSONObject obj, String key, Map value) { - try { - obj.put(key, value); - } catch (JSONException e) { - // Ignore: the JSONObject is just too happy about throwing exceptions. - } - } - static public void safePut(JSONObject obj, String key, Object value) { try { obj.put(key, value); @@ -393,4 +468,13 @@ public class JSONUtilities { destArray.put(srcArray.get(i)); } } + + // temporary method used during migratino + static public ObjectNode jsonObjectToObjectNode(JSONObject obj) { + return ParsingUtilities.evaluateJsonStringToObjectNode(obj.toString()); + } + + public static JSONObject objectNodeToJsonNode(ObjectNode fieldJsonObj) { + return new JSONObject(fieldJsonObj.toString()); + } } diff --git a/main/src/com/google/refine/util/ParsingUtilities.java b/main/src/com/google/refine/util/ParsingUtilities.java index f4426a989..9717c880b 100644 --- a/main/src/com/google/refine/util/ParsingUtilities.java +++ b/main/src/com/google/refine/util/ParsingUtilities.java @@ -62,9 +62,12 @@ import org.json.JSONTokener; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.ser.FilterProvider; import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider; @@ -269,4 +272,28 @@ public class ParsingUtilities { cal.setTimeInMillis(offsetDateTime.toInstant().toEpochMilli()); return cal; } + + public static ObjectNode evaluateJsonStringToObjectNode(String optionsString) { + try { + JsonNode tree = mapper.readTree(optionsString); + if(tree instanceof ObjectNode) { + return (ObjectNode)tree; + } + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + public static ArrayNode evaluateJsonStringToArrayNode(String parameter) { + try { + JsonNode tree = mapper.readTree(parameter); + if(tree instanceof ArrayNode) { + return (ArrayNode)tree; + } + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } } diff --git a/main/src/com/google/refine/util/Pool.java b/main/src/com/google/refine/util/Pool.java index d91ca0f9a..ab12395fa 100644 --- a/main/src/com/google/refine/util/Pool.java +++ b/main/src/com/google/refine/util/Pool.java @@ -46,7 +46,6 @@ import java.util.HashMap; import java.util.Map; import com.fasterxml.jackson.annotation.JsonProperty; - import com.google.refine.RefineServlet; import com.google.refine.model.Recon; import com.google.refine.model.ReconCandidate; diff --git a/main/src/com/google/refine/util/SerializationFilters.java b/main/src/com/google/refine/util/SerializationFilters.java index 4c937c25f..37902617d 100644 --- a/main/src/com/google/refine/util/SerializationFilters.java +++ b/main/src/com/google/refine/util/SerializationFilters.java @@ -11,7 +11,6 @@ import com.fasterxml.jackson.databind.ser.PropertyFilter; import com.fasterxml.jackson.databind.ser.PropertyWriter; import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter; import com.fasterxml.jackson.databind.ser.std.StdSerializer; - import com.google.refine.model.Recon; import com.google.refine.model.Recon.Judgment; diff --git a/main/tests/server/src/com/google/refine/tests/RefineTest.java b/main/tests/server/src/com/google/refine/tests/RefineTest.java index 26d21eae6..a4429c74b 100644 --- a/main/tests/server/src/com/google/refine/tests/RefineTest.java +++ b/main/tests/server/src/com/google/refine/tests/RefineTest.java @@ -54,6 +54,11 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeSuite; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import com.google.refine.ProjectManager; import com.google.refine.RefineServlet; import com.google.refine.importers.SeparatorBasedImporter; @@ -165,7 +170,7 @@ public class RefineTest { ProjectMetadata metadata = new ProjectMetadata(); metadata.setName(projectName); - JSONObject options = mock(JSONObject.class); + ObjectNode options = mock(ObjectNode.class); prepareImportOptions(options, ",", -1, 0, 0, 1, false, false); ImportingJob job = ImportingManager.createJob(); @@ -193,7 +198,7 @@ public class RefineTest { * @param guessValueType * @param ignoreQuotes */ - private void prepareImportOptions(JSONObject options, + private void prepareImportOptions(ObjectNode options, String sep, int limit, int skip, int ignoreLines, int headerLines, boolean guessValueType, boolean ignoreQuotes) { @@ -281,26 +286,51 @@ public class RefineTest { when(JSONUtilities.getBoolean(options, name, def)).thenReturn(def); } + static public void whenGetBooleanOption(String name, ObjectNode options, Boolean def){ + when(options.has(name)).thenReturn(true); + when(options.get(name)).thenReturn(def ? BooleanNode.TRUE : BooleanNode.FALSE); + } + static public void whenGetIntegerOption(String name, JSONObject options, int def){ when(options.has(name)).thenReturn(true); when(JSONUtilities.getInt(options, name, def)).thenReturn(def); } + static public void whenGetIntegerOption(String name, ObjectNode options, int def){ + when(options.has(name)).thenReturn(true); + when(options.get(name)).thenReturn(new IntNode(def)); + } + static public void whenGetStringOption(String name, JSONObject options, String def){ when(options.has(name)).thenReturn(true); when(JSONUtilities.getString(options, name, def)).thenReturn(def); } + static public void whenGetStringOption(String name, ObjectNode options, String def){ + when(options.has(name)).thenReturn(true); + when(options.get(name)).thenReturn(new TextNode(def)); + } + static public void whenGetObjectOption(String name, JSONObject options, JSONObject def){ when(options.has(name)).thenReturn(true); when(JSONUtilities.getObject(options, name)).thenReturn(def); } + static public void whenGetObjectOption(String name, ObjectNode options, ObjectNode def){ + when(options.has(name)).thenReturn(true); + when(options.get(name)).thenReturn(def); + } + static public void whenGetArrayOption(String name, JSONObject options, JSONArray def){ when(options.has(name)).thenReturn(true); when(JSONUtilities.getArray(options, name)).thenReturn(def); } + static public void whenGetArrayOption(String name, ObjectNode options, ArrayNode def){ + when(options.has(name)).thenReturn(true); + when(options.get(name)).thenReturn(def); + } + static public void verifyGetOption(String name, JSONObject options){ verify(options, times(1)).has(name); try { @@ -311,13 +341,9 @@ public class RefineTest { } // Works for both int, String, and JSON arrays - static public void verifyGetArrayOption(String name, JSONObject options){ + static public void verifyGetArrayOption(String name, ObjectNode options){ verify(options, times(1)).has(name); - try { - verify(options, times(1)).getJSONArray(name); - } catch (JSONException e) { - Assert.fail("JSONException",e); - } + verify(options, times(1)).get(name); } protected ButterflyModule getCoreModule() { diff --git a/main/tests/server/src/com/google/refine/tests/browsing/EngineTests.java b/main/tests/server/src/com/google/refine/tests/browsing/EngineTests.java index e6d875452..4ed43f040 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/EngineTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/EngineTests.java @@ -1,6 +1,7 @@ package com.google.refine.tests.browsing; import static org.mockito.Mockito.mock; + import org.testng.annotations.Test; import com.google.refine.browsing.Engine; diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/EngineConfigTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/EngineConfigTests.java index d63cd2f76..51847b3ef 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/facets/EngineConfigTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/EngineConfigTests.java @@ -4,8 +4,8 @@ import org.json.JSONObject; import org.testng.Assert; import org.testng.annotations.Test; -import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.Engine.Mode; +import com.google.refine.browsing.EngineConfig; import com.google.refine.tests.util.TestUtils; public class EngineConfigTests { diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/ListFacetTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/ListFacetTests.java index 200134d6f..a9994954f 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/facets/ListFacetTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/ListFacetTests.java @@ -6,7 +6,6 @@ import org.testng.annotations.Test; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.facets.Facet; import com.google.refine.browsing.facets.ListFacet.ListFacetConfig; diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/RangeFacetTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/RangeFacetTests.java index 21374439d..ecfd51b26 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/facets/RangeFacetTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/RangeFacetTests.java @@ -6,7 +6,6 @@ import org.testng.annotations.Test; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.facets.RangeFacet; import com.google.refine.browsing.facets.RangeFacet.RangeFacetConfig; diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/ScatterplotFacetTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/ScatterplotFacetTests.java index 29e7e9112..1aceb9392 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/facets/ScatterplotFacetTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/ScatterplotFacetTests.java @@ -6,7 +6,6 @@ import org.testng.annotations.Test; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.facets.ScatterplotFacet; import com.google.refine.browsing.facets.ScatterplotFacet.ScatterplotFacetConfig; diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java index 5d8903bc8..5d9b14baf 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java @@ -44,7 +44,6 @@ import org.testng.annotations.Test; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; - import com.google.refine.browsing.RowFilter; import com.google.refine.browsing.facets.TextSearchFacet; import com.google.refine.browsing.facets.TextSearchFacet.TextSearchFacetConfig; diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/TimeRangeFacetTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/TimeRangeFacetTests.java index a0741a03c..4c77b4684 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/facets/TimeRangeFacetTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/TimeRangeFacetTests.java @@ -7,7 +7,6 @@ import org.testng.annotations.Test; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; - import com.google.refine.browsing.Engine; import com.google.refine.browsing.facets.TimeRangeFacet; import com.google.refine.browsing.facets.TimeRangeFacet.TimeRangeFacetConfig; diff --git a/main/tests/server/src/com/google/refine/tests/browsing/util/ExpressionNominalValueGrouperTests.java b/main/tests/server/src/com/google/refine/tests/browsing/util/ExpressionNominalValueGrouperTests.java index 6b482856e..c0bbf169d 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/util/ExpressionNominalValueGrouperTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/util/ExpressionNominalValueGrouperTests.java @@ -30,24 +30,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.browsing.util; -import com.google.refine.expr.MetaParser; -import org.testng.annotations.AfterMethod; -import org.testng.Assert; - import java.io.IOException; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.Properties; - import org.json.JSONException; import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; import com.google.refine.browsing.util.ExpressionNominalValueGrouper; import com.google.refine.expr.Evaluable; +import com.google.refine.expr.MetaParser; import com.google.refine.model.Cell; import com.google.refine.model.ModelException; import com.google.refine.model.Project; diff --git a/main/tests/server/src/com/google/refine/tests/commands/expr/ExpressionCommandTestBase.java b/main/tests/server/src/com/google/refine/tests/commands/expr/ExpressionCommandTestBase.java index f2ffb2244..34c55c990 100644 --- a/main/tests/server/src/com/google/refine/tests/commands/expr/ExpressionCommandTestBase.java +++ b/main/tests/server/src/com/google/refine/tests/commands/expr/ExpressionCommandTestBase.java @@ -16,7 +16,6 @@ import org.testng.annotations.BeforeMethod; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; - import com.google.refine.commands.Command; import com.google.refine.io.FileProjectManager; import com.google.refine.tests.util.TestUtils; diff --git a/main/tests/server/src/com/google/refine/tests/commands/expr/GetExpressionLanguageInfoCommandTests.java b/main/tests/server/src/com/google/refine/tests/commands/expr/GetExpressionLanguageInfoCommandTests.java index b80159be2..ef895d8d6 100644 --- a/main/tests/server/src/com/google/refine/tests/commands/expr/GetExpressionLanguageInfoCommandTests.java +++ b/main/tests/server/src/com/google/refine/tests/commands/expr/GetExpressionLanguageInfoCommandTests.java @@ -8,7 +8,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import com.fasterxml.jackson.databind.JsonNode; - import com.google.refine.commands.expr.GetExpressionLanguageInfoCommand; import com.google.refine.tests.util.TestUtils; import com.google.refine.util.ParsingUtilities; diff --git a/main/tests/server/src/com/google/refine/tests/commands/expr/PreviewExpressionCommandTests.java b/main/tests/server/src/com/google/refine/tests/commands/expr/PreviewExpressionCommandTests.java index 268107b51..ab4ee0390 100644 --- a/main/tests/server/src/com/google/refine/tests/commands/expr/PreviewExpressionCommandTests.java +++ b/main/tests/server/src/com/google/refine/tests/commands/expr/PreviewExpressionCommandTests.java @@ -1,12 +1,12 @@ package com.google.refine.tests.commands.expr; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; diff --git a/main/tests/server/src/com/google/refine/tests/commands/expr/ToggleStarredExpressionCommandTests.java b/main/tests/server/src/com/google/refine/tests/commands/expr/ToggleStarredExpressionCommandTests.java index 019fa43ba..91a6d296e 100644 --- a/main/tests/server/src/com/google/refine/tests/commands/expr/ToggleStarredExpressionCommandTests.java +++ b/main/tests/server/src/com/google/refine/tests/commands/expr/ToggleStarredExpressionCommandTests.java @@ -1,8 +1,9 @@ package com.google.refine.tests.commands.expr; -import java.io.IOException; import static org.mockito.Mockito.when; +import java.io.IOException; + import javax.servlet.ServletException; import org.testng.annotations.BeforeMethod; diff --git a/main/tests/server/src/com/google/refine/tests/commands/project/ValidateOperationTests.java b/main/tests/server/src/com/google/refine/tests/commands/project/ValidateOperationTests.java index 528a43e60..8582d657c 100644 --- a/main/tests/server/src/com/google/refine/tests/commands/project/ValidateOperationTests.java +++ b/main/tests/server/src/com/google/refine/tests/commands/project/ValidateOperationTests.java @@ -28,10 +28,10 @@ import org.testng.Assert; import com.google.refine.ProjectManager; import com.google.refine.importers.SeparatorBasedImporter; import com.google.refine.model.Column; -import com.google.refine.model.metadata.validator.ValidateOperation; import com.google.refine.model.metadata.DataPackageMetadata; import com.google.refine.model.metadata.MetadataFactory; import com.google.refine.model.metadata.MetadataFormat; +import com.google.refine.model.metadata.validator.ValidateOperation; import com.google.refine.tests.importers.TsvCsvImporterTests; import com.google.refine.util.ParsingUtilities; diff --git a/main/tests/server/src/com/google/refine/tests/commands/row/GetRowsCommandTest.java b/main/tests/server/src/com/google/refine/tests/commands/row/GetRowsCommandTest.java index d35315c7d..52e5e06d6 100644 --- a/main/tests/server/src/com/google/refine/tests/commands/row/GetRowsCommandTest.java +++ b/main/tests/server/src/com/google/refine/tests/commands/row/GetRowsCommandTest.java @@ -1,5 +1,12 @@ package com.google.refine.tests.commands.row; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; + import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -13,13 +20,6 @@ import com.google.refine.model.Project; import com.google.refine.tests.RefineTest; import com.google.refine.tests.util.TestUtils; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; - public class GetRowsCommandTest extends RefineTest { HttpServletRequest request = null; diff --git a/main/tests/server/src/com/google/refine/tests/commands/util/CancelProcessesCommandTests.java b/main/tests/server/src/com/google/refine/tests/commands/util/CancelProcessesCommandTests.java index a1ed70c2a..45d5356ab 100644 --- a/main/tests/server/src/com/google/refine/tests/commands/util/CancelProcessesCommandTests.java +++ b/main/tests/server/src/com/google/refine/tests/commands/util/CancelProcessesCommandTests.java @@ -33,7 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.commands.util; -import static org.mockito.Matchers.anyLong; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; diff --git a/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java b/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java index 0b3073c20..9243596be 100644 --- a/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java +++ b/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java @@ -71,7 +71,6 @@ import com.google.refine.model.Row; import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.tests.ProjectManagerStub; import com.google.refine.tests.RefineTest; -import com.google.refine.util.JSONUtilities; import com.google.refine.util.ParsingUtilities; diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/ContainsFunctionTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/ContainsFunctionTests.java index 735c82dfb..7f5c5ac3c 100644 --- a/main/tests/server/src/com/google/refine/tests/expr/functions/ContainsFunctionTests.java +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/ContainsFunctionTests.java @@ -1,11 +1,9 @@ package com.google.refine.tests.expr.functions; -import com.google.refine.RefineServlet; -import com.google.refine.grel.ControlFunctionRegistry; -import com.google.refine.grel.Function; -import com.google.refine.tests.RefineServletStub; -import com.google.refine.tests.RefineTest; +import java.util.Properties; +import java.util.regex.Pattern; + import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -13,8 +11,11 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import java.util.Properties; -import java.util.regex.Pattern; +import com.google.refine.RefineServlet; +import com.google.refine.grel.ControlFunctionRegistry; +import com.google.refine.grel.Function; +import com.google.refine.tests.RefineServletStub; +import com.google.refine.tests.RefineTest; /** * Test cases for find function. diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java index 7d64488fd..cb7f467c4 100644 --- a/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java @@ -44,8 +44,8 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; import com.google.refine.expr.EvalError; -import com.google.refine.expr.util.CalendarParserException; import com.google.refine.expr.util.CalendarParser; +import com.google.refine.expr.util.CalendarParserException; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.Function; import com.google.refine.tests.RefineTest; diff --git a/main/tests/server/src/com/google/refine/tests/history/HistoryTests.java b/main/tests/server/src/com/google/refine/tests/history/HistoryTests.java index 759fa8273..81641b3fa 100644 --- a/main/tests/server/src/com/google/refine/tests/history/HistoryTests.java +++ b/main/tests/server/src/com/google/refine/tests/history/HistoryTests.java @@ -47,6 +47,7 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; import com.google.refine.ProjectManager; +import com.google.refine.history.Change; import com.google.refine.history.History; import com.google.refine.history.HistoryEntry; import com.google.refine.history.HistoryEntryManager; @@ -54,7 +55,6 @@ import com.google.refine.model.Project; import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.tests.RefineTest; import com.google.refine.tests.util.TestUtils; -import com.google.refine.history.Change; public class HistoryTests extends RefineTest { diff --git a/main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java b/main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java index efc36c8d9..86732e1d0 100644 --- a/main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java +++ b/main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java @@ -53,9 +53,7 @@ import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook; -import org.json.JSONArray; import org.json.JSONException; -import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -63,8 +61,9 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.refine.importers.ExcelImporter; -import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; public class ExcelImporterTests extends ImporterTest { @@ -101,11 +100,10 @@ public class ExcelImporterTests extends ImporterTest { //---------------------read tests------------------------ @Test - public void readXls() throws FileNotFoundException, JSONException{ + public void readXls() throws FileNotFoundException, IOException{ - JSONArray sheets = new JSONArray(); - JSONUtilities.append(sheets, - new JSONObject("{name: \"file-source#Test Sheet 0\", fileNameAndSheetIndex: \"file-source#0\", rows: 31, selected: true}")); + ArrayNode sheets = ParsingUtilities.mapper.createArrayNode(); + sheets.add(ParsingUtilities.mapper.readTree("{name: \"file-source#Test Sheet 0\", fileNameAndSheetIndex: \"file-source#0\", rows: 31, selected: true}")); whenGetArrayOption("sheets", options, sheets); whenGetIntegerOption("ignoreLines", options, 0); @@ -134,11 +132,11 @@ public class ExcelImporterTests extends ImporterTest { Assert.assertNull((String)project.rows.get(1).getCellValue(5)); try { - verify(options, times(1)).getInt("ignoreLines"); - verify(options, times(1)).getInt("headerLines"); - verify(options, times(1)).getInt("skipDataLines"); - verify(options, times(1)).getInt("limit"); - verify(options, times(1)).getBoolean("storeBlankCellsAsNulls"); + verify(options, times(1)).get("ignoreLines"); + verify(options, times(1)).get("headerLines"); + verify(options, times(1)).get("skipDataLines"); + verify(options, times(1)).get("limit"); + verify(options, times(1)).get("storeBlankCellsAsNulls"); } catch (JSONException e) { Assert.fail("JSON exception",e); } diff --git a/main/tests/server/src/com/google/refine/tests/importers/FixedWidthImporterTests.java b/main/tests/server/src/com/google/refine/tests/importers/FixedWidthImporterTests.java index 2fbd05ab2..0a1633f4e 100644 --- a/main/tests/server/src/com/google/refine/tests/importers/FixedWidthImporterTests.java +++ b/main/tests/server/src/com/google/refine/tests/importers/FixedWidthImporterTests.java @@ -1,13 +1,8 @@ package com.google.refine.tests.importers; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - import java.io.StringReader; -import org.json.JSONArray; -import org.json.JSONException; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -15,8 +10,10 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.refine.importers.FixedWidthImporter; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; public class FixedWidthImporterTests extends ImporterTest { @Override @@ -50,16 +47,16 @@ public class FixedWidthImporterTests extends ImporterTest { public void readFixedWidth(){ StringReader reader = new StringReader(SAMPLE_ROW + "\nTooShort"); - JSONArray columnWidths = new JSONArray(); + ArrayNode columnWidths = ParsingUtilities.mapper.createArrayNode(); JSONUtilities.append(columnWidths, 6); JSONUtilities.append(columnWidths, 9); JSONUtilities.append(columnWidths, 5); whenGetArrayOption("columnWidths", options, columnWidths); - JSONArray columnNames = new JSONArray(); - JSONUtilities.append(columnNames, "Col 1"); - JSONUtilities.append(columnNames, "Col 2"); - JSONUtilities.append(columnNames, "Col 3"); + ArrayNode columnNames = ParsingUtilities.mapper.createArrayNode(); + columnNames.add("Col 1"); + columnNames.add("Col 2"); + columnNames.add("Col 3"); whenGetArrayOption("columnNames", options, columnNames); whenGetIntegerOption("ignoreLines", options, 0); @@ -83,17 +80,5 @@ public class FixedWidthImporterTests extends ImporterTest { Assert.assertEquals((String)project.rows.get(2).getCellValue(0), "TooSho"); Assert.assertEquals((String)project.rows.get(2).getCellValue(1), "rt"); Assert.assertNull(project.rows.get(2).getCellValue(2)); - - verifyGetArrayOption("columnNames", options); - try { - verify(options, times(1)).getJSONArray("columnWidths"); - verify(options, times(1)).getInt("ignoreLines"); - verify(options, times(1)).getInt("headerLines"); - verify(options, times(1)).getInt("skipDataLines"); - verify(options, times(1)).getInt("limit"); - verify(options, times(1)).getBoolean("storeBlankCellsAsNulls"); - } catch (JSONException e) { - Assert.fail("JSON exception",e); - } } } diff --git a/main/tests/server/src/com/google/refine/tests/importers/ImporterTest.java b/main/tests/server/src/com/google/refine/tests/importers/ImporterTest.java index 46161e043..58704b353 100644 --- a/main/tests/server/src/com/google/refine/tests/importers/ImporterTest.java +++ b/main/tests/server/src/com/google/refine/tests/importers/ImporterTest.java @@ -9,9 +9,9 @@ import java.io.Reader; import java.util.ArrayList; import java.util.List; -import org.json.JSONObject; import org.mockito.Mockito; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.RefineServlet; import com.google.refine.importers.ImportingParserBase; import com.google.refine.importers.tree.ImportColumnGroup; @@ -23,6 +23,7 @@ import com.google.refine.model.Project; import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.tests.RefineServletStub; import com.google.refine.tests.RefineTest; +import com.google.refine.util.ParsingUtilities; abstract public class ImporterTest extends RefineTest { //mock dependencies @@ -31,7 +32,7 @@ abstract public class ImporterTest extends RefineTest { protected ImportingJob job; protected RefineServlet servlet; - protected JSONObject options; + protected ObjectNode options; public void setUp(){ //FIXME - should we try and use mock(Project.class); - seems unnecessary complexity @@ -42,9 +43,9 @@ abstract public class ImporterTest extends RefineTest { metadata = new ProjectMetadata(); ImportingJob spiedJob = ImportingManager.createJob(); job = Mockito.spy(spiedJob); - when(job.getRetrievalRecord()).thenReturn(new JSONObject()); + when(job.getRetrievalRecord()).thenReturn(ParsingUtilities.mapper.createObjectNode()); - options = Mockito.mock(JSONObject.class); + options = Mockito.mock(ObjectNode.class); } public void tearDown(){ @@ -102,12 +103,12 @@ abstract public class ImporterTest extends RefineTest { project.columnModel.update(); } - protected void parseOneFile(TreeImportingParserBase parser, InputStream inputStream, JSONObject options) { + protected void parseOneFile(TreeImportingParserBase parser, InputStream inputStream, ObjectNode options) { parseOneInputStreamAsReader(parser, inputStream, options); } protected void parseOneInputStream( - TreeImportingParserBase parser, InputStream inputStream, JSONObject options) { + TreeImportingParserBase parser, InputStream inputStream, ObjectNode options) { ImportColumnGroup rootColumnGroup = new ImportColumnGroup(); List exceptions = new ArrayList(); @@ -126,7 +127,7 @@ abstract public class ImporterTest extends RefineTest { } protected void parseOneInputStreamAsReader( - TreeImportingParserBase parser, InputStream inputStream, JSONObject options) { + TreeImportingParserBase parser, InputStream inputStream, ObjectNode options) { ImportColumnGroup rootColumnGroup = new ImportColumnGroup(); List exceptions = new ArrayList(); diff --git a/main/tests/server/src/com/google/refine/tests/importers/JsonImporterTests.java b/main/tests/server/src/com/google/refine/tests/importers/JsonImporterTests.java index 3f6ba1973..b893d3834 100644 --- a/main/tests/server/src/com/google/refine/tests/importers/JsonImporterTests.java +++ b/main/tests/server/src/com/google/refine/tests/importers/JsonImporterTests.java @@ -39,8 +39,6 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.util.LinkedList; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -48,6 +46,8 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importers.JsonImporter; import com.google.refine.importers.JsonImporter.JSONTreeReader; import com.google.refine.importers.tree.TreeImportingParserBase; @@ -55,6 +55,7 @@ import com.google.refine.importers.tree.TreeReader.Token; import com.google.refine.importing.ImportingJob; import com.google.refine.model.Row; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; public class JsonImporterTests extends ImporterTest { @Override @@ -166,12 +167,12 @@ public class JsonImporterTests extends ImporterTest { public void testElementWithMqlReadOutput(){ String mqlOutput = "{\"code\":\"/api/status/ok\",\"result\":[{\"armed_force\":{\"id\":\"/en/wehrmacht\"},\"id\":\"/en/afrika_korps\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/en/sacred_band_of_thebes\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/en/british_army\"},\"id\":\"/en/british_16_air_assault_brigade\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/en/british_army\"},\"id\":\"/en/pathfinder_platoon\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0ch7qgz\"},\"id\":\"/en/sacred_band\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/en/polish_navy\"},\"id\":\"/en/3rd_ship_flotilla\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxn9\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxq9\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxqh\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxqp\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxqw\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c1wxl3\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c1wxlp\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0ck96kz\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0cm3j23\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0cw8hb4\",\"type\":\"/military/military_unit\"}],\"status\":\"200 OK\",\"transaction_id\":\"cache;cache01.p01.sjc1:8101;2010-10-04T15:04:33Z;0007\"}"; - JSONObject options = SUT.createParserUIInitializationData( - job, new LinkedList(), "text/json"); - JSONArray path = new JSONArray(); - JSONUtilities.append(path, JsonImporter.ANONYMOUS); - JSONUtilities.append(path, "result"); - JSONUtilities.append(path, JsonImporter.ANONYMOUS); + ObjectNode options = SUT.createParserUIInitializationData( + job, new LinkedList<>(), "text/json"); + ArrayNode path = ParsingUtilities.mapper.createArrayNode(); + path.add(JsonImporter.ANONYMOUS); + path.add("result"); + path.add(JsonImporter.ANONYMOUS); JSONUtilities.safePut(options, "recordPath", path); RunTest(mqlOutput, options); @@ -404,11 +405,11 @@ public class JsonImporterTests extends ImporterTest { return sb.toString(); } - private static JSONObject getOptions(ImportingJob job, TreeImportingParserBase parser, String pathSelector) { - JSONObject options = parser.createParserUIInitializationData( - job, new LinkedList(), "text/json"); + private static ObjectNode getOptions(ImportingJob job, TreeImportingParserBase parser, String pathSelector) { + ObjectNode options = parser.createParserUIInitializationData( + job, new LinkedList<>(), "text/json"); - JSONArray path = new JSONArray(); + ArrayNode path = ParsingUtilities.mapper.createArrayNode(); JSONUtilities.append(path, JsonImporter.ANONYMOUS); JSONUtilities.append(path, pathSelector); @@ -507,7 +508,7 @@ public class JsonImporterTests extends ImporterTest { RunTest(testString, getOptions(job, SUT, "institutes")); } - private void RunTest(String testString, JSONObject options) { + private void RunTest(String testString, ObjectNode options) { try { inputStream = new ByteArrayInputStream( testString.getBytes( "UTF-8" ) ); } catch (UnsupportedEncodingException e1) { diff --git a/main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java b/main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java index 21d29db68..8481a98d6 100644 --- a/main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java +++ b/main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java @@ -33,13 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.importers; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - import java.io.StringReader; -import org.json.JSONArray; -import org.json.JSONException; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -49,6 +44,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import com.google.refine.importers.SeparatorBasedImporter; +import com.google.refine.util.ParsingUtilities; public class TsvCsvImporterTests extends ImporterTest { @@ -522,7 +518,7 @@ public class TsvCsvImporterTests extends ImporterTest { String input = "data1" + inputSeparator + "data2" + inputSeparator + "data3\n"; try { - prepareOptions(sep, -1, 0, 0, 1, false, false,"\"","[col1,col2,col3]"); + prepareOptions(sep, -1, 0, 0, 1, false, false,"\"","[\"col1\",\"col2\",\"col3\"]"); parseOneFile(SUT, new StringReader(input)); } catch (Exception e) { Assert.fail("Exception during file parse",e); @@ -554,8 +550,6 @@ public class TsvCsvImporterTests extends ImporterTest { Assert.assertEquals((String)project.rows.get(0).cells.get(0).value, "NDB_No"); Assert.assertEquals((String)project.rows.get(0).cells.get(1).value, "Shrt_Desc"); Assert.assertEquals((String)project.rows.get(0).cells.get(2).value, "Water"); - - verifyOptions(); } @Test @@ -577,8 +571,6 @@ public class TsvCsvImporterTests extends ImporterTest { Assert.assertEquals((String)project.rows.get(0).cells.get(1).value, "data2"); Assert.assertEquals((String)project.rows.get(0).cells.get(2).value, "data3"); Assert.assertEquals((String)project.rows.get(0).cells.get(3).value, "data4"); - - verifyOptions(); } //--helpers-- @@ -618,23 +610,6 @@ public class TsvCsvImporterTests extends ImporterTest { whenGetBooleanOption("guessCellValueTypes", options, guessValueType); whenGetBooleanOption("processQuotes", options, !ignoreQuotes); whenGetBooleanOption("storeBlankCellsAsNulls", options, true); - whenGetArrayOption("columnNames", options, new JSONArray(columnNames)); + whenGetArrayOption("columnNames", options, ParsingUtilities.evaluateJsonStringToArrayNode(columnNames)); } - - private void verifyOptions() { - try { - verify(options, times(1)).getString("separator"); - verify(options, times(1)).getInt("limit"); - verify(options, times(1)).getInt("skipDataLines"); - verify(options, times(1)).getInt("ignoreLines"); - verify(options, times(1)).getInt("headerLines"); - verify(options, times(1)).getBoolean("guessCellValueTypes"); - verify(options, times(1)).getBoolean("processQuotes"); - verify(options, times(1)).getBoolean("storeBlankCellsAsNulls"); - verify(options, times(1)).getJSONArray("columnNames"); - } catch (JSONException e) { - Assert.fail("JSON exception",e); - } - - } } diff --git a/main/tests/server/src/com/google/refine/tests/importers/XmlImporterTests.java b/main/tests/server/src/com/google/refine/tests/importers/XmlImporterTests.java index 6ca5f195d..40d35d882 100644 --- a/main/tests/server/src/com/google/refine/tests/importers/XmlImporterTests.java +++ b/main/tests/server/src/com/google/refine/tests/importers/XmlImporterTests.java @@ -39,8 +39,6 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.util.LinkedList; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -48,11 +46,14 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importers.XmlImporter; import com.google.refine.importers.tree.TreeImportingParserBase; import com.google.refine.importing.ImportingJob; import com.google.refine.model.Row; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; public class XmlImporterTests extends ImporterTest { @@ -223,11 +224,11 @@ public class XmlImporterTests extends ImporterTest { return sb.toString(); } - public static JSONObject getOptions(ImportingJob job, TreeImportingParserBase parser) { - JSONObject options = parser.createParserUIInitializationData( - job, new LinkedList(), "text/json"); + public static ObjectNode getOptions(ImportingJob job, TreeImportingParserBase parser) { + ObjectNode options = parser.createParserUIInitializationData( + job, new LinkedList<>(), "text/json"); - JSONArray path = new JSONArray(); + ArrayNode path = ParsingUtilities.mapper.createArrayNode(); JSONUtilities.append(path, "library"); JSONUtilities.append(path, "book"); @@ -235,11 +236,11 @@ public class XmlImporterTests extends ImporterTest { return options; } - public static JSONObject getNestedOptions(ImportingJob job, TreeImportingParserBase parser) { - JSONObject options = parser.createParserUIInitializationData( - job, new LinkedList(), "text/json"); + public static ObjectNode getNestedOptions(ImportingJob job, TreeImportingParserBase parser) { + ObjectNode options = parser.createParserUIInitializationData( + job, new LinkedList<>(), "text/json"); - JSONArray path = new JSONArray(); + ArrayNode path = ParsingUtilities.mapper.createArrayNode(); JSONUtilities.append(path, "nest"); JSONUtilities.append(path, "nest2"); JSONUtilities.append(path, "library"); @@ -336,7 +337,7 @@ public class XmlImporterTests extends ImporterTest { RunTest(testString, getOptions(job, SUT)); } - private void RunTest(String testString, JSONObject options) { + private void RunTest(String testString, ObjectNode objectNode) { try { inputStream = new ByteArrayInputStream(testString.getBytes( "UTF-8" )); } catch (UnsupportedEncodingException e1) { @@ -344,7 +345,7 @@ public class XmlImporterTests extends ImporterTest { } try { - parseOneFile(SUT, inputStream, options); + parseOneFile(SUT, inputStream, objectNode); } catch (Exception e) { e.printStackTrace(); Assert.fail(); @@ -352,7 +353,7 @@ public class XmlImporterTests extends ImporterTest { } @Override - protected void parseOneFile(TreeImportingParserBase parser, InputStream inputStream, JSONObject options) { + protected void parseOneFile(TreeImportingParserBase parser, InputStream inputStream, ObjectNode options) { parseOneInputStream(parser, inputStream, options); } } diff --git a/main/tests/server/src/com/google/refine/tests/importing/ImportingUtilitiesTests.java b/main/tests/server/src/com/google/refine/tests/importing/ImportingUtilitiesTests.java index 7bcf306bc..0edd996d8 100644 --- a/main/tests/server/src/com/google/refine/tests/importing/ImportingUtilitiesTests.java +++ b/main/tests/server/src/com/google/refine/tests/importing/ImportingUtilitiesTests.java @@ -4,12 +4,12 @@ package com.google.refine.tests.importing; import java.io.InputStream; import java.util.LinkedList; -import org.json.JSONArray; -import org.json.JSONObject; import org.testng.Assert; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.importers.XmlImporter; import com.google.refine.importers.tree.TreeImportingParserBase; import com.google.refine.importing.ImportingJob; @@ -18,6 +18,7 @@ import com.google.refine.importing.ImportingUtilities; import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.tests.importers.ImporterTest; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; public class ImportingUtilitiesTests extends ImporterTest { @@ -30,7 +31,7 @@ public class ImportingUtilitiesTests extends ImporterTest { @Test public void createProjectMetadataTest() throws Exception { - JSONObject optionObj = new JSONObject( + ObjectNode optionObj = ParsingUtilities.evaluateJsonStringToObjectNode( "{\"projectName\":\"acme\",\"projectTags\":[],\"created\":\"2017-12-18T13:28:40.659\",\"modified\":\"2017-12-20T09:28:06.654\",\"creator\":\"\",\"contributors\":\"\",\"subject\":\"\",\"description\":\"\",\"rowCount\":50,\"customMetadata\":{}}"); ProjectMetadata pm = ImportingUtilities.createProjectMetadata(optionObj); Assert.assertEquals(pm.getName(), "acme"); @@ -60,14 +61,13 @@ public class ImportingUtilitiesTests extends ImporterTest { Assert.assertTrue(project.columnModel.getColumnByName("result - person - sexe").getType().equals("boolean")); } - private JSONObject getNestedOptions(ImportingJob job, TreeImportingParserBase parser) { - JSONObject options = parser.createParserUIInitializationData( - job, new LinkedList(), "text/json"); + private ObjectNode getNestedOptions(ImportingJob job, TreeImportingParserBase parser) { + ObjectNode options = parser.createParserUIInitializationData( + job, new LinkedList<>(), "text/json"); - JSONArray path = new JSONArray(); - JSONUtilities.append(path, "results"); - JSONUtilities.append(path, "result"); -// JSONUtilities.append(path, "object"); + ArrayNode path = ParsingUtilities.mapper.createArrayNode(); + path.add("results"); + path.add("result"); JSONUtilities.safePut(options, "recordPath", path); return options; diff --git a/main/tests/server/src/com/google/refine/tests/model/CacheTests.java b/main/tests/server/src/com/google/refine/tests/model/CacheTests.java index 4cf52dc52..cccacf68f 100644 --- a/main/tests/server/src/com/google/refine/tests/model/CacheTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/CacheTests.java @@ -53,7 +53,6 @@ import com.google.refine.model.Cell; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; -import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.row.RowRemovalOperation; import com.google.refine.tests.RefineTest; diff --git a/main/tests/server/src/com/google/refine/tests/model/ColumnTests.java b/main/tests/server/src/com/google/refine/tests/model/ColumnTests.java index 162483a24..1a5c95d6a 100644 --- a/main/tests/server/src/com/google/refine/tests/model/ColumnTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/ColumnTests.java @@ -1,5 +1,8 @@ package com.google.refine.tests.model; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import org.testng.annotations.Test; import com.google.refine.model.Column; @@ -8,8 +11,6 @@ import com.google.refine.model.recon.StandardReconConfig; import com.google.refine.tests.util.TestUtils; import edu.mit.simile.butterfly.ButterflyModule; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class ColumnTests { @Test diff --git a/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java b/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java index b0b631340..1bb8a6137 100644 --- a/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java @@ -12,12 +12,12 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.google.refine.history.Change; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.changes.CellAtRow; import com.google.refine.model.changes.ColumnAdditionChange; import com.google.refine.model.changes.MassChange; -import com.google.refine.history.Change; import com.google.refine.tests.RefineTest; public class MassChangeTests extends RefineTest { diff --git a/main/tests/server/src/com/google/refine/tests/operations/cell/KeyValueColumnizeTests.java b/main/tests/server/src/com/google/refine/tests/operations/cell/KeyValueColumnizeTests.java index 0ebeadb82..a80bca841 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/cell/KeyValueColumnizeTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/cell/KeyValueColumnizeTests.java @@ -43,7 +43,6 @@ import java.util.List; import java.util.Properties; import org.json.JSONException; -import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -51,6 +50,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.ProjectManager; import com.google.refine.RefineServlet; import com.google.refine.importers.SeparatorBasedImporter; @@ -75,7 +75,7 @@ public class KeyValueColumnizeTests extends RefineTest { private RefineServlet servlet; private Project project; private ProjectMetadata pm; - private JSONObject options; + private ObjectNode options; private ImportingJob job; private SeparatorBasedImporter importer; @@ -95,7 +95,7 @@ public class KeyValueColumnizeTests extends RefineTest { pm = new ProjectMetadata(); pm.setName("KeyValueColumnize test"); ProjectManager.singleton.registerProject(project, pm); - options = mock(JSONObject.class); + options = mock(ObjectNode.class); OperationRegistry.registerOperation(getCoreModule(), "key-value-columnize", KeyValueColumnizeOperation.class); ImportingManager.initialize(servlet); diff --git a/main/tests/server/src/com/google/refine/tests/operations/cell/MassOperationTests.java b/main/tests/server/src/com/google/refine/tests/operations/cell/MassOperationTests.java index d604fedc3..629e1eafb 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/cell/MassOperationTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/cell/MassOperationTests.java @@ -8,7 +8,6 @@ import org.testng.annotations.BeforeSuite; import org.testng.annotations.Test; import com.fasterxml.jackson.core.type.TypeReference; - import com.google.refine.operations.OperationRegistry; import com.google.refine.operations.cell.MassEditOperation; import com.google.refine.operations.cell.MassEditOperation.Edit; diff --git a/main/tests/server/src/com/google/refine/tests/process/QuickHistoryEntryProcessTests.java b/main/tests/server/src/com/google/refine/tests/process/QuickHistoryEntryProcessTests.java index 538eb3b01..238c35298 100644 --- a/main/tests/server/src/com/google/refine/tests/process/QuickHistoryEntryProcessTests.java +++ b/main/tests/server/src/com/google/refine/tests/process/QuickHistoryEntryProcessTests.java @@ -1,13 +1,14 @@ package com.google.refine.tests.process; import static org.mockito.Mockito.mock; + import org.testng.annotations.Test; import com.google.refine.history.HistoryEntry; import com.google.refine.model.Project; +import com.google.refine.process.Process; import com.google.refine.process.QuickHistoryEntryProcess; import com.google.refine.tests.util.TestUtils; -import com.google.refine.process.Process; public class QuickHistoryEntryProcessTests { public static class QuickHistoryEntryProcessStub extends QuickHistoryEntryProcess { diff --git a/main/tests/server/src/com/google/refine/tests/util/PatternSyntaxExceptionParserTests.java b/main/tests/server/src/com/google/refine/tests/util/PatternSyntaxExceptionParserTests.java index f42dcabc0..b87e32e40 100644 --- a/main/tests/server/src/com/google/refine/tests/util/PatternSyntaxExceptionParserTests.java +++ b/main/tests/server/src/com/google/refine/tests/util/PatternSyntaxExceptionParserTests.java @@ -33,14 +33,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.util; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; - import com.google.refine.tests.RefineTest; import com.google.refine.util.PatternSyntaxExceptionParser; diff --git a/main/tests/server/src/com/google/refine/tests/util/TestUtils.java b/main/tests/server/src/com/google/refine/tests/util/TestUtils.java index f35f20184..64bc6bd8d 100644 --- a/main/tests/server/src/com/google/refine/tests/util/TestUtils.java +++ b/main/tests/server/src/com/google/refine/tests/util/TestUtils.java @@ -17,7 +17,6 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; - import com.google.refine.util.ParsingUtilities;