s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove Jsonizable interface and write methods

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-10-03 21:33:37 +01:00
parent e99a491338
commit 1790799bfe
135 changed files with 508 additions and 2900 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/database/src/com/google/refine/extension/database/DatabaseModuleImpl.java
View File

@ -40,7 +40,6 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.Jsonizable;
import edu.mit.simile.butterfly.ButterflyModuleImpl;

92
extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java
View File

@ -27,31 +27,55 @@ package org.openrefine.wikidata.commands;
import static org.openrefine.wikidata.commands.CommandUtilities.respondError;
import java.io.IOException;
import java.io.StringWriter;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONWriter;
import org.openrefine.wikidata.qa.EditInspector;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.qa.QAWarning.Severity;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.browsing.Engine;
import com.google.refine.commands.Command;
import com.google.refine.model.Project;
public class PreviewWikibaseSchemaCommand extends Command {
protected static class PreviewResults {
@JsonProperty("warnings")
List<QAWarning> warnings;
@JsonProperty("max_severity")
Severity maxSeverity;
@JsonProperty("nb_warnings")
int nbWarnings;
@JsonProperty("edit_count")
int editCount;
@JsonProperty("edits_preview")
List<ItemUpdate> editsPreview;
protected PreviewResults(
List<QAWarning> warnings,
Severity maxSeverity,
int nbWarnings,
int editCount,
List<ItemUpdate> editsPreview) {
this.warnings = warnings;
this.maxSeverity = maxSeverity;
this.nbWarnings = nbWarnings;
this.editCount = editCount;
this.editsPreview = editsPreview;
}
}
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
@ -87,50 +111,24 @@ public class PreviewWikibaseSchemaCommand extends Command {
Engine engine = getEngine(request, project);
List<ItemUpdate> editBatch = schema.evaluate(project, engine, warningStore);
StringWriter sb = new StringWriter(2048);
JSONWriter writer = new JSONWriter(sb);
writer.object();
// Inspect the edits and generate warnings
EditInspector inspector = new EditInspector(warningStore);
inspector.inspect(editBatch);
// Dump the first 10 edits, scheduled with the default scheduler
WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler();
List<ItemUpdate> nonNullEdits = scheduler.schedule(editBatch).stream()
.filter(e -> !e.isNull())
.collect(Collectors.toList());
List<ItemUpdate> firstEdits = nonNullEdits.stream()
.limit(10)
.collect(Collectors.toList());
{
// Inspect the edits and generate warnings
EditInspector inspector = new EditInspector(warningStore);
inspector.inspect(editBatch);
writer.key("warnings");
writer.array();
for (QAWarning warning : warningStore.getWarnings()) {
warning.write(writer, new Properties());
}
writer.endArray();
// Add max warning level
writer.key("max_severity");
writer.value(warningStore.getMaxSeverity().toString());
// this is not the length of the warnings array written before,
// but the total number of issues raised (before deduplication)
writer.key("nb_warnings");
writer.value(warningStore.getNbWarnings());
// Dump the first 10 edits, scheduled with the default scheduler
WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler();
List<ItemUpdate> nonNullEdits = scheduler.schedule(editBatch).stream()
.filter(e -> !e.isNull())
.collect(Collectors.toList());
writer.key("edit_count");
writer.value(nonNullEdits.size());
List<ItemUpdate> firstEdits = nonNullEdits.stream()
.limit(10)
.collect(Collectors.toList());
ObjectMapper mapper = new ObjectMapper();
String firstEditsJson = mapper.writeValueAsString(firstEdits);
writer.key("edits_preview");
writer.value(new JSONArray(firstEditsJson));
}
writer.endObject();
respond(response, sb.toString());
respondJSON(response, new PreviewResults(
warningStore.getWarnings(),
warningStore.getMaxSeverity(),
warningStore.getNbWarnings(),
nonNullEdits.size(), firstEdits));
} catch (Exception e) {
respondException(response, e);
}

3
extensions/wikidata/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommand.java
View File

@ -23,6 +23,8 @@
******************************************************************************/
package org.openrefine.wikidata.commands;
import static org.openrefine.wikidata.commands.CommandUtilities.respondError;
import java.io.IOException;
import java.util.Properties;
@ -34,7 +36,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation;
import org.openrefine.wikidata.schema.WikibaseSchema;
import static org.openrefine.wikidata.commands.CommandUtilities.respondError;
import com.google.refine.commands.Command;
import com.google.refine.model.AbstractOperation;

26
extensions/wikidata/src/org/openrefine/wikidata/editing/WikibaseCredentials.java
View File

@ -23,13 +23,11 @@
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* This is just the necessary bits to store Wikidata credentials in OpenRefine's
@ -38,9 +36,11 @@ import com.google.refine.Jsonizable;
* @author Antonin Delpeuch
*
*/
class WikibaseCredentials implements Jsonizable {
class WikibaseCredentials {
@JsonProperty("username")
private String username;
@JsonProperty("password")
private String password;
public WikibaseCredentials() {
@ -64,18 +64,10 @@ class WikibaseCredentials implements Jsonizable {
public boolean isNonNull() {
return username != null && password != null && !"null".equals(username) && !"null".equals(password);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("class");
writer.value(this.getClass().getName());
writer.key("username");
writer.value(username);
writer.key("password");
writer.value(password);
writer.endObject();
@JsonProperty("class")
public String getClassName() {
return getClass().getName();
}
public static WikibaseCredentials load(JSONObject obj)

17
extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java
View File

@ -46,6 +46,7 @@ import org.wikidata.wdtk.wikibaseapi.ApiConnection;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.browsing.Engine;
@ -64,6 +65,7 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
static final Logger logger = LoggerFactory.getLogger(PerformWikibaseEditsOperation.class);
@JsonProperty("summary")
private String summary;
public PerformWikibaseEditsOperation(EngineConfig engineConfig, String summary) {
@ -84,21 +86,6 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
EngineConfig.reconstruct(engineConfig), summary);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op");
writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description");
writer.value("Perform Wikibase edits");
writer.key("summary");
writer.value(summary);
writer.key("engineConfig");
getEngineConfig().write(writer, options);
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Peform edits on Wikidata";

25
extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java
View File

@ -33,6 +33,8 @@ import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.schema.WikibaseSchema;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
@ -44,6 +46,7 @@ import com.google.refine.util.Pool;
public class SaveWikibaseSchemaOperation extends AbstractOperation {
final public String operationDescription = "Save Wikibase schema";
@JsonProperty("schema")
final protected WikibaseSchema _schema;
public SaveWikibaseSchemaOperation(WikibaseSchema schema) {
@ -56,22 +59,9 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
return new SaveWikibaseSchemaOperation(WikibaseSchema.reconstruct(obj.getJSONObject("schema")));
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op");
writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description");
writer.value(operationDescription);
writer.key("schema");
_schema.write(writer, options);
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Save Wikibase schema skelton";
return operationDescription;
}
@Override
@ -149,12 +139,7 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
static protected void writeWikibaseSchema(WikibaseSchema s, Writer writer)
throws IOException {
if (s != null) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
s.write(jsonWriter, new Properties());
} catch (JSONException e) {
e.printStackTrace();
}
ParsingUtilities.defaultWriter.writeValue(writer, s);
}
}
}

3
extensions/wikidata/src/org/openrefine/wikidata/qa/QAWarning.java
View File

@ -28,7 +28,6 @@ import java.util.Map;
import java.util.Map.Entry;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.utils.JacksonJsonizable;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
@ -42,7 +41,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
* @author Antonin Delpeuch
*
*/
public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning> {
public class QAWarning implements Comparable<QAWarning> {
public enum Severity {
INFO, // We just report something to the user but it is probably fine

3
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java
View File

@ -30,7 +30,6 @@ import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.openrefine.wikidata.utils.JacksonJsonizable;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
@ -48,7 +47,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
public class WbItemDocumentExpr extends JacksonJsonizable implements WbExpression<ItemUpdate> {
public class WbItemDocumentExpr implements WbExpression<ItemUpdate> {
private WbExpression<? extends ItemIdValue> subject;
private List<WbNameDescExpr> nameDescs;

14
extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java
View File

@ -63,6 +63,7 @@ public class WikibaseSchema implements OverlayModel {
final static Logger logger = LoggerFactory.getLogger("RdfSchema");
@JsonProperty("itemDocuments")
protected List<WbItemDocumentExpr> itemDocumentExprs = new ArrayList<WbItemDocumentExpr>();
protected String baseIri = "http://www.wikidata.org/entity/";
@ -198,19 +199,6 @@ public class WikibaseSchema implements OverlayModel {
throw new JSONException(e.toString());
}
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("itemDocuments");
writer.array();
for (WbItemDocumentExpr changeExpr : itemDocumentExprs) {
changeExpr.write(writer, options);
}
writer.endArray();
writer.endObject();
}
static public WikibaseSchema load(Project project, JSONObject obj)
throws Exception {

79
extensions/wikidata/src/org/openrefine/wikidata/utils/JacksonJsonizable.java
View File

@ -1,79 +0,0 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.utils;
import java.io.IOException;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.Jsonizable;
/**
* This class is inefficient because it serializes the object to string and then
* deserializes it back. Unfortunately, this is the only simple way to bridge
* Jackson to org.json. This conversion should be removed when (if ?) we migrate
* OpenRefine a better JSON library.
*
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public abstract class JacksonJsonizable implements Jsonizable {
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
ObjectMapper mapper = new ObjectMapper();
try {
writer.value(new JSONObject(mapper.writeValueAsString(this)));
} catch (JsonProcessingException e) {
throw new JSONException(e.toString());
}
}
public static <T> T fromJSONClass(JSONObject obj, Class<T> klass)
throws JSONException {
ObjectMapper mapper = new ObjectMapper();
String json = obj.toString();
try {
return mapper.readValue(json, klass);
} catch (JsonParseException e) {
throw new JSONException(e.toString());
} catch (JsonMappingException e) {
throw new JSONException(e.toString());
} catch (IOException e) {
throw new JSONException(e.toString());
}
}
}

5
extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java
View File

@ -33,7 +33,6 @@ import java.io.StringWriter;
import java.util.Properties;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@ -43,6 +42,7 @@ import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.tests.RefineTest;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
import edu.mit.simile.butterfly.ButterflyModule;
@ -77,8 +77,7 @@ public abstract class OperationTest extends RefineTest {
JSONObject json = getJson();
AbstractOperation op = reconstruct();
StringWriter writer = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(writer);
op.write(jsonWriter, new Properties());
ParsingUtilities.defaultWriter.writeValue(writer, op);
JacksonSerializationTest.assertJsonEquals(json.toString(), writer.toString());
}

14
extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java
View File

@ -26,16 +26,13 @@ package org.openrefine.wikidata.schema;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.testing.TestingData;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder;
@ -58,7 +55,7 @@ import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.model.Project;
import com.google.refine.tests.RefineTest;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.tests.util.TestUtils;
public class WikibaseSchemaTest extends RefineTest {
@ -103,14 +100,7 @@ public class WikibaseSchemaTest extends RefineTest {
throws JSONException, IOException {
JSONObject serialized = TestingData.jsonFromFile("schema/history_of_medicine.json");
WikibaseSchema parsed = WikibaseSchema.reconstruct(serialized);
StringWriter writer = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(writer);
parsed.write(jsonWriter, new Properties());
writer.close();
JSONObject newSerialized = ParsingUtilities.evaluateJsonStringToObject(writer.toString());
// toString because it looks like JSONObject equality isn't great…
assertEquals(TestingData.jsonFromFile("schema/history_of_medicine_normalized.json").toString(),
newSerialized.toString());
TestUtils.isSerializedTo(parsed, TestingData.jsonFromFile("data/schema/history_of_medicine_normalized.json").toString());
}
@Test

49
main/src/com/google/refine/Jsonizable.java
View File

@ -1,49 +0,0 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
/**
* Interface for streaming out JSON, either into HTTP responses or
* serialization files.
*
* @author dfhuynh
*/
public interface Jsonizable {
public void write(JSONWriter writer, Properties options) throws JSONException;
}

16
main/src/com/google/refine/browsing/DecoratedValue.java
View File

@ -34,14 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.browsing;
import java.time.OffsetDateTime;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.util.StringUtils;
/**
@ -51,7 +46,7 @@ import com.google.refine.util.StringUtils;
*
* Facet choices that are presented to the user as text are stored as decorated values.
*/
public class DecoratedValue implements Jsonizable {
public class DecoratedValue {
@JsonProperty("v")
final public Object value;
@JsonProperty("l")
@ -65,13 +60,4 @@ public class DecoratedValue implements Jsonizable {
}
this.label = label;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("v"); writer.value(value);
writer.key("l"); writer.value(label);
writer.endObject();
}
}

20
main/src/com/google/refine/browsing/Engine.java
View File

@ -36,17 +36,14 @@ package com.google.refine.browsing;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import com.google.refine.Jsonizable;
import com.google.refine.browsing.facets.Facet;
import com.google.refine.browsing.util.ConjunctiveFilteredRecords;
import com.google.refine.browsing.util.ConjunctiveFilteredRows;
@ -57,7 +54,7 @@ import com.google.refine.model.Row;
/**
* Faceted browsing engine.
*/
public class Engine implements Jsonizable {
public class Engine {
static public enum Mode {
@JsonProperty("row-based")
RowBased,
@ -205,19 +202,4 @@ public class Engine implements Jsonizable {
throw new InternalError("Unknown mode.");
}
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("facets");
writer.array();
for (Facet facet : _facets) {
facet.write(writer, options);
}
writer.endArray();
writer.key(MODE); writer.value(_config.getMode().equals(Mode.RowBased) ? MODE_ROW_BASED : MODE_RECORD_BASED);
writer.endObject();
}
}

21
main/src/com/google/refine/browsing/EngineConfig.java
View File

@ -3,16 +3,12 @@ package com.google.refine.browsing;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.browsing.Engine.Mode;
import com.google.refine.browsing.facets.FacetConfig;
import com.google.refine.browsing.facets.ListFacet.ListFacetConfig;
@ -22,7 +18,7 @@ import com.google.refine.browsing.facets.TextSearchFacet.TextSearchFacetConfig;
import com.google.refine.browsing.facets.TimeRangeFacet.TimeRangeFacetConfig;
public class EngineConfig implements Jsonizable {
public class EngineConfig {
protected final List<FacetConfig> _facets;
protected final Mode _mode;
@ -88,19 +84,4 @@ public class EngineConfig implements Jsonizable {
return new EngineConfig(facets, mode);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("facets");
writer.array();
for (FacetConfig facet : _facets) {
facet.write(writer, options);
}
writer.endArray();
writer.key(Engine.MODE); writer.value(_mode == Mode.RowBased ? Engine.MODE_ROW_BASED : Engine.MODE_RECORD_BASED);
writer.endObject();
}
}

3
main/src/com/google/refine/browsing/facets/Facet.java
View File

@ -33,7 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.browsing.facets;
import com.google.refine.Jsonizable;
import com.google.refine.browsing.FilteredRecords;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RecordFilter;
@ -43,7 +42,7 @@ import com.google.refine.model.Project;
/**
* Interface of facets.
*/
public interface Facet extends Jsonizable {
public interface Facet {
public RowFilter getRowFilter(Project project);
public RecordFilter getRecordFilter(Project project);

3
main/src/com/google/refine/browsing/facets/FacetConfig.java
View File

@ -4,7 +4,6 @@ import org.json.JSONObject;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.model.Project;
@ -17,7 +16,7 @@ import com.google.refine.model.Project;
* @author antonin
*
*/
public interface FacetConfig extends Jsonizable {
public interface FacetConfig {
/**
* Reads the facet configuration from a JSON object (will be removed once we migrate to Jackson)
* @param fo

36
main/src/com/google/refine/browsing/facets/ListFacet.java
View File

@ -35,20 +35,16 @@ package com.google.refine.browsing.facets;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.DecoratedValue;
@ -115,33 +111,6 @@ public class ListFacet implements Facet {
@JsonProperty("selectError")
public boolean selectError;
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("type"); writer.value("list");
writer.key("name"); writer.value(name);
writer.key("expression"); writer.value(expression);
writer.key("columnName"); writer.value(columnName);
writer.key("invert"); writer.value(invert);
writer.key("selection"); writer.array();
for (DecoratedValue choice : selection) {
writer.object();
writer.key("v");
choice.write(writer, options);
writer.endObject();
}
writer.endArray();
writer.key("selectNumber"); writer.value(selectNumber);
writer.key("selectDateTime"); writer.value(selectDateTime);
writer.key("selectBoolean"); writer.value(selectBoolean);
writer.key("omitBlank"); writer.value(omitBlank);
writer.key("selectBlank"); writer.value(selectBlank);
writer.key("omitError"); writer.value(omitError);
writer.key("selectError"); writer.value(selectError);
writer.endObject();
}
@JsonProperty("selection")
public List<DecoratedValueWrapper> getWrappedSelection() {
return selection.stream()
@ -292,6 +261,7 @@ public class ListFacet implements Facet {
}
return null;
}
<<<<<<< HEAD
@Override
public void write(JSONWriter writer, Properties options)
@ -353,6 +323,8 @@ public class ListFacet implements Facet {
writer.endObject();
}
=======
>>>>>>> Remove Jsonizable interface and write methods
protected int getLimit() {
Object v = ProjectManager.singleton.getPreferenceStore().get("ui.browsing.listFacet.limit");

18
main/src/com/google/refine/browsing/facets/NominalFacetChoice.java
View File

@ -33,21 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.browsing.facets;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.browsing.DecoratedValue;
/**
* Store a facet choice that has a decorated value, a count of matched rows,
* and a flag of whether it has been selected.
*/
public class NominalFacetChoice implements Jsonizable {
public class NominalFacetChoice {
@JsonProperty("v")
final public DecoratedValue decoratedValue;
@JsonProperty("c")
@ -58,14 +52,4 @@ public class NominalFacetChoice implements Jsonizable {
public NominalFacetChoice(DecoratedValue decoratedValue) {
this.decoratedValue = decoratedValue;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("v"); decoratedValue.write(writer, options);
writer.key("c"); writer.value(count);
writer.key("s"); writer.value(selected);
writer.endObject();
}
}

74
main/src/com/google/refine/browsing/facets/RangeFacet.java
View File

@ -33,11 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.browsing.facets;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
@ -96,24 +92,6 @@ public class RangeFacet implements Facet {
protected boolean _selected; // false if we're certain that all rows will match
// and there isn't any filtering to do
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("type"); writer.value("range");
writer.key("name"); writer.value(_name);
writer.key("expression"); writer.value(_expression);
writer.key("columnName"); writer.value(_columnName);
writer.key(FROM); writer.value(_from);
writer.key(TO); writer.value(_to);
writer.key("selectNumeric"); writer.value(_selectNumeric);
writer.key("selectNonNumeric"); writer.value(_selectNonNumeric);
writer.key("selectError"); writer.value(_selectError);
writer.key("selectBlank"); writer.value(_selectBlank);
writer.endObject();
}
@Override
public void initializeFromJSON(JSONObject o) {
_name = o.getString("name");
@ -282,56 +260,8 @@ public class RangeFacet implements Facet {
return _config._to;
}
return null;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("name"); writer.value(_config._name);
writer.key("expression"); writer.value(_config._expression);
writer.key("columnName"); writer.value(_config._columnName);
if (_errorMessage != null) {
writer.key("error"); writer.value(_errorMessage);
} else {
if (!Double.isInfinite(_min) && !Double.isInfinite(_max)) {
writer.key(MIN); writer.value(_min);
writer.key(MAX); writer.value(_max);
writer.key("step"); writer.value(_step);
writer.key("bins"); writer.array();
for (int b : _bins) {
writer.value(b);
}
writer.endArray();
writer.key("baseBins"); writer.array();
for (int b : _baseBins) {
writer.value(b);
}
writer.endArray();
writer.key(FROM); writer.value(_config._from);
writer.key(TO); writer.value(_config._to);
} else {
writer.key("error"); writer.value(ERR_NO_NUMERIC_VALUE_PRESENT);
}
writer.key("baseNumericCount"); writer.value(_baseNumericCount);
writer.key("baseNonNumericCount"); writer.value(_baseNonNumericCount);
writer.key("baseBlankCount"); writer.value(_baseBlankCount);
writer.key("baseErrorCount"); writer.value(_baseErrorCount);
writer.key("numericCount"); writer.value(_numericCount);
writer.key("nonNumericCount"); writer.value(_nonNumericCount);
writer.key("blankCount"); writer.value(_blankCount);
writer.key("errorCount"); writer.value(_errorCount);
}
writer.endObject();
}
}
public void initializeFromConfig(RangeFacetConfig config, Project project) {
_config = config;

75
main/src/com/google/refine/browsing/facets/ScatterplotFacet.java
View File

@ -40,14 +40,11 @@ import java.awt.image.BufferedImage;
import java.awt.image.RenderedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Properties;
import javax.imageio.ImageIO;
import org.apache.commons.codec.binary.Base64;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -140,36 +137,6 @@ public class ScatterplotFacet implements Facet {
return dim_y == LIN ? "lin" : "log";
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("type"); writer.value("scatterplot");
writer.key(NAME); writer.value(name);
writer.key(X_COLUMN_NAME); writer.value(columnName_x);
writer.key(X_EXPRESSION); writer.value(expression_x);
writer.key(Y_COLUMN_NAME); writer.value(columnName_y);
writer.key(Y_EXPRESSION); writer.value(expression_y);
writer.key(SIZE); writer.value(size);
writer.key(DOT); writer.value(dot);
if(!rotation_str.isEmpty()) {
writer.key(ROTATION); writer.value(rotation_str);
}
writer.key(DIM_X); writer.value(dim_x == LIN ? "lin" : "log");
writer.key(DIM_Y); writer.value(dim_y == LIN ? "lin" : "log");
if(!"000000".equals(color_str)) {
writer.key(COLOR); writer.value(color_str);
}
writer.key(FROM_X); writer.value(from_x);
writer.key(TO_X); writer.value(to_x);
writer.key(FROM_Y); writer.value(from_y);
writer.key(TO_Y); writer.value(to_y);
writer.endObject();
}
@Override
public ScatterplotFacet apply(Project project) {
ScatterplotFacet facet = new ScatterplotFacet();
@ -404,48 +371,6 @@ public class ScatterplotFacet implements Facet {
}
return null;
}
@Override
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.object();
writer.key(NAME); writer.value(config.name);
writer.key(X_COLUMN_NAME); writer.value(config.columnName_x);
writer.key(X_EXPRESSION); writer.value(config.expression_x);
writer.key(Y_COLUMN_NAME); writer.value(config.columnName_y);
writer.key(Y_EXPRESSION); writer.value(config.expression_y);
writer.key(SIZE); writer.value(config.size);
writer.key(DOT); writer.value(config.dot);
writer.key(ROTATION); writer.value(config.rotation);
writer.key(DIM_X); writer.value(config.dim_x);
writer.key(DIM_Y); writer.value(config.dim_y);
writer.key(COLOR); writer.value(config.color_str);
if (IMAGE_URI) {
writer.key(IMAGE); writer.value(image);
}
if (errorMessage_x != null) {
writer.key(ERROR_X); writer.value(errorMessage_x);
} else {
if (!Double.isInfinite(min_x) && !Double.isInfinite(max_x)) {
writer.key(FROM_X); writer.value(config.from_x);
writer.key(TO_X); writer.value(config.to_x);
}
}
if (errorMessage_y != null) {
writer.key(ERROR_Y); writer.value(errorMessage_y);
} else {
if (!Double.isInfinite(min_y) && !Double.isInfinite(max_y)) {
writer.key(FROM_Y); writer.value(config.from_y);
writer.key(TO_Y); writer.value(config.to_y);
}
}
writer.endObject();
}
public void initializeFromConfig(ScatterplotFacetConfig configuration, Project project) {
config = configuration;

32
main/src/com/google/refine/browsing/facets/TextSearchFacet.java
View File

@ -33,12 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.browsing.facets;
import java.util.Properties;
import java.util.regex.Pattern;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -73,20 +71,6 @@ public class TextSearchFacet implements Facet {
@JsonProperty("invert")
protected boolean _invert;
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("name"); writer.value(_name);
writer.key("columnName"); writer.value(_columnName);
writer.key("query"); writer.value(_query);
writer.key("mode"); writer.value(_mode);
writer.key("caseSensitive"); writer.value(_caseSensitive);
writer.key("invert"); writer.value(_invert);
writer.key("type"); writer.value("text");
writer.endObject();
}
@Override
public TextSearchFacet apply(Project project) {
TextSearchFacet facet = new TextSearchFacet();
@ -152,21 +136,7 @@ public class TextSearchFacet implements Facet {
public boolean isInverted() {
return _config._invert;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("name"); writer.value(_config._name);
writer.key("columnName"); writer.value(_config._columnName);
writer.key("query"); writer.value(_config._query);
writer.key("mode"); writer.value(_config._mode);
writer.key("caseSensitive"); writer.value(_config._caseSensitive);
writer.key("invert"); writer.value(_config._invert);
writer.endObject();
}
public void initializeFromConfig(TextSearchFacetConfig config, Project project) {
_config = config;

65
main/src/com/google/refine/browsing/facets/TimeRangeFacet.java
View File

@ -33,11 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.browsing.facets;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
@ -93,23 +90,6 @@ public class TimeRangeFacet implements Facet {
protected boolean _selected; // false if we're certain that all rows will match
// and there isn't any filtering to do
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("type"); writer.value("timerange");
writer.key("name"); writer.value(_name);
writer.key("expression"); writer.value(_expression);
writer.key("columnName"); writer.value(_columnName);
writer.key("selectTime"); writer.value(_selectTime);
writer.key("selectNonTime"); writer.value(_selectNonTime);
writer.key("selectBlank"); writer.value(_selectBlank);
writer.key("selectError"); writer.value(_selectError);
writer.key(FROM); writer.value((long)_from);
writer.key(TO); writer.value((long)_to);
writer.endObject();
}
@Override
public void initializeFromJSON(JSONObject o) throws JSONException {
_name = o.getString("name");
@ -266,51 +246,6 @@ public class TimeRangeFacet implements Facet {
return null;
}
@Override
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.object();
writer.key("name"); writer.value(_config._name);
writer.key("expression"); writer.value(_config._expression);
writer.key("columnName"); writer.value(_config._columnName);
if (_errorMessage != null) {
writer.key("error"); writer.value(_errorMessage);
} else {
if (!Double.isInfinite(_min) && !Double.isInfinite(_max)) {
writer.key(MIN); writer.value((long)_min);
writer.key(MAX); writer.value((long)_max);
writer.key("step"); writer.value((long)_step);
writer.key("bins"); writer.array();
for (int b : _bins) {
writer.value(b);
}
writer.endArray();
writer.key("baseBins"); writer.array();
for (int b : _baseBins) {
writer.value(b);
}
writer.endArray();
writer.key(FROM); writer.value((long)_config._from);
writer.key(TO); writer.value((long)_config._to);
}
writer.key("baseTimeCount"); writer.value(_baseTimeCount);
writer.key("baseNonTimeCount"); writer.value(_baseNonTimeCount);
writer.key("baseBlankCount"); writer.value(_baseBlankCount);
writer.key("baseErrorCount"); writer.value(_baseErrorCount);
writer.key("timeCount"); writer.value(_timeCount);
writer.key("nonTimeCount"); writer.value(_nonTimeCount);
writer.key("blankCount"); writer.value(_blankCount);
writer.key("errorCount"); writer.value(_errorCount);
}
writer.endObject();
}
public void initializeFromConfig(TimeRangeFacetConfig config, Project project) {
_config = config;
if (_config._columnName.length() > 0) {

3
main/src/com/google/refine/clustering/Clusterer.java
View File

@ -33,12 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.clustering;
import com.google.refine.Jsonizable;
import com.google.refine.browsing.Engine;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
public abstract class Clusterer implements Jsonizable {
public abstract class Clusterer {
protected Project _project;
protected int _colindex;

3
main/src/com/google/refine/clustering/ClustererConfig.java
View File

@ -4,7 +4,6 @@ import org.json.JSONObject;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.model.Project;
/**
@ -12,7 +11,7 @@ import com.google.refine.model.Project;
* @author Antonin Delpeuch
*
*/
public abstract class ClustererConfig implements Jsonizable {
public abstract class ClustererConfig {
protected String columnName;

50
main/src/com/google/refine/clustering/binning/BinningClusterer.java
View File

@ -57,7 +57,6 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import com.google.refine.Jsonizable;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
@ -99,20 +98,6 @@ public class BinningClusterer extends Clusterer {
public BinningParameters getParameters() {
return _parameters;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("function"); writer.value(_keyerName);
writer.key("type"); writer.value("binning");
writer.key("column"); writer.value(getColumnName());
if(_parameters != null) {
writer.key("params");
_parameters.write(writer, options);
}
writer.endObject();
}
@Override
public BinningClusterer apply(Project project) {
@ -128,21 +113,10 @@ public class BinningClusterer extends Clusterer {
}
public static class BinningParameters implements Jsonizable {
public static class BinningParameters {
@JsonProperty("ngram-size")
@JsonInclude(Include.NON_DEFAULT)
public int ngramSize;
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
if(ngramSize > 0) {
writer.key("ngram-size");
writer.value(ngramSize);
}
writer.endObject();
}
public static BinningParameters reconstruct(JSONObject o) {
BinningParameters parameters = new BinningParameters();
@ -274,28 +248,6 @@ public class BinningClusterer extends Clusterer {
Collections.sort(_clusters, new SizeComparator());
}
@Override
public void write(JSONWriter writer, Properties options) throws JSONException {
EntriesComparator c = new EntriesComparator();
writer.array();
for (Map<String,Integer> m : _clusters) {
if (m.size() > 1) {
writer.array();
List<Entry<String,Integer>> entries = new ArrayList<Entry<String,Integer>>(m.entrySet());
Collections.sort(entries,c);
for (Entry<String,Integer> e : entries) {
writer.object();
writer.key("v"); writer.value(e.getKey());
writer.key("c"); writer.value(e.getValue());
writer.endObject();
}
writer.endArray();
}
}
writer.endArray();
}
protected static Map<String,Object> entryToMap(Entry<String,Integer> entry) {
Map<String,Object> map = new HashMap<>();
map.put("v", entry.getKey());

58
main/src/com/google/refine/clustering/knn/kNNClusterer.java
View File

@ -34,21 +34,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.clustering.knn;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -56,7 +51,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import com.google.refine.Jsonizable;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
@ -86,21 +80,7 @@ public class kNNClusterer extends Clusterer {
private String _distanceStr;
private Distance _distance;
private kNNClustererConfigParameters _parameters;
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("function"); writer.value(_distanceStr);
writer.key("type"); writer.value("knn");
writer.key("column"); writer.value(getColumnName());
if(_parameters != null) {
writer.key("params");
_parameters.write(writer, options);
}
writer.endObject();
}
public void initializeFromJSON(JSONObject o) {
super.initializeFromJSON(o);
_distanceStr = o.getString("function");
@ -136,7 +116,7 @@ public class kNNClusterer extends Clusterer {
}
public static class kNNClustererConfigParameters implements Jsonizable {
public static class kNNClustererConfigParameters {
public static final double defaultRadius = 1.0d;
public static final int defaultBlockingNgramSize = 6;
@JsonProperty("radius")
@ -144,16 +124,6 @@ public class kNNClusterer extends Clusterer {
@JsonProperty("blocking-ngram-size")
public int blockingNgramSize = defaultBlockingNgramSize;
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("radius"); writer.value(radius);
writer.key("blocking-ngram-size");
writer.value(blockingNgramSize);
writer.endObject();
}
public static kNNClustererConfigParameters reconstruct(JSONObject o) {
kNNClustererConfigParameters params = new kNNClustererConfigParameters();
if(o.has("radius")) {
@ -294,30 +264,6 @@ public class kNNClusterer extends Clusterer {
}
}
@Override
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.array();
for (Set<Serializable> m : _clusters) {
if (m.size() > 1) {
Map<Serializable,Integer> internal_counts = new HashMap<Serializable,Integer>();
for (Serializable s : m) {
internal_counts.put(s,_counts.get(s));
}
List<Entry<Serializable,Integer>> values = new ArrayList<Entry<Serializable,Integer>>(internal_counts.entrySet());
Collections.sort(values, new ValuesComparator());
writer.array();
for (Entry<Serializable,Integer> e : values) {
writer.object();
writer.key("v"); writer.value(e.getKey());
writer.key("c"); writer.value(e.getValue());
writer.endObject();
}
writer.endArray();
}
}
writer.endArray();
}
protected List<ClusteredEntry> getClusteredEntries(Set<Serializable> s) {
return s.stream()
.map(e -> new ClusteredEntry(e, _counts.get(e)))

33
main/src/com/google/refine/commands/Command.java
View File

@ -50,7 +50,8 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.Jsonizable;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet;
import com.google.refine.browsing.Engine;
@ -237,6 +238,16 @@ public abstract class Command {
}
return null;
}
protected static class HistoryEntryResponse {
@JsonProperty("code")
protected String getCode() { return "ok"; }
@JsonProperty("historyEntry")
protected HistoryEntry historyEntry;
protected HistoryEntryResponse(HistoryEntry entry) {
historyEntry = entry;
}
}
static protected void performProcessAndRespond(
HttpServletRequest request,
@ -244,23 +255,18 @@ public abstract class Command {
Project project,
Process process
) throws Exception {
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
HistoryEntry historyEntry = project.processManager.queueProcess(process);
if (historyEntry != null) {
Writer w = response.getWriter();
JSONWriter writer = new JSONWriter(w);
Properties options = new Properties();
writer.object();
writer.key("code"); writer.value("ok");
writer.key("historyEntry"); historyEntry.write(writer, options);
writer.endObject();
ParsingUtilities.defaultWriter.writeValue(w, new HistoryEntryResponse(historyEntry));
w.flush();
w.close();
} else {
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
respond(response, "{ \"code\" : \"pending\" }");
}
}
@ -293,14 +299,14 @@ public abstract class Command {
w.close();
}
static protected void respondJSON(HttpServletResponse response, Jsonizable o)
static protected void respondJSON(HttpServletResponse response, Object o)
throws IOException, JSONException {
respondJSON(response, o, new Properties());
}
static protected void respondJSON(
HttpServletResponse response, Jsonizable o, Properties options)
HttpServletResponse response, Object o, Properties options)
throws IOException, JSONException {
response.setCharacterEncoding("UTF-8");
@ -308,9 +314,8 @@ public abstract class Command {
response.setHeader("Cache-Control", "no-cache");
Writer w = response.getWriter();
JSONWriter writer = new JSONWriter(w);
ParsingUtilities.defaultWriter.writeValue(w, o);
o.write(writer, options);
w.flush();
w.close();
}

25
main/src/com/google/refine/commands/GetPreferenceCommand.java
View File

@ -34,47 +34,24 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.commands;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.ProjectManager;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.preference.TopList;
public class GetPreferenceCommand extends Command {
protected static class PreferenceValue implements Jsonizable {
protected static class PreferenceValue {
@JsonProperty("value")
protected Object pref;
protected PreferenceValue(Object o) {
pref = o;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("value");
if (pref == null || pref instanceof String || pref instanceof Number || pref instanceof Boolean) {
writer.value(pref);
} else if (pref instanceof TopList) {
TopList tl = (TopList) pref;
tl.write(writer, new Properties());
} else {
writer.value(pref.toString());
}
writer.endObject();
}
}
@Override

9
main/src/com/google/refine/commands/HttpUtilities.java
View File

@ -17,7 +17,6 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.Jsonizable;
import com.google.refine.RefineServlet;
import com.google.refine.util.ParsingUtilities;
@ -56,23 +55,21 @@ abstract public class HttpUtilities {
}
}
static public void respondJSON(HttpServletResponse response, Jsonizable o)
static public void respondJSON(HttpServletResponse response, Object o)
throws IOException, JSONException {
respondJSON(response, o, new Properties());
}
static public void respondJSON(
HttpServletResponse response, Jsonizable o, Properties options)
HttpServletResponse response, Object o, Properties options)
throws IOException, JSONException {
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
Writer w = response.getWriter();
JSONWriter writer = new JSONWriter(w);
o.write(writer, options);
ParsingUtilities.defaultWriter.writeValue(w, o);
w.flush();
w.close();
}

46
main/src/com/google/refine/commands/cell/EditOneCellCommand.java
View File

@ -35,13 +35,14 @@ package com.google.refine.commands.cell;
import java.io.IOException;
import java.io.Serializable;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.commands.Command;
import com.google.refine.history.Change;
@ -55,6 +56,32 @@ import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
public class EditOneCellCommand extends Command {
protected static class EditResult {
@JsonProperty("code")
protected String code;
@JsonProperty("historyEntry")
@JsonInclude(Include.NON_NULL)
protected HistoryEntry historyEntry;
@JsonProperty("cell")
@JsonInclude(Include.NON_NULL)
protected Cell cell;
@JsonProperty("pool")
@JsonInclude(Include.NON_NULL)
protected Pool pool;
protected EditResult(
String code,
HistoryEntry historyEntry,
Cell cell,
Pool pool) {
this.code = code;
this.historyEntry = historyEntry;
this.cell = cell;
this.pool = pool;
}
}
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
@ -96,18 +123,11 @@ public class EditOneCellCommand extends Command {
* If the operation has been done, return the new cell's data
* so the client side can update the cell's rendering right away.
*/
JSONWriter writer = new JSONWriter(response.getWriter());
Pool pool = new Pool();
Properties options = new Properties();
options.put("pool", pool);
writer.object();
writer.key("code"); writer.value("ok");
writer.key("historyEntry"); historyEntry.write(writer, options);
writer.key("cell"); process.newCell.write(writer, options);
writer.key("pool"); pool.write(writer, options);
writer.endObject();
if(process.newCell != null && process.newCell.recon != null) {
pool.pool(process.newCell.recon);
}
respondJSON(response, new EditResult("ok", historyEntry, process.newCell, pool));
} else {
respond(response, "{ \"code\" : \"pending\" }");
}

32
main/src/com/google/refine/commands/expr/GetExpressionHistoryCommand.java
View File

@ -37,7 +37,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
@ -45,19 +44,15 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
import com.google.refine.preference.TopList;
public class GetExpressionHistoryCommand extends Command {
public static class ExpressionState implements Jsonizable {
public static class ExpressionState {
@JsonProperty("code")
protected String code;
@JsonProperty("global")
@ -69,38 +64,15 @@ public class GetExpressionHistoryCommand extends Command {
this.code = code;
this.starred = starred;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("code"); writer.value(code);
writer.key("global"); writer.value(false);
writer.key("starred"); writer.value(starred);
writer.endObject();
}
}
public static class ExpressionsList implements Jsonizable {
public static class ExpressionsList {
@JsonProperty("expressions")
List<ExpressionState> expressions;
protected ExpressionsList(List<ExpressionState> states) {
this.expressions = states;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("expressions");
writer.array();
for (ExpressionState e : expressions) {
e.write(writer, options);
}
writer.endArray();
writer.endObject();
}
}
static protected List<String> toExpressionList(Object o) {

37
main/src/com/google/refine/commands/expr/GetExpressionLanguageInfoCommand.java
View File

@ -35,19 +35,13 @@ package com.google.refine.commands.expr;
import java.io.IOException;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.commands.Command;
import com.google.refine.grel.Control;
import com.google.refine.grel.ControlFunctionRegistry;
@ -55,7 +49,7 @@ import com.google.refine.grel.Function;
public class GetExpressionLanguageInfoCommand extends Command {
public static class LanguageInfo implements Jsonizable {
public static class LanguageInfo {
@JsonProperty("functions")
Map<String, Function> functions;
@ -66,35 +60,6 @@ public class GetExpressionLanguageInfoCommand extends Command {
functions = ControlFunctionRegistry.getFunctionMap();
controls = ControlFunctionRegistry.getControlMap();
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("functions");
writer.object();
{
for (Entry<String, Function> entry : functions.entrySet()) {
writer.key(entry.getKey());
entry.getValue().write(writer, options);
}
}
writer.endObject();
writer.key("controls");
writer.object();
{
for (Entry<String, Control> entry : controls.entrySet()) {
writer.key(entry.getKey());
entry.getValue().write(writer, options);
}
}
writer.endObject();
writer.endObject();
}
}
@Override

28
main/src/com/google/refine/commands/expr/GetStarredExpressionsCommand.java
View File

@ -2,19 +2,14 @@ package com.google.refine.commands.expr;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
import com.google.refine.preference.TopList;
@ -22,39 +17,20 @@ import com.google.refine.preference.TopList;
public class GetStarredExpressionsCommand extends Command {
protected static class Expression implements Jsonizable {
protected static class Expression {
@JsonProperty("code")
protected String code;
protected Expression(String c) {
code = c;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("code"); writer.value(code);
writer.endObject();
}
}
protected static class ExpressionList implements Jsonizable {
protected static class ExpressionList {
@JsonProperty("expressions")
protected List<Expression> expressions;
protected ExpressionList(List<Expression> e) {
expressions = e;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("expressions");
writer.array();
for (Expression s : expressions) {
s.write(writer, options);
}
writer.endArray();
writer.endObject();
}
}
public static ExpressionList getExpressionsList() {

43
main/src/com/google/refine/commands/expr/PreviewExpressionCommand.java
View File

@ -47,14 +47,12 @@ import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import com.google.refine.Jsonizable;
import com.google.refine.commands.Command;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.Evaluable;
@ -71,21 +69,13 @@ import com.google.refine.util.ParsingUtilities;
public class PreviewExpressionCommand extends Command {
protected static interface ExpressionValue extends Jsonizable { }
protected static interface ExpressionValue { }
protected static class ErrorMessage implements ExpressionValue {
@JsonProperty("message")
protected String message;
public ErrorMessage(String m) {
message = m;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("message"); writer.value(message);
writer.endObject();
}
}
protected static class SuccessfulEvaluation implements ExpressionValue {
@JsonValue
@ -93,15 +83,9 @@ public class PreviewExpressionCommand extends Command {
protected SuccessfulEvaluation(String value) {
this.value = value;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.value(value);
}
}
protected static class PreviewResult implements Jsonizable {
protected static class PreviewResult {
@JsonProperty("code")
protected String code;
@JsonProperty("message")
@ -111,6 +95,7 @@ public class PreviewExpressionCommand extends Command {
@JsonInclude(Include.NON_NULL)
protected String type;
@JsonProperty("results")
@JsonInclude(Include.NON_NULL)
List<ExpressionValue> results;
public PreviewResult(String code, String message, String type) {
@ -126,28 +111,6 @@ public class PreviewExpressionCommand extends Command {
this.type = null;
this.results = evaluated;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("code"); writer.value(code);
if(message != null) {
writer.key("message"); writer.value(message);
}
if(type != null) {
writer.key("type"); writer.value(type);
}
if(results != null) {
writer.key("results");
writer.array();
for(ExpressionValue v : results) {
v.write(writer, options);
}
writer.endArray();
}
writer.endObject();
}
}
@Override

37
main/src/com/google/refine/commands/history/GetOperationsCommand.java
View File

@ -35,28 +35,23 @@ package com.google.refine.commands.history;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.commands.Command;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
public class GetOperationsCommand extends Command {
protected static class SimpleHistoryEntry implements Jsonizable {
protected static class SimpleHistoryEntry {
protected HistoryEntry entry;
public SimpleHistoryEntry(HistoryEntry e) {
@ -73,22 +68,9 @@ public class GetOperationsCommand extends Command {
public AbstractOperation getOperation() {
return entry.operation;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("description"); writer.value(entry.description);
if (entry.operation != null) {
writer.key("operation");
entry.operation.write(writer, options);
}
writer.endObject();
}
}
protected static class HistoryEntries implements Jsonizable {
protected static class HistoryEntries {
@JsonProperty("entries")
List<SimpleHistoryEntry> entries;
@ -97,19 +79,6 @@ public class GetOperationsCommand extends Command {
.map(e -> new SimpleHistoryEntry(e))
.collect(Collectors.toList());
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("entries"); writer.array();
for (SimpleHistoryEntry entry : entries) {
entry.write(writer, options);
}
writer.endArray();
writer.endObject();
}
}
@Override

44
main/src/com/google/refine/commands/importing/GetImportingJobStatusCommand.java
View File

@ -35,20 +35,38 @@ package com.google.refine.commands.importing;
import java.io.IOException;
import java.io.Writer;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.commands.Command;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.util.ParsingUtilities;
public class GetImportingJobStatusCommand extends Command {
protected static class JobStatusResponse {
@JsonProperty("code")
protected String code;
@JsonProperty("message")
@JsonInclude(Include.NON_NULL)
protected String message;
@JsonProperty("job")
@JsonInclude(Include.NON_NULL)
protected ImportingJob job;
protected JobStatusResponse(String code, String message, ImportingJob job) {
this.code = code;
this.message = message;
this.job = job;
}
}
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
@ -57,22 +75,10 @@ public class GetImportingJobStatusCommand extends Command {
ImportingJob job = ImportingManager.getJob(jobID);
Writer w = response.getWriter();
JSONWriter writer = new JSONWriter(w);
try {
writer.object();
if (job == null) {
writer.key("code"); writer.value("error");
writer.key("message"); writer.value("No such import job");
} else {
writer.key("code"); writer.value("ok");
writer.key("job"); job.write(writer, new Properties());
}
writer.endObject();
} catch (JSONException e) {
throw new ServletException(e);
} finally {
w.flush();
w.close();
if (job == null) {
ParsingUtilities.defaultWriter.writeValue(w, new JobStatusResponse("error", "No such import job", null));
} else {
ParsingUtilities.defaultWriter.writeValue(w, new JobStatusResponse("ok", null, job));
}
}
}

2
main/src/com/google/refine/commands/project/CreateProjectCommand.java
View File

@ -52,8 +52,8 @@ import com.google.refine.commands.Command;
import com.google.refine.commands.HttpUtilities;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.importing.ImportingUtilities;
import com.google.refine.importing.ImportingManager.Format;
import com.google.refine.importing.ImportingUtilities;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities;

1
main/src/com/google/refine/commands/project/ExportRowsCommand.java
View File

@ -57,7 +57,6 @@ import com.google.refine.exporters.ExporterRegistry;
import com.google.refine.exporters.StreamExporter;
import com.google.refine.exporters.WriterExporter;
import com.google.refine.exporters.sql.SqlExporterException;
import com.google.refine.model.Project;
public class ExportRowsCommand extends Command {

2
main/src/com/google/refine/commands/project/GetMetadataCommand.java
View File

@ -38,7 +38,7 @@ public class GetMetadataCommand extends Command {
}
IMetadata metadata = MetadataFactory.buildDataPackageMetadata(project);
respondJSONObject(response, metadata.getJSON());
respondJSON(response, metadata);
} catch (JSONException e) {
respondException(response, e);
} catch (ValidationException e) {

90
main/src/com/google/refine/commands/project/GetModelsCommand.java
View File

@ -34,26 +34,29 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.commands.project;
import java.io.IOException;
import java.util.Properties;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.commands.Command;
import com.google.refine.commands.HttpUtilities;
import com.google.refine.commands.HttpHeadersSupport;
import com.google.refine.commands.HttpHeadersSupport.HttpHeaderInfo;
import com.google.refine.commands.HttpUtilities;
import com.google.refine.expr.MetaParser;
import com.google.refine.expr.MetaParser.LanguageInfo;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.model.ColumnModel;
import com.google.refine.model.OverlayModel;
import com.google.refine.model.Project;
import com.google.refine.model.RecordModel;
public class GetModelsCommand extends Command {
@Override
@ -67,6 +70,32 @@ public class GetModelsCommand extends Command {
internalRespond(request, response);
}
protected static class ModelsResponse {
@JsonProperty("columnModel")
protected ColumnModel columnModel;
@JsonProperty("recordModel")
protected RecordModel recordModel;
@JsonProperty("overlayModels")
protected Map<String, OverlayModel> overlayModels;
@JsonProperty("scripting")
protected Map<String, LanguageInfo> scripting;
@JsonProperty("httpHeaders")
protected Map<String, HttpHeaderInfo> httpHeaders;
protected ModelsResponse(
ColumnModel columns,
RecordModel records,
Map<String, OverlayModel> overlays,
Map<String, LanguageInfo> languageInfos,
Map<String, HttpHeaderInfo> headers) {
columnModel = columns;
recordModel = records;
overlayModels = overlays;
scripting = languageInfos;
httpHeaders = headers;
}
}
protected void internalRespond(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
@ -86,53 +115,26 @@ public class GetModelsCommand extends Command {
}
try {
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
response.setHeader("Cache-Control", "no-cache");
Properties options = new Properties();
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
writer.key("columnModel"); project.columnModel.write(writer, options);
writer.key("recordModel"); project.recordModel.write(writer, options);
writer.key("overlayModels"); writer.object();
for (String modelName : project.overlayModels.keySet()) {
OverlayModel overlayModel = project.overlayModels.get(modelName);
if (overlayModel != null) {
writer.key(modelName);
project.overlayModels.get(modelName).write(writer, options);
}
}
writer.endObject();
writer.key("scripting"); writer.object();
Map<String, LanguageInfo> prefixesMap = new HashMap<>();
for (String languagePrefix : MetaParser.getLanguagePrefixes()) {
LanguageInfo info = MetaParser.getLanguageInfo(languagePrefix);
writer.key(languagePrefix);
writer.object();
writer.key("name"); writer.value(info.name);
writer.key("defaultExpression"); writer.value(info.defaultExpression);
writer.endObject();
prefixesMap.put(languagePrefix, info);
}
writer.endObject();
writer.key("httpHeaders");
writer.object();
Map<String, HttpHeaderInfo> headersMap = new HashMap<>();
for (String headerLabel : HttpHeadersSupport.getHttpHeaderLabels()) {
HttpHeaderInfo info = HttpHeadersSupport.getHttpHeaderInfo(headerLabel);
writer.key(headerLabel);
writer.object();
writer.key("header"); writer.value(info.header);
writer.key("defaultValue"); writer.value(info.defaultValue);
writer.endObject();
headersMap.put(headerLabel, info);
}
writer.endObject();
writer.endObject();
respondJSON(response, new ModelsResponse(
project.columnModel,
project.recordModel,
project.overlayModels,
prefixesMap,
headersMap));
} catch (JSONException e) {
HttpUtilities.respondException(response, e);
}

34
main/src/com/google/refine/commands/recon/ReconClearOneCellCommand.java
View File

@ -42,6 +42,8 @@ import javax.servlet.http.HttpServletResponse;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.commands.Command;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.history.Change;
@ -57,6 +59,23 @@ import com.google.refine.process.QuickHistoryEntryProcess;
import com.google.refine.util.Pool;
public class ReconClearOneCellCommand extends Command {
protected static class CellResponse {
@JsonProperty("code")
protected String code = "ok";
@JsonProperty("historyEntry")
protected HistoryEntry entry;
@JsonProperty("cell")
Cell cell;
@JsonProperty("pool")
Pool pool;
protected CellResponse(HistoryEntry historyEntry, Cell newCell, Pool newPool) {
entry = historyEntry;
cell = newCell;
pool = newPool;
}
}
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
@ -80,18 +99,13 @@ public class ReconClearOneCellCommand extends Command {
* If the process is done, write back the cell's data so that the
* client side can update its UI right away.
*/
JSONWriter writer = new JSONWriter(response.getWriter());
Pool pool = new Pool();
Properties options = new Properties();
options.put("pool", pool);
if(process.newCell != null && process.newCell.recon != null) {
pool.pool(process.newCell.recon);
}
writer.object();
writer.key("code"); writer.value("ok");
writer.key("historyEntry"); historyEntry.write(writer, options);
writer.key("cell"); process.newCell.write(writer, options);
writer.key("pool"); pool.write(writer, options);
writer.endObject();
respondJSON(response, new CellResponse(historyEntry, process.newCell, pool));
} else {
respond(response, "{ \"code\" : \"pending\" }");
}

13
main/src/com/google/refine/commands/recon/ReconJudgeOneCellCommand.java
View File

@ -103,18 +103,13 @@ public class ReconJudgeOneCellCommand extends Command {
* If the process is done, write back the cell's data so that the
* client side can update its UI right away.
*/
JSONWriter writer = new JSONWriter(response.getWriter());
Pool pool = new Pool();
Properties options = new Properties();
options.put("pool", pool);
if (process.newCell != null && process.newCell.recon != null) {
pool.pool(process.newCell.recon);
}
writer.object();
writer.key("code"); writer.value("ok");
writer.key("historyEntry"); historyEntry.write(writer, options);
writer.key("cell"); process.newCell.write(writer, options);
writer.key("pool"); pool.write(writer, options);
writer.endObject();
respondJSON(response, new ReconClearOneCellCommand.CellResponse(historyEntry, process.newCell, pool));
} else {
respond(response, "{ \"code\" : \"pending\" }");
}

61
main/src/com/google/refine/commands/row/GetRowsCommand.java
View File

@ -37,7 +37,6 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@ -45,14 +44,12 @@ import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
import com.google.refine.Jsonizable;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.Engine.Mode;
import com.google.refine.browsing.FilteredRecords;
@ -62,6 +59,7 @@ import com.google.refine.browsing.RowVisitor;
import com.google.refine.commands.Command;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Record;
import com.google.refine.model.Row;
@ -73,12 +71,12 @@ import com.google.refine.util.Pool;
public class GetRowsCommand extends Command {
protected static class WrappedRow implements Jsonizable {
protected static class WrappedRow {
@JsonUnwrapped
protected final Row row;
@JsonProperty("rowIndex")
@JsonProperty("i")
protected final int rowIndex;
@JsonProperty("recordIndex")
@JsonProperty("j")
@JsonInclude(Include.NON_NULL)
protected final Integer recordIndex;
@ -87,23 +85,9 @@ public class GetRowsCommand extends Command {
this.rowIndex = rowIndex;
this.recordIndex = recordIndex;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
options.put("rowIndex", rowIndex);
if(recordIndex != null) {
options.put("recordIndex", recordIndex);
}
row.write(writer, options);
if(recordIndex != null) {
options.remove("recordIndex");
}
}
}
protected static class JsonResult implements Jsonizable {
protected static class JsonResult {
@JsonProperty("mode")
protected final Mode mode;
@JsonProperty("rows")
@ -129,24 +113,6 @@ public class GetRowsCommand extends Command {
this.limit = limit;
this.pool = pool;
}
@Override
public void write(JSONWriter jsonWriter, Properties options)
throws JSONException {
jsonWriter.object();
jsonWriter.key("mode"); jsonWriter.value(mode == Mode.RowBased ? "row-based" : "record-based");
jsonWriter.key("rows"); jsonWriter.array();
for(WrappedRow row : rows) {
row.write(jsonWriter, options);
}
jsonWriter.endArray();
jsonWriter.key("filtered"); jsonWriter.value(filtered);
jsonWriter.key("total"); jsonWriter.value(totalCount);
jsonWriter.key("start"); jsonWriter.value(start);
jsonWriter.key("limit"); jsonWriter.value(limit);
jsonWriter.key("pool"); pool.write(jsonWriter, options);
jsonWriter.endObject();
}
}
@Override
@ -186,10 +152,10 @@ public class GetRowsCommand extends Command {
int limit = Math.min(project.rows.size() - start, Math.max(0, getIntegerParameter(request, "limit", 20)));
Pool pool = new Pool();
Properties options = new Properties();
/* Properties options = new Properties();
options.put("project", project);
options.put("reconCandidateOmitTypes", true);
options.put("pool", pool);
options.put("pool", pool); */
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", callback == null ? "application/json" : "text/javascript");
@ -200,8 +166,6 @@ public class GetRowsCommand extends Command {
writer.write("(");
}
JSONWriter jsonWriter = new JSONWriter(writer);
RowWritingVisitor rwv = new RowWritingVisitor(start, limit);
SortingConfig sortingConfig = null;
@ -243,12 +207,21 @@ public class GetRowsCommand extends Command {
filteredRecords.accept(project, visitor);
}
// Pool all the recons occuring in the rows seen
for(WrappedRow wr : rwv.results) {
for(Cell c : wr.row.cells) {
if(c != null && c.recon != null) {
pool.pool(c.recon);
}
}
}
JsonResult result = new JsonResult(engine.getMode(),
rwv.results, rwv.total,
engine.getMode() == Mode.RowBased ? project.rows.size() : project.recordModel.getRecordCount(),
start, limit, pool);
result.write(jsonWriter, options);
ParsingUtilities.defaultWriter.writeValue(writer, result);
if (callback != null) {
writer.write(")");
}

31
main/src/com/google/refine/commands/workspace/GetAllProjectMetadataCommand.java
View File

@ -35,29 +35,24 @@ package com.google.refine.commands.workspace;
import java.io.IOException;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonRawValue;
import com.google.refine.Jsonizable;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
import com.google.refine.model.metadata.ProjectMetadata;
public class GetAllProjectMetadataCommand extends Command {
public static class AllProjectMetadata implements Jsonizable {
public static class AllProjectMetadata {
@JsonProperty("projects")
protected Map<Long, ProjectMetadata> projects;
@JsonProperty("customMetadataColumns")
@ -69,30 +64,6 @@ public class GetAllProjectMetadataCommand extends Command {
projects = map;
customMetadataColumns = json;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("projects");
writer.object();
for (Entry<Long,ProjectMetadata> e : projects.entrySet()) {
ProjectMetadata pm = e.getValue();
if (pm != null) {
writer.key(e.getKey().toString());
pm.write(writer, options);
}
}
writer.endObject();
if (customMetadataColumns != null) {
writer.key("customMetadataColumns");
writer.value(new JSONArray(customMetadataColumns));
}
writer.endObject();
}
}
@Override

22
main/src/com/google/refine/commands/workspace/GetAllProjectTagsCommand.java
View File

@ -27,9 +27,7 @@ package com.google.refine.commands.workspace;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import javax.servlet.ServletException;
@ -37,17 +35,15 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.ProjectManager;
import com.google.refine.commands.Command;
public class GetAllProjectTagsCommand extends Command {
public static class AllProjectsTags implements Jsonizable {
public static class AllProjectsTags {
@JsonProperty("tags")
protected Set<String> tags;
@ -55,22 +51,6 @@ public class GetAllProjectTagsCommand extends Command {
protected AllProjectsTags(Set<String> tags) {
this.tags = tags;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("tags");
writer.array();
if (tags != null) {
for (String tag : tags) {
writer.value(tag);
}
}
writer.endArray();
writer.endObject();
}
}
@Override

17
main/src/com/google/refine/expr/EvalError.java
View File

@ -34,14 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr;
import java.io.Serializable;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
/**
* An error that occurs during the evaluation of an Evaluable. Errors are values, too
@ -49,7 +44,7 @@ import com.google.refine.Jsonizable;
* thrown because an error might occupy just one element in an array and doesn't need
* to make the whole array erroneous.
*/
public class EvalError implements Serializable, Jsonizable {
public class EvalError implements Serializable {
private static final long serialVersionUID = -102681220092874080L;
@JsonProperty("message")
@ -69,16 +64,6 @@ public class EvalError implements Serializable, Jsonizable {
return this.message;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("type"); writer.value("error");
writer.key("message"); writer.value(message);
writer.endObject();
}
@JsonProperty("type")
public String getType() {
return "error";

4
main/src/com/google/refine/expr/MetaParser.java
View File

@ -39,11 +39,11 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import com.google.refine.grel.Parser;
import clojure.lang.IFn;
import clojure.lang.RT;
import com.google.refine.grel.Parser;
abstract public class MetaParser {
static public class LanguageInfo {

19
main/src/com/google/refine/grel/Control.java
View File

@ -35,14 +35,10 @@ package com.google.refine.grel;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.expr.Evaluable;
/**
@ -50,7 +46,7 @@ import com.google.refine.expr.Evaluable;
* decide which part of the code to execute and can affect the environment bindings.
* Functions, on the other hand, can't do either.
*/
public interface Control extends Jsonizable {
public interface Control {
public Object call(Properties bindings, Evaluable[] args);
public String checkArguments(Evaluable[] args);
@ -66,17 +62,4 @@ public interface Control extends Jsonizable {
@JsonProperty("returns")
public String getReturns();
@Override
default public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("description"); writer.value(getDescription());
if (!getParams().isEmpty()) {
writer.key("params"); writer.value(getParams());
}
writer.key("returns"); writer.value(getReturns());
writer.endObject();
}
}

6
main/src/com/google/refine/grel/ControlFunctionRegistry.java
View File

@ -108,19 +108,19 @@ import com.google.refine.expr.functions.strings.Contains;
import com.google.refine.expr.functions.strings.Diff;
import com.google.refine.expr.functions.strings.EndsWith;
import com.google.refine.expr.functions.strings.Escape;
import com.google.refine.expr.functions.strings.Find;
import com.google.refine.expr.functions.strings.Fingerprint;
import com.google.refine.expr.functions.strings.IndexOf;
import com.google.refine.expr.functions.strings.LastIndexOf;
import com.google.refine.expr.functions.strings.MD5;
import com.google.refine.expr.functions.strings.Match;
import com.google.refine.expr.functions.strings.Find;
import com.google.refine.expr.functions.strings.NGram;
import com.google.refine.expr.functions.strings.NGramFingerprint;
import com.google.refine.expr.functions.strings.ParseJson;
import com.google.refine.expr.functions.strings.Partition;
import com.google.refine.expr.functions.strings.Phonetic;
import com.google.refine.expr.functions.strings.Range;
import com.google.refine.expr.functions.strings.RPartition;
import com.google.refine.expr.functions.strings.Range;
import com.google.refine.expr.functions.strings.Reinterpret;
import com.google.refine.expr.functions.strings.Replace;
import com.google.refine.expr.functions.strings.ReplaceChars;
@ -144,8 +144,8 @@ import com.google.refine.grel.controls.ForNonBlank;
import com.google.refine.grel.controls.ForRange;
import com.google.refine.grel.controls.If;
import com.google.refine.grel.controls.IsBlank;
import com.google.refine.grel.controls.IsError;
import com.google.refine.grel.controls.IsEmptyString;
import com.google.refine.grel.controls.IsError;
import com.google.refine.grel.controls.IsNonBlank;
import com.google.refine.grel.controls.IsNotNull;
import com.google.refine.grel.controls.IsNull;

16
main/src/com/google/refine/grel/Function.java
View File

@ -35,18 +35,15 @@ package com.google.refine.grel;
import java.util.Properties;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
/**
* Interface for functions. When a function is called, its arguments have already
* been evaluated down into non-error values.
*/
public interface Function extends Jsonizable {
public interface Function {
public Object call(Properties bindings, Object[] args);
@JsonProperty("description")
@ -60,15 +57,4 @@ public interface Function extends Jsonizable {
@JsonProperty("returns")
public String getReturns();
@Override
default public void write(JSONWriter writer, Properties options) {
writer.object();
writer.key("description"); writer.value(getDescription());
if (!getParams().isEmpty()) {
writer.key("params"); writer.value(getParams());
}
writer.key("returns"); writer.value(getReturns());
writer.endObject();
}
}

1
main/src/com/google/refine/history/Change.java
View File

@ -50,5 +50,6 @@ public interface Change {
public void apply(Project project);
public void revert(Project project);
public void save(Writer writer, Properties options) throws IOException;
}

27
main/src/com/google/refine/history/History.java
View File

@ -46,12 +46,8 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet;
import com.google.refine.model.Project;
@ -64,7 +60,7 @@ import com.google.refine.util.Pool;
* the history entries are much smaller and can be kept in memory, while the change objects
* are only loaded into memory on demand.
*/
public class History implements Jsonizable {
public class History {
static public Change readOneChange(InputStream in, Pool pool) throws Exception {
LineNumberReader reader = new LineNumberReader(new InputStreamReader(in, "UTF-8"));
try {
@ -263,27 +259,6 @@ public class History implements Jsonizable {
}
}
@Override
synchronized public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("past"); writer.array();
for (HistoryEntry entry : _pastEntries) {
entry.write(writer, options);
}
writer.endArray();
writer.key("future"); writer.array();
for (HistoryEntry entry : _futureEntries) {
entry.write(writer, options);
}
writer.endArray();
writer.endObject();
}
/*
* NOTE: this method is called from the autosave thread with the Project
* lock already held, so no other synchronized method here can aquire that

21
main/src/com/google/refine/history/HistoryEntry.java
View File

@ -34,13 +34,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.history;
import java.io.Writer;
import java.time.ZoneId;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -48,7 +46,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
import com.google.refine.Jsonizable;
import com.google.refine.ProjectManager;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
@ -60,7 +57,7 @@ import com.google.refine.util.ParsingUtilities;
* This is the metadata of a Change. It's small, so we can load it in order to
* obtain information about a change without actually loading the change.
*/
public class HistoryEntry implements Jsonizable {
public class HistoryEntry {
final static Logger logger = LoggerFactory.getLogger("HistoryEntry");
@JsonProperty("id")
final public long id;
@ -117,20 +114,6 @@ public class HistoryEntry implements Jsonizable {
}
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(id);
writer.key("description"); writer.value(description);
writer.key("time"); writer.value(ParsingUtilities.dateToString(time));
if ("save".equals(options.getProperty("mode")) && operation != null) {
writer.key(OPERATION); operation.write(writer, options);
}
writer.endObject();
}
public void save(Writer writer, Properties options){
_manager.save(this, writer, options);
}

16
main/src/com/google/refine/history/HistoryProcess.java
View File

@ -33,11 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.history;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -98,17 +93,6 @@ public class HistoryProcess extends Process {
public void startPerforming(ProcessManager manager) {
throw new RuntimeException(WARN);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("description"); writer.value(_description);
writer.key("immediate"); writer.value(true);
writer.key("status"); writer.value(getStatus());
writer.endObject();
}
@JsonProperty("status")
public String getStatus() {

30
main/src/com/google/refine/importing/DefaultImportingController.java
View File

@ -50,6 +50,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.RefineServlet;
import com.google.refine.commands.HttpUtilities;
import com.google.refine.importing.ImportingManager.Format;
@ -272,6 +274,19 @@ public class DefaultImportingController implements ImportingController {
}
}
protected static class JobResponse {
@JsonProperty("code")
protected String code;
@JsonProperty("job")
protected ImportingJob job;
protected JobResponse(String code, ImportingJob job) {
this.code = code;
this.job = job;
}
}
/**
* return the job to the front end.
* @param request
@ -284,18 +299,9 @@ public class DefaultImportingController implements ImportingController {
throws ServletException, IOException {
Writer w = response.getWriter();
JSONWriter writer = new JSONWriter(w);
try {
writer.object();
writer.key("code"); writer.value("ok");
writer.key("job"); job.write(writer, new Properties());
writer.endObject();
} catch (JSONException e) {
throw new ServletException(e);
} finally {
w.flush();
w.close();
}
ParsingUtilities.defaultWriter.writeValue(w, job);
w.flush();
w.close();
}
static public void writeErrors(JSONWriter writer, List<Exception> exceptions) throws JSONException {

17
main/src/com/google/refine/importing/ImportingJob.java
View File

@ -37,22 +37,18 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
import com.google.refine.ProjectManager;
import com.google.refine.model.Project;
import com.google.refine.model.metadata.ProjectMetadata;
import com.google.refine.util.JSONUtilities;
public class ImportingJob implements Jsonizable {
public class ImportingJob {
final public long id;
final public File dir; // Temporary directory where the data about this job is stored
@ -205,15 +201,4 @@ public class ImportingJob implements Jsonizable {
dir2.mkdirs();
return dir2;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
synchronized(lock) {
writer.object();
writer.key("config"); writer.value(config);
writer.endObject();
}
}
}

12
main/src/com/google/refine/io/FileHistoryEntryManager.java
View File

@ -35,6 +35,7 @@ package com.google.refine.io;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Writer;
import java.util.Properties;
@ -42,13 +43,11 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.ProjectManager;
import com.google.refine.history.History;
import com.google.refine.history.HistoryEntry;
import com.google.refine.history.HistoryEntryManager;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
@ -64,10 +63,9 @@ public class FileHistoryEntryManager implements HistoryEntryManager{
@Override
public void save(HistoryEntry historyEntry, Writer writer, Properties options) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
historyEntry.write(jsonWriter, options);
} catch (JSONException e) {
ParsingUtilities.defaultWriter.writeValue(writer, historyEntry);
} catch (IOException e) {
e.printStackTrace();
}
}
@ -116,6 +114,8 @@ public class FileHistoryEntryManager implements HistoryEntryManager{
out.putNextEntry(new ZipEntry("change.txt"));
try {
History.writeOneChange(out, historyEntry.getChange(), pool);
} catch(Exception e) {
e.printStackTrace();
} finally {
out.closeEntry();
}

32
main/src/com/google/refine/io/FileProjectManager.java
View File

@ -43,7 +43,6 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@ -55,14 +54,12 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.ProjectManager;
import com.google.refine.history.HistoryEntryManager;
import com.google.refine.model.Project;
@ -71,9 +68,10 @@ import com.google.refine.model.metadata.IMetadata;
import com.google.refine.model.metadata.MetadataFormat;
import com.google.refine.model.metadata.ProjectMetadata;
import com.google.refine.preference.TopList;
import com.google.refine.util.ParsingUtilities;
public class FileProjectManager extends ProjectManager implements Jsonizable {
public class FileProjectManager extends ProjectManager {
final static protected String PROJECT_DIR_SUFFIX = ".project";
protected File _workspaceDir;
@ -251,7 +249,9 @@ public class FileProjectManager extends ProjectManager implements Jsonizable {
ProjectMetadataUtilities.save(metadata, projectDir);
} else if (metadata.getFormatName() == MetadataFormat.DATAPACKAGE_METADATA) {
DataPackageMetadata dp = (DataPackageMetadata)metadata;
dp.writeToFile(new File(projectDir, DataPackageMetadata.DEFAULT_FILE_NAME));
FileWriter writer = new FileWriter(new File(projectDir, DataPackageMetadata.DEFAULT_FILE_NAME));
ParsingUtilities.defaultWriter.writeValue(writer, dp);
writer.close();
}
logger.info("metadata saved in " + metadata.getFormatName());
@ -326,8 +326,7 @@ public class FileProjectManager extends ProjectManager implements Jsonizable {
FileWriter writer = new FileWriter(file);
boolean saveWasNeeded = saveNeeded();
try {
JSONWriter jsonWriter = new JSONWriter(writer);
write(jsonWriter, new Properties());
ParsingUtilities.defaultWriter.writeValue(writer, this);
saveProjectMetadata();
} finally {
writer.close();
@ -486,25 +485,6 @@ public class FileProjectManager extends ProjectManager implements Jsonizable {
gos.close();
}
}
@Override
public void write(JSONWriter jsonWriter, Properties options)
throws JSONException {
jsonWriter.object();
jsonWriter.key("projectIDs");
jsonWriter.array();
for (Long id : _projectsMetadata.keySet()) {
ProjectMetadata metadata = _projectsMetadata.get(id);
if (metadata != null) {
jsonWriter.value(id);
}
}
jsonWriter.endArray();
jsonWriter.key("preferences");
_preferenceStore.write(jsonWriter, new Properties());
jsonWriter.endObject();
}
@JsonProperty("projectIDs")
public Set<Long> getProjectIds() {

5
main/src/com/google/refine/io/ProjectMetadataUtilities.java
View File

@ -46,13 +46,13 @@ import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.json.JSONException;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.model.Project;
import com.google.refine.model.metadata.IMetadata;
import com.google.refine.model.metadata.ProjectMetadata;
import com.google.refine.util.ParsingUtilities;
public class ProjectMetadataUtilities {
final static Logger logger = LoggerFactory.getLogger("project_metadata_utilities");
@ -78,8 +78,7 @@ public class ProjectMetadataUtilities {
protected static void saveToFile(IMetadata projectMeta, File metadataFile) throws JSONException, IOException {
Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile));
try {
JSONWriter jsonWriter = new JSONWriter(writer);
projectMeta.write(jsonWriter, false);
ParsingUtilities.defaultWriter.writeValue(writer, projectMeta);
} finally {
writer.close();
}

3
main/src/com/google/refine/model/AbstractOperation.java
View File

@ -37,7 +37,6 @@ import java.util.Properties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.history.HistoryEntry;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.process.Process;
@ -47,7 +46,7 @@ import com.google.refine.process.QuickHistoryEntryProcess;
* An abstract operation can be applied to different but similar
* projects.
*/
abstract public class AbstractOperation implements Jsonizable {
abstract public class AbstractOperation {
public Process createProcess(Project project, Properties options) throws Exception {
return new QuickHistoryEntryProcess(project, getBriefDescription(null)) {
@Override

42
main/src/com/google/refine/model/Cell.java
View File

@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model;
import java.io.IOException;
import java.io.Serializable;
import java.io.Writer;
import java.time.Instant;
@ -41,9 +42,6 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
@ -52,7 +50,6 @@ import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.google.refine.Jsonizable;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.HasFields;
@ -60,7 +57,7 @@ import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
import com.google.refine.util.StringUtils;
public class Cell implements HasFields, Jsonizable {
public class Cell implements HasFields {
@JsonIgnore
final public Serializable value;
@JsonIgnore
@ -85,31 +82,6 @@ public class Cell implements HasFields, Jsonizable {
public boolean fieldAlsoHasFields(String name) {
return "recon".equals(name);
}
@Override
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.object();
if (ExpressionUtils.isError(value)) {
writer.key("e");
writer.value(((EvalError) value).message);
} else {
writer.key("v");
writer.value(getValueAsString());
String jsonType = getTypeString();
if(jsonType != null) {
writer.key("t"); writer.value(jsonType);
}
}
if (recon != null) {
writer.key("r");
writer.value(Long.toString(recon.id));
Pool pool = (Pool) options.get("pool");
pool.pool(recon);
}
writer.endObject();
}
@JsonProperty("e")
@JsonInclude(Include.NON_NULL)
@ -171,15 +143,17 @@ public class Cell implements HasFields, Jsonizable {
if (recon != null) {
return Long.toString(recon.id);
}
// TODO pool the recon??
return null;
}
public void save(Writer writer, Properties options) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
write(jsonWriter, options);
} catch (JSONException e) {
Pool pool = (Pool)options.get("pool");
if(pool != null && recon != null) {
pool.pool(recon);
}
ParsingUtilities.saveWriter.writeValue(writer, this);
} catch (IOException e) {
e.printStackTrace();
}
}

37
main/src/com/google/refine/model/Column.java
View File

@ -33,23 +33,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model;
import java.io.IOException;
import java.io.Writer;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.InterProjectModel;
import com.google.refine.Jsonizable;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities;
@ -59,7 +56,7 @@ import io.frictionlessdata.tableschema.TypeInferrer;
import io.frictionlessdata.tableschema.exceptions.ConstraintsException;
import io.frictionlessdata.tableschema.exceptions.InvalidCastException;
public class Column implements Jsonizable {
public class Column {
final private int _cellIndex;
final private String _originalName;
private String _name;
@ -118,31 +115,6 @@ public class Column implements Jsonizable {
public ReconStats getReconStats() {
return _reconStats;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("cellIndex"); writer.value(_cellIndex);
writer.key("originalName"); writer.value(_originalName);
writer.key("name"); writer.value(_name);
writer.key("type"); writer.value(type);
writer.key("format"); writer.value(format);
writer.key("title"); writer.value(title);
writer.key("description"); writer.value(description);
writer.key("constraints"); writer.value(new JSONObject(constraints).toString());
if (_reconConfig != null) {
writer.key("reconConfig");
_reconConfig.write(writer, options);
}
if (_reconStats != null) {
writer.key("reconStats");
_reconStats.write(writer, options);
}
writer.endObject();
}
/**
* Clear all cached precomputed values.
@ -230,10 +202,9 @@ public class Column implements Jsonizable {
}
public void save(Writer writer) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
write(jsonWriter, new Properties());
} catch (JSONException e) {
ParsingUtilities.defaultWriter.writeValue(writer, this);
} catch (IOException e) {
e.printStackTrace();
}
}

32
main/src/com/google/refine/model/ColumnGroup.java
View File

@ -33,25 +33,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model;
import java.io.IOException;
import java.io.Writer;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
import com.google.refine.Jsonizable;
import com.google.refine.util.JsonViews;
import com.google.refine.util.ParsingUtilities;
public class ColumnGroup implements Jsonizable {
public class ColumnGroup {
final public int startColumnIndex;
final public int columnSpan;
final public int keyColumnIndex; // could be -1 if there is no key cell
@ -66,26 +63,6 @@ public class ColumnGroup implements Jsonizable {
internalInitialize();
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("startColumnIndex"); writer.value(startColumnIndex);
writer.key("columnSpan"); writer.value(columnSpan);
writer.key("keyColumnIndex"); writer.value(keyColumnIndex);
if (!"save".equals(options.get("mode")) && (subgroups != null) && (subgroups.size() > 0)) {
writer.key("subgroups"); writer.array();
for (ColumnGroup g : subgroups) {
g.write(writer, options);
}
writer.endArray();
}
writer.endObject();
}
@JsonProperty("startColumnIndex")
public int getStartColumnIndex() {
return startColumnIndex;
@ -114,10 +91,9 @@ public class ColumnGroup implements Jsonizable {
}
public void save(Writer writer) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
write(jsonWriter, new Properties());
} catch (JSONException e) {
ParsingUtilities.defaultWriter.writeValue(writer, this);
} catch (IOException e) {
e.printStackTrace();
}
}

34
main/src/com/google/refine/model/ColumnModel.java
View File

@ -45,17 +45,13 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
public class ColumnModel implements Jsonizable {
public class ColumnModel {
@JsonProperty("columns")
final public List<Column> columns = new LinkedList<Column>();
@JsonProperty("columnGroups")
@ -179,34 +175,6 @@ public class ColumnModel implements Jsonizable {
synchronized public List<String> getColumnNames() {
return _columnNames;
}
@Override
synchronized public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("columns");
writer.array();
for (Column column : columns) {
column.write(writer, options);
}
writer.endArray();
if (columns.size() > 0) {
writer.key("keyCellIndex"); writer.value(getKeyColumnIndex());
writer.key("keyColumnName"); writer.value(columns.get(_keyColumnIndex).getName());
}
writer.key("columnGroups");
writer.array();
for (ColumnGroup g : _rootColumnGroups) {
g.write(writer, options);
}
writer.endArray();
writer.endObject();
}
@JsonProperty("keyCellIndex")
@JsonInclude(Include.NON_NULL)

3
main/src/com/google/refine/model/OverlayModel.java
View File

@ -33,9 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model;
import com.google.refine.Jsonizable;
public interface OverlayModel extends Jsonizable {
public interface OverlayModel {
public void onBeforeSave(Project project);
public void onAfterSave(Project project);

10
main/src/com/google/refine/model/Project.java
View File

@ -48,9 +48,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -160,13 +158,7 @@ public class Project {
writer.write(modelName);
writer.write("=");
try {
JSONWriter jsonWriter = new JSONWriter(writer);
overlayModels.get(modelName).write(jsonWriter, options);
} catch (JSONException e) {
e.printStackTrace();
}
ParsingUtilities.saveWriter.writeValue(writer, overlayModels.get(modelName));
writer.write('\n');
}

56
main/src/com/google/refine/model/Recon.java
View File

@ -40,9 +40,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonFilter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
@ -53,13 +50,12 @@ import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.google.refine.Jsonizable;
import com.google.refine.expr.HasFields;
import com.google.refine.util.JsonViews;
import com.google.refine.util.Pool;
@JsonFilter("reconCandidateFilter")
public class Recon implements HasFields, Jsonizable {
public class Recon implements HasFields {
/**
* Freebase schema URLs kept for compatibility with legacy reconciliation results
@ -366,56 +362,6 @@ public class Recon implements HasFields, Jsonizable {
return null;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
boolean saveMode = "save".equals(options.getProperty("mode"));
writer.object();
writer.key("id"); writer.value(id);
if (saveMode) {
writer.key("judgmentHistoryEntry"); writer.value(judgmentHistoryEntry);
}
writer.key("service"); writer.value(service);
writer.key("identifierSpace"); writer.value(identifierSpace);
writer.key("schemaSpace"); writer.value(schemaSpace);
writer.key("j"); writer.value(judgmentToString());
if (match != null) {
writer.key("m");
match.write(writer, options);
}
if (match == null || saveMode) {
writer.key("c"); writer.array();
if (candidates != null) {
for (ReconCandidate c : candidates) {
c.write(writer, options);
}
}
writer.endArray();
}
if (saveMode) {
writer.key("f");
writer.array();
for (Object o : features) {
writer.value(o);
}
writer.endArray();
writer.key("judgmentAction"); writer.value(judgmentAction);
writer.key("judgmentBatchSize"); writer.value(judgmentBatchSize);
if (match != null) {
writer.key("matchRank"); writer.value(matchRank);
}
}
writer.endObject();
}
static public Recon loadStreaming(String s, Pool pool) throws Exception {
JsonFactory jsonFactory = new JsonFactory();
JsonParser jp = jsonFactory.createJsonParser(s);

27
main/src/com/google/refine/model/ReconCandidate.java
View File

@ -37,19 +37,14 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.google.refine.Jsonizable;
import com.google.refine.expr.HasFields;
public class ReconCandidate implements HasFields, Jsonizable {
public class ReconCandidate implements HasFields {
@JsonProperty("id")
final public String id;
@JsonProperty("name")
@ -84,26 +79,6 @@ public class ReconCandidate implements HasFields, Jsonizable {
public boolean fieldAlsoHasFields(String name) {
return false;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(id);
writer.key("name"); writer.value(name);
writer.key("score"); writer.value(score);
/* if (!options.containsKey("reconCandidateOmitTypes")) */ {
writer.key("types"); writer.array();
for (String typeID : types) {
writer.value(typeID);
}
writer.endArray();
}
writer.endObject();
}
static public ReconCandidate loadStreaming(String s) throws Exception {
JsonFactory jsonFactory = new JsonFactory();

24
main/src/com/google/refine/model/ReconStats.java
View File

@ -33,20 +33,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model;
import java.io.IOException;
import java.io.Writer;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.util.ParsingUtilities;
public class ReconStats implements Jsonizable {
public class ReconStats {
static public ReconStats load(JSONObject obj) throws Exception {
return new ReconStats(
obj.getInt("nonBlanks"),
@ -67,17 +65,6 @@ public class ReconStats implements Jsonizable {
this.newTopics = newTopics;
this.matchedTopics = matchedTopics;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("nonBlanks"); writer.value(nonBlanks);
writer.key("newTopics"); writer.value(newTopics);
writer.key("matchedTopics"); writer.value(matchedTopics);
writer.endObject();
}
static public ReconStats create(Project project, int cellIndex) {
int nonBlanks = 0;
@ -103,10 +90,9 @@ public class ReconStats implements Jsonizable {
}
public void save(Writer writer) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
write(jsonWriter, new Properties());
} catch (JSONException e) {
ParsingUtilities.defaultWriter.writeValue(writer, this);
} catch (IOException e) {
e.printStackTrace();
}
}

17
main/src/com/google/refine/model/ReconType.java
View File

@ -33,22 +33,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
/**
* This represents a type from the reconciliation
* service. It is used when extending data to
* store the (expected) types of new columns.
*/
public class ReconType implements Jsonizable {
public class ReconType {
@JsonProperty("id")
public String id;
@JsonProperty("name")
@ -59,16 +54,6 @@ public class ReconType implements Jsonizable {
this.name = name;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(id);
writer.key("name"); writer.value(name);
writer.endObject();
}
static public ReconType load(JSONObject obj) throws Exception {
if (obj == null) {
return null;

16
main/src/com/google/refine/model/RecordModel.java
View File

@ -38,20 +38,16 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.expr.ExpressionUtils;
public class RecordModel implements Jsonizable {
public class RecordModel {
final static Logger logger = LoggerFactory.getLogger("RecordModel");
final static public class CellDependency {
@ -108,16 +104,6 @@ public class RecordModel implements Jsonizable {
}
return null;
}
@Override
synchronized public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("hasRecords");
writer.value(hasRecords());
writer.endObject();
}
@JsonProperty("hasRecords")
public boolean hasRecords() {

71
main/src/com/google/refine/model/Row.java
View File

@ -33,32 +33,28 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model;
import java.io.IOException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
import java.util.Properties;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
import com.google.refine.expr.CellTuple;
import com.google.refine.expr.HasFields;
import com.google.refine.util.JsonViews;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
/**
* Class representing a single Row which contains a list of {@link Cell}s. There may
* be multiple rows in a {@link Record}.
*/
public class Row implements HasFields, Jsonizable {
public class Row implements HasFields {
public boolean flagged;
public boolean starred;
final public List<Cell> cells;
@ -165,50 +161,6 @@ public class Row implements HasFields, Jsonizable {
return new CellTuple(project, this);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key(FLAGGED); writer.value(flagged);
writer.key(STARRED); writer.value(starred);
writer.key("cells"); writer.array();
for (Cell cell : cells) {
if (cell != null) {
cell.write(writer, options);
} else {
writer.value(null);
}
}
writer.endArray();
if (!"save".equals(options.getProperty("mode"))) {
if (options.containsKey("rowIndex")) {
int rowIndex = (Integer) options.get("rowIndex");
writer.key("i"); writer.value(rowIndex);
if (options.containsKey("recordIndex")) {
int recordIndex = (Integer) options.get("recordIndex");
writer.key("j"); writer.value(recordIndex);
}
}
if (options.containsKey("extra")) {
Properties extra = (Properties) options.get("extra");
if (extra != null) {
for (Entry<Object,Object> e : extra.entrySet()) {
writer.key((String) e.getKey());
writer.value(e.getValue());
}
}
}
}
writer.endObject();
}
@JsonProperty(FLAGGED)
public boolean isFlagged() {
return flagged;
@ -230,10 +182,21 @@ public class Row implements HasFields, Jsonizable {
*/
public void save(Writer writer, Properties options) {
JSONWriter jsonWriter = new JSONWriter(writer);
if (options.containsKey("rowIndex")) {
// See GetRowsCommand to serialize a row with indices
throw new IllegalArgumentException("Serializing with row indices is not supported anymore.");
}
try {
write(jsonWriter, options);
} catch (JSONException e) {
ParsingUtilities.saveWriter.writeValue(writer, this);
Pool pool = (Pool)options.get("pool");
if(pool != null) {
for(Cell c : cells) {
if (c != null && c.recon != null) {
pool.pool(c.recon);
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
}

13
main/src/com/google/refine/model/changes/ColumnChange.java
View File

@ -40,11 +40,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.history.Change;
import com.google.refine.model.ColumnGroup;
import com.google.refine.util.ParsingUtilities;
abstract public class ColumnChange implements Change {
@ -52,13 +50,8 @@ abstract public class ColumnChange implements Change {
List<ColumnGroup> oldColumnGroups) throws IOException {
writer.write("oldColumnGroupCount=");
writer.write(Integer.toString(oldColumnGroups.size())); writer.write('\n');
for (ColumnGroup cg : oldColumnGroups) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
cg.write(jsonWriter, options);
} catch (JSONException e) {
throw new IOException(e);
}
for (ColumnGroup cg : oldColumnGroups) {
ParsingUtilities.saveWriter.writeValue(writer, cg);
writer.write('\n');
}
}

9
main/src/com/google/refine/model/changes/DataExtensionChange.java
View File

@ -307,13 +307,8 @@ public class DataExtensionChange implements Change {
}
writer.write("columnTypeCount="); writer.write(Integer.toString(_columnTypes.size())); writer.write('\n');
for (ReconType type : _columnTypes) {
try {
if(type != null) {
JSONWriter jsonWriter = new JSONWriter(writer);
type.write(jsonWriter, options);
}
} catch (JSONException e) {
// ???
if(type != null) {
ParsingUtilities.defaultWriter.writeValue(writer, type);
}
writer.write('\n');
}

8
main/src/com/google/refine/model/changes/MassReconChange.java
View File

@ -50,6 +50,7 @@ import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Row;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
public class MassReconChange implements Change {
@ -110,12 +111,7 @@ public class MassReconChange implements Change {
Pool pool = (Pool) options.get("pool");
pool.poolReconCandidates(recon);
JSONWriter jsonWriter = new JSONWriter(writer);
try {
recon.write(jsonWriter, options);
} catch (JSONException e) {
e.printStackTrace();
}
ParsingUtilities.saveWriter.writeValue(writer, recon);
writer.write("\n");
}
}

23
main/src/com/google/refine/model/metadata/AbstractMetadata.java
View File

@ -2,12 +2,9 @@ package com.google.refine.model.metadata;
import java.io.File;
import java.time.LocalDateTime;
import java.util.Properties;
import org.apache.commons.beanutils.PropertyUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
public abstract class AbstractMetadata implements IMetadata {
private MetadataFormat formatName = MetadataFormat.UNKNOWN;
@ -30,9 +27,6 @@ public abstract class AbstractMetadata implements IMetadata {
@Override
public abstract void loadFromFile(File metadataFile);
@Override
public abstract void writeToFile(File metadataFile);
@Override
public boolean isDirty() {
return written == null || _modified.isAfter(written);
@ -48,23 +42,6 @@ public abstract class AbstractMetadata implements IMetadata {
_modified = LocalDateTime.now();
}
/**
* @param jsonWriter
* writer to save metadatea to
* @param onlyIfDirty
* true to not write unchanged metadata
* @throws JSONException
*/
@Override
public void write(JSONWriter jsonWriter, boolean onlyIfDirty) throws JSONException {
if (!onlyIfDirty || isDirty()) {
Properties options = new Properties();
options.setProperty("mode", "save");
write(jsonWriter, options);
}
}
protected static boolean propertyExists(Object bean, String property) {
return PropertyUtils.isReadable(bean, property) &&
PropertyUtils.isWriteable(bean, property);

42
main/src/com/google/refine/model/metadata/DataPackageMetadata.java
View File

@ -3,21 +3,20 @@ package com.google.refine.model.metadata;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.everit.json.schema.ValidationException;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonRawValue;
import com.fasterxml.jackson.annotation.JsonValue;
import io.frictionlessdata.datapackage.Package;
import io.frictionlessdata.datapackage.Resource;
import io.frictionlessdata.datapackage.exceptions.DataPackageException;
@ -30,6 +29,12 @@ public class DataPackageMetadata extends AbstractMetadata {
private Package _pkg;
@JsonValue
@JsonRawValue
public String getJson() {
return _pkg.getJson().toString();
}
public DataPackageMetadata() {
setFormatName(MetadataFormat.DATAPACKAGE_METADATA);
@ -61,30 +66,6 @@ public class DataPackageMetadata extends AbstractMetadata {
loadFromJSON(new JSONObject(jsonString));
}
/**
* Write the package to a json file.
*/
@Override
public void writeToFile(File metadataFile) {
try {
this._pkg.save(metadataFile.getAbsolutePath());
} catch (IOException e) {
logger.error("IO exception when writing to file " + metadataFile.getAbsolutePath(),
ExceptionUtils.getStackTrace(e));
} catch (DataPackageException e) {
logger.error("Data package exception when writing to file " + metadataFile.getAbsolutePath(),
ExceptionUtils.getStackTrace(e));
}
}
@Override
public void write(JSONWriter jsonWriter, Properties options)
throws JSONException {
StringWriter sw = new StringWriter();
_pkg.getJson().write(sw);
jsonWriter = new JSONWriter(sw);
}
@Override
public void loadFromStream(InputStream inputStream) {
try {
@ -107,11 +88,6 @@ public class DataPackageMetadata extends AbstractMetadata {
return listResources;
}
@Override
public JSONObject getJSON() {
return _pkg.getJson();
}
public Package getPackage() {
return _pkg;

16
main/src/com/google/refine/model/metadata/IMetadata.java
View File

@ -5,30 +5,18 @@ import java.io.InputStream;
import java.time.LocalDateTime;
import java.util.List;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
/**
* Interface to import/export metadata
*/
public interface IMetadata extends Jsonizable {
public interface IMetadata {
public void loadFromJSON(JSONObject obj);
public void loadFromFile(File metadataFile);
public void loadFromStream(InputStream inputStream);
public void writeToFile(File metadataFile);
/**
* @param jsonWriter writer to save metadatea to
* @param onlyIfDirty true to not write unchanged metadata
* @throws JSONException
*/
public void write(JSONWriter jsonWriter, boolean onlyIfDirty);
public MetadataFormat getFormatName();
public void setFormatName(MetadataFormat format);
@ -38,7 +26,5 @@ public interface IMetadata extends Jsonizable {
public boolean isDirty();
public JSONObject getJSON();
public List<Exception> validate();
}

168
main/src/com/google/refine/model/metadata/ProjectMetadata.java
View File

@ -64,10 +64,19 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonRawValue;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.google.refine.ProjectManager;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.preference.TopList;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.JsonViews;
import com.google.refine.util.ParsingUtilities;
public class ProjectMetadata extends AbstractMetadata {
@ -75,36 +84,77 @@ public class ProjectMetadata extends AbstractMetadata {
final public static String TEMP_FILE_NAME = "metadata.temp.json";
final public static String OLD_FILE_NAME = "metadata.old.json";
@JsonProperty("created")
private final LocalDateTime _created;
@JsonProperty("name")
private String _name = "";
@JsonProperty("password")
@JsonView(JsonViews.SaveMode.class)
private String _password = "";
@JsonProperty("encoding")
@JsonView(JsonViews.SaveMode.class)
private String _encoding = "";
@JsonProperty("encodingConfidence")
@JsonView(JsonViews.SaveMode.class)
private int _encodingConfidence;
@JsonProperty("rowCount")
private int _rowCount;
// user metadata
private JSONArray _userMetadata = new JSONArray();;
private JSONArray _userMetadata = new JSONArray();
// _tags maps to keywords of the data package metadata
@JsonProperty("tags")
private String[] _tags = new String[0];
@JsonProperty("creator")
private String _creator = "";
@JsonProperty("contributors")
private String _contributors = "";
@JsonProperty("subject")
private String _subject = ""; // Several refine projects may be linked
@JsonProperty("description")
private String _description = ""; // free form of comment
// import options is an array for 1-n data sources
private JSONArray _importOptionMetadata = new JSONArray();
@JsonUnwrapped
private Map<String, Serializable> _customMetadata = new HashMap<String, Serializable>();
@JsonProperty("preferences")
@JsonView(JsonViews.SaveMode.class)
private PreferenceStore _preferenceStore = new PreferenceStore();
// below 5 fields are from data package metadata:
@JsonProperty("title")
private String title = "";
@JsonProperty("homepage")
private String homepage;
@JsonProperty("image")
private String image = "";
@JsonProperty("license")
private String license = "";
@JsonProperty("version")
private String version = "";
@JsonProperty("userMetadata")
@JsonRawValue
@JsonInclude(Include.NON_NULL)
public String getJsonUserMetadata() {
if (_userMetadata.length() > 0) {
return _userMetadata.toString();
}
return null;
}
@JsonProperty("importOptionMetadata")
@JsonRawValue
@JsonInclude(Include.NON_NULL)
public String getJsonImportOptionMetadata() {
if (_importOptionMetadata.length() > 0) {
return _importOptionMetadata.toString();
}
return null;
}
private final static Logger logger = LoggerFactory.getLogger("project_metadata");
@ -130,86 +180,12 @@ public class ProjectMetadata extends AbstractMetadata {
updateModified();
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("name");
writer.value(_name);
writer.key("tags");
writer.array();
for (String tag : _tags) {
writer.value(tag);
}
writer.endArray();
writer.key("created"); writer.value(ParsingUtilities.localDateToString(_created));
writer.key("modified"); writer.value(ParsingUtilities.localDateToString(_modified));
writer.key("creator"); writer.value(_creator);
writer.key("contributors"); writer.value(_contributors);
writer.key("subject"); writer.value(_subject);
writer.key("description"); writer.value(_description);
writer.key("rowCount"); writer.value(_rowCount);
writer.key("title"); writer.value(title);
writer.key("homepage"); writer.value(homepage);
writer.key("image"); writer.value(image);
writer.key("license"); writer.value(license);
writer.key("version"); writer.value(version);
writer.key("customMetadata");
writer.object();
for (String key : _customMetadata.keySet()) {
Serializable value = _customMetadata.get(key);
writer.key(key);
writer.value(value);
}
writer.endObject();
// write JSONArray directly
if (_importOptionMetadata.length() > 0) {
writer.key("importOptionMetadata");
writer.value(_importOptionMetadata);
}
// write user metadata in {name, value, display} form:
if (_userMetadata.length() > 0) {
writer.key(PreferenceStore.USER_METADATA_KEY);
writer.value(_userMetadata);
}
if (isSaveMode(options)) {
writer.key("password");
writer.value(_password);
writer.key("encoding");
writer.value(_encoding);
writer.key("encodingConfidence");
writer.value(_encodingConfidence);
writer.key("preferences");
_preferenceStore.write(writer, options);
}
writer.endObject();
if (isSaveMode(options)) {
written = LocalDateTime.now();
}
}
public void writeWithoutOption(JSONWriter writer)
throws JSONException {
write(writer, new Properties());
}
private boolean isSaveMode(Properties options) {
return "save".equals(options.getProperty("mode"));
}
public void write(JSONWriter jsonWriter)
throws JSONException {
write(jsonWriter, false);
@JsonProperty("saveModeWritten")
@JsonView(JsonViews.SaveMode.class)
@JsonInclude(Include.NON_NULL)
public String setSaveModeWritten() {
written = LocalDateTime.now();
return null;
}
public void loadFromJSON(JSONObject obj) {
@ -555,25 +531,6 @@ public class ProjectMetadata extends AbstractMetadata {
loadFromStream(targetStream);
}
@Override
public void writeToFile(File metadataFile) {
Writer writer = null;
try {
writer = new OutputStreamWriter(new FileOutputStream(metadataFile));
JSONWriter jsonWriter = new JSONWriter(writer);
write(jsonWriter, false);
} catch (FileNotFoundException e) {
logger.error(ExceptionUtils.getStackTrace(e));
} finally {
try {
writer.close();
} catch (IOException e) {
logger.error(ExceptionUtils.getStackTrace(e));
}
}
}
@Override
public void loadFromStream(InputStream inputStream) {
try (InputStreamReader reader = new InputStreamReader(inputStream)) {
@ -586,15 +543,6 @@ public class ProjectMetadata extends AbstractMetadata {
}
}
@Override
public JSONObject getJSON() {
StringWriter writer = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(writer);
writeWithoutOption(jsonWriter);
return new JSONObject(jsonWriter.toString());
}
@Override
public List<Exception> validate() {
return null;

11
main/src/com/google/refine/model/metadata/validator/ValidateOperation.java
View File

@ -1,10 +1,6 @@
package com.google.refine.model.metadata.validator;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
@ -22,11 +18,4 @@ public class ValidateOperation extends AbstractOperation {
return ValidatorInspector.inspect(project, options);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
// TODO Auto-generated method stub
}
}

9
main/src/com/google/refine/model/recon/DataExtensionReconConfig.java
View File

@ -33,21 +33,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model.recon;
import java.util.List;
import java.util.Properties;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.model.ReconType;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconType;
import com.google.refine.model.Row;
import com.google.refine.model.recon.StandardReconConfig;
import com.google.refine.model.recon.ReconJob;
public class DataExtensionReconConfig extends StandardReconConfig {
final public ReconType type;

15
main/src/com/google/refine/model/recon/ReconConfig.java
View File

@ -33,31 +33,29 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model.recon;
import java.io.IOException;
import java.io.Writer;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Row;
import com.google.refine.util.ParsingUtilities;
import edu.mit.simile.butterfly.ButterflyModule;
abstract public class ReconConfig implements Jsonizable {
abstract public class ReconConfig {
final static protected Logger LOGGER = LoggerFactory.getLogger("recon-config");
static final public Map<String, List<Class<? extends ReconConfig>>> s_opNameToClass =
@ -125,11 +123,10 @@ abstract public class ReconConfig implements Jsonizable {
abstract public Recon createNewRecon(long historyEntryID);
public void save(Writer writer) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
write(jsonWriter, new Properties());
} catch (JSONException e) {
LOGGER.error("Save failed",e);
ParsingUtilities.defaultWriter.writeValue(writer, this);
} catch (IOException e) {
e.printStackTrace();
}
}

88
main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java
View File

@ -47,35 +47,35 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.Jsonizable;
import com.google.refine.expr.functions.ToDate;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.ReconType;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.JsonViews;
import com.google.refine.util.ParsingUtilities;
public class ReconciledDataExtensionJob {
static public class DataExtensionProperty implements Jsonizable {
static public class DataExtensionProperty {
@JsonProperty("id")
public final String id;
@JsonProperty("name")
@JsonView(JsonViews.NonSaveMode.class)
public final String name;
@JsonProperty("settings")
@JsonInclude(Include.NON_NULL)
@ -92,31 +92,10 @@ public class ReconciledDataExtensionJob {
this.id = id;
this.name = name;
this.settings = settings;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(id);
if(!"query".equals(options.getProperty("mode"))) {
writer.key("name"); writer.value(name);
}
if (settings != null) {
writer.key("settings");
writer.object();
for(Map.Entry<String, Object> entry : settings.entrySet()) {
writer.key(entry.getKey());
writer.value(entry.getValue());
}
writer.endObject();
}
writer.endObject();
}
}
}
static public class DataExtensionConfig implements Jsonizable {
static public class DataExtensionConfig {
@JsonProperty("properties")
public final List<DataExtensionProperty> properties;
@ -135,22 +114,7 @@ public class ReconciledDataExtensionJob {
} catch(IOException e) {
throw new JSONException(e.toString());
}
}
@Override
public void write(JSONWriter jsonWriter, Properties options)
throws JSONException {
jsonWriter.object();
jsonWriter.key("properties");
jsonWriter.array();
for (DataExtensionProperty property : properties) {
property.write(jsonWriter, options);
}
jsonWriter.endArray();
jsonWriter.endObject();
}
}
}
static public class DataExtensionQuery extends DataExtensionConfig {
@ -166,34 +130,7 @@ public class ReconciledDataExtensionJob {
List<DataExtensionProperty> properties) {
super(properties);
this.ids = ids;
}
@Override
public void write(JSONWriter jsonWriter, Properties options)
throws JSONException {
jsonWriter.object();
if(ids != null) {
jsonWriter.key("ids");
jsonWriter.array();
for (String id : ids) {
if (id != null) {
jsonWriter.value(id);
}
}
jsonWriter.endArray();
}
jsonWriter.key("properties");
jsonWriter.array();
for (DataExtensionProperty property : properties) {
property.write(jsonWriter, options);
}
jsonWriter.endArray();
jsonWriter.endObject();
}
}
}
static public class DataExtension {
@ -382,12 +319,9 @@ public class ReconciledDataExtensionJob {
}
static protected void formulateQuery(Set<String> ids, DataExtensionConfig node, Writer writer) throws JSONException {
JSONWriter jsonWriter = new JSONWriter(writer);
Properties options = new Properties();
static protected void formulateQuery(Set<String> ids, DataExtensionConfig node, Writer writer) throws IOException {
DataExtensionQuery query = new DataExtensionQuery(ids.stream().collect(Collectors.toList()), node.properties);
options.setProperty("mode", "query");
query.write(jsonWriter, options);
ParsingUtilities.saveWriter.writeValue(writer, query);
}
static protected void gatherColumnInfo(JSONArray meta, List<ColumnInfo> columns) throws JSONException {

44
main/src/com/google/refine/model/recon/StandardReconConfig.java
View File

@ -42,7 +42,6 @@ import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
@ -56,7 +55,6 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
@ -71,7 +69,7 @@ import com.google.refine.util.ParsingUtilities;
public class StandardReconConfig extends ReconConfig {
final static Logger logger = LoggerFactory.getLogger("refine-standard-recon");
static public class ColumnDetail implements Jsonizable {
static public class ColumnDetail {
@JsonProperty("column")
final public String columnName;
@JsonProperty("propertyName")
@ -84,17 +82,6 @@ public class StandardReconConfig extends ReconConfig {
this.propertyName = propertyName;
this.propertyID = propertyID;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("column"); writer.value(columnName);
writer.key("propertyName"); writer.value(propertyName);
writer.key("propertyID"); writer.value(propertyID);
writer.endObject();
}
}
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
@ -214,35 +201,6 @@ public class StandardReconConfig extends ReconConfig {
this.columnDetails = columnDetails;
this.limit = limit;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("mode"); writer.value("standard-service");
writer.key("service"); writer.value(service);
writer.key("identifierSpace"); writer.value(identifierSpace);
writer.key("schemaSpace"); writer.value(schemaSpace);
writer.key("type");
if (typeID == null) {
writer.value(null);
} else {
writer.object();
writer.key("id"); writer.value(typeID);
writer.key("name"); writer.value(typeName);
writer.endObject();
}
writer.key("autoMatch"); writer.value(autoMatch);
writer.key("columnDetails");
writer.array();
for (ColumnDetail c : columnDetails) {
c.write(writer, options);
}
writer.endArray();
writer.key("limit"); writer.value(limit);
writer.endObject();
}
@JsonProperty("type")
public ReconType getReconType() {

16
main/src/com/google/refine/operations/cell/BlankDownOperation.java
View File

@ -34,11 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.operations.cell;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.browsing.Engine.Mode;
import com.google.refine.browsing.EngineConfig;
@ -51,7 +48,6 @@ import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.changes.CellChange;
import com.google.refine.operations.EngineDependentMassCellOperation;
import com.google.refine.operations.OperationRegistry;
public class BlankDownOperation extends EngineDependentMassCellOperation {
@ -71,18 +67,6 @@ public class BlankDownOperation extends EngineDependentMassCellOperation {
super(engineConfig, columnName, true);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); getEngineConfig().write(writer, options);
writer.key("columnName"); writer.value(_columnName);
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Blank down cells in column " + _columnName;

16
main/src/com/google/refine/operations/cell/FillDownOperation.java
View File

@ -34,11 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.operations.cell;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.Engine.Mode;
@ -52,7 +49,6 @@ import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.changes.CellChange;
import com.google.refine.operations.EngineDependentMassCellOperation;
import com.google.refine.operations.OperationRegistry;
public class FillDownOperation extends EngineDependentMassCellOperation {
@ -72,18 +68,6 @@ public class FillDownOperation extends EngineDependentMassCellOperation {
super(engineConfig, columnName, true);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); getEngineConfig().write(writer, options);
writer.key("columnName"); writer.value(_columnName);
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Fill down cells in column " + _columnName;

20
main/src/com/google/refine/operations/cell/KeyValueColumnizeOperation.java
View File

@ -37,11 +37,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -53,7 +50,6 @@ import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.changes.MassRowColumnChange;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.JSONUtilities;
public class KeyValueColumnizeOperation extends AbstractOperation {
@ -78,22 +74,6 @@ public class KeyValueColumnizeOperation extends AbstractOperation {
_valueColumnName = valueColumnName;
_noteColumnName = noteColumnName;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(
"Columnize by key column " +
_keyColumnName + " and value column " + _valueColumnName +
(_noteColumnName != null ? (" with note column " + _noteColumnName) : ""));
writer.key("keyColumnName"); writer.value(_keyColumnName);
writer.key("valueColumnName"); writer.value(_valueColumnName);
writer.key("noteColumnName"); writer.value(_noteColumnName);
writer.endObject();
}
@JsonProperty("keyColumnName")
public String getKeyColumnName() {

42
main/src/com/google/refine/operations/cell/MassEditOperation.java
View File

@ -41,13 +41,10 @@ import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.expr.Evaluable;
@ -60,7 +57,6 @@ import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.changes.CellChange;
import com.google.refine.operations.EngineDependentMassCellOperation;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.StringUtils;
@ -68,7 +64,7 @@ public class MassEditOperation extends EngineDependentMassCellOperation {
final protected String _expression;
final protected List<Edit> _edits;
static public class Edit implements Jsonizable {
static public class Edit {
@JsonProperty("from")
final public List<String> from;
@JsonProperty("fromBlank")
@ -84,23 +80,6 @@ public class MassEditOperation extends EngineDependentMassCellOperation {
this.fromError = fromError;
this.to = to;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("fromBlank"); writer.value(fromBlank);
writer.key("fromError"); writer.value(fromError);
writer.key("from");
writer.array();
for (String s : from) {
writer.value(s);
}
writer.endArray();
writer.key("to"); writer.value(to);
writer.endObject();
}
}
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
@ -157,25 +136,6 @@ public class MassEditOperation extends EngineDependentMassCellOperation {
_expression = expression;
_edits = edits;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); getEngineConfig().write(writer, options);
writer.key("columnName"); writer.value(_columnName);
writer.key("expression"); writer.value(_expression);
writer.key("edits");
writer.array();
for (Edit edit : _edits) {
edit.write(writer, options);
}
writer.endArray();
writer.endObject();
}
@JsonProperty("expression")
public String getExpression() {

17
main/src/com/google/refine/operations/cell/MultiValuedCellJoinOperation.java
View File

@ -35,11 +35,8 @@ package com.google.refine.operations.cell;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -51,7 +48,6 @@ import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.changes.MassRowChange;
import com.google.refine.operations.OperationRegistry;
public class MultiValuedCellJoinOperation extends AbstractOperation {
final protected String _columnName;
@ -75,19 +71,6 @@ public class MultiValuedCellJoinOperation extends AbstractOperation {
_keyColumnName = keyColumnName;
_separator = separator;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("columnName"); writer.value(_columnName);
writer.key("keyColumnName"); writer.value(_keyColumnName);
writer.key("separator"); writer.value(_separator);
writer.endObject();
}
@JsonProperty("columnName")
public String getColumnName() {

27
main/src/com/google/refine/operations/cell/MultiValuedCellSplitOperation.java
View File

@ -35,13 +35,10 @@ package com.google.refine.operations.cell;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
@ -54,7 +51,6 @@ import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.changes.MassRowChange;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.JSONUtilities;
public class MultiValuedCellSplitOperation extends AbstractOperation {
@ -148,29 +144,6 @@ public class MultiValuedCellSplitOperation extends AbstractOperation {
return _fieldLengths;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value("Split multi-valued cells in column " + _columnName);
writer.key("columnName"); writer.value(_columnName);
writer.key("keyColumnName"); writer.value(_keyColumnName);
writer.key("mode"); writer.value(_mode);
if ("separator".equals(_mode)) {
writer.key("separator"); writer.value(_separator);
writer.key("regex"); writer.value(_regex);
} else {
writer.key("fieldLengths"); writer.array();
for (int l : _fieldLengths) {
writer.value(l);
}
writer.endArray();
}
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Split multi-valued cells in column " + _columnName;

19
main/src/com/google/refine/operations/cell/TextTransformOperation.java
View File

@ -37,9 +37,7 @@ import java.io.Serializable;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.RowVisitor;
@ -55,7 +53,6 @@ import com.google.refine.model.Row;
import com.google.refine.model.changes.CellChange;
import com.google.refine.operations.EngineDependentMassCellOperation;
import com.google.refine.operations.OnError;
import com.google.refine.operations.OperationRegistry;
public class TextTransformOperation extends EngineDependentMassCellOperation {
final protected String _expression;
@ -110,22 +107,6 @@ public class TextTransformOperation extends EngineDependentMassCellOperation {
_repeatCount = repeatCount;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); getEngineConfig().write(writer, options);
writer.key("columnName"); writer.value(_columnName);
writer.key("expression"); writer.value(_expression);
writer.key("onError"); writer.value(onErrorToString(_onError));
writer.key("repeat"); writer.value(_repeat);
writer.key("repeatCount"); writer.value(_repeatCount);
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Text transform on cells in column " + _columnName + " using expression " + _expression;

26
main/src/com/google/refine/operations/cell/TransposeColumnsIntoRowsOperation.java
View File

@ -35,11 +35,8 @@ package com.google.refine.operations.cell;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
@ -48,7 +45,6 @@ import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.changes.MassRowColumnChange;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.JSONUtilities;
public class TransposeColumnsIntoRowsOperation extends AbstractOperation {
@ -131,28 +127,6 @@ public class TransposeColumnsIntoRowsOperation extends AbstractOperation {
_valueColumnName = valueColumnName;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription());
writer.key("startColumnName"); writer.value(_startColumnName);
writer.key("columnCount"); writer.value(_columnCount);
writer.key("ignoreBlankCells"); writer.value(_ignoreBlankCells);
writer.key("fillDown"); writer.value(_fillDown);
if (_combinedColumnName != null) {
writer.key("combinedColumnName"); writer.value(_combinedColumnName);
writer.key("prependColumnName"); writer.value(_prependColumnName);
writer.key("separator"); writer.value(_separator);
} else {
writer.key("keyColumnName"); writer.value(_keyColumnName);
writer.key("valueColumnName"); writer.value(_valueColumnName);
}
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return getBriefDescription();

16
main/src/com/google/refine/operations/cell/TransposeRowsIntoColumnsOperation.java
View File

@ -35,11 +35,8 @@ package com.google.refine.operations.cell;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -50,7 +47,6 @@ import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.changes.MassRowColumnChange;
import com.google.refine.operations.OperationRegistry;
public class TransposeRowsIntoColumnsOperation extends AbstractOperation {
final protected String _columnName;
@ -70,18 +66,6 @@ public class TransposeRowsIntoColumnsOperation extends AbstractOperation {
_columnName = columnName;
_rowCount = rowCount;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value("Transpose every " + _rowCount + " cells in column " + _columnName + " into separate columns");
writer.key("columnName"); writer.value(_columnName);
writer.key("rowCount"); writer.value(_rowCount);
writer.endObject();
}
@JsonProperty("rowCount")
public int getRowCount() {

40
main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java
View File

@ -48,7 +48,6 @@ import java.util.concurrent.TimeUnit;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -56,7 +55,6 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.refine.Jsonizable;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.FilteredRows;
@ -76,7 +74,6 @@ import com.google.refine.model.changes.CellAtRow;
import com.google.refine.model.changes.ColumnAdditionChange;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OnError;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.operations.cell.TextTransformOperation;
import com.google.refine.process.LongRunningProcess;
import com.google.refine.process.Process;
@ -84,7 +81,7 @@ import com.google.refine.util.ParsingUtilities;
public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperation {
public static final class HttpHeader implements Jsonizable {
public static final class HttpHeader {
@JsonProperty("name")
final public String name;
@JsonProperty("value")
@ -94,15 +91,6 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
this.name = name;
this.value = value;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("name"); writer.value(name);
writer.key("value"); writer.value(value);
writer.endObject();
}
}
final protected String _baseColumnName;
@ -166,32 +154,6 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
_cacheResponses = cacheResponses;
_httpHeadersJson = httpHeadersJson;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); getEngineConfig().write(writer, options);
writer.key("newColumnName"); writer.value(_newColumnName);
writer.key("columnInsertIndex"); writer.value(_columnInsertIndex);
writer.key("baseColumnName"); writer.value(_baseColumnName);
writer.key("urlExpression"); writer.value(_urlExpression);
writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError));
writer.key("delay"); writer.value(_delay);
writer.key("cacheResponses"); writer.value(_cacheResponses);
if (_httpHeadersJson != null) {
writer.key("httpHeadersJson");
writer.array();
for(HttpHeader header : _httpHeadersJson) {
header.write(writer, options);
}
writer.endArray();
}
writer.endObject();
}
@JsonProperty("newColumnName")
public String getNewColumnName() {

19
main/src/com/google/refine/operations/column/ColumnAdditionOperation.java
View File

@ -38,9 +38,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -63,7 +61,6 @@ import com.google.refine.model.changes.CellAtRow;
import com.google.refine.model.changes.ColumnAdditionChange;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OnError;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.operations.cell.TextTransformOperation;
public class ColumnAdditionOperation extends EngineDependentOperation {
@ -104,22 +101,6 @@ public class ColumnAdditionOperation extends EngineDependentOperation {
_newColumnName = newColumnName;
_columnInsertIndex = columnInsertIndex;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("engineConfig"); getEngineConfig().write(writer, options);
writer.key("newColumnName"); writer.value(_newColumnName);
writer.key("columnInsertIndex"); writer.value(_columnInsertIndex);
writer.key("baseColumnName"); writer.value(_baseColumnName);
writer.key("expression"); writer.value(_expression);
writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError));
writer.endObject();
}
@JsonProperty("newColumnName")
public String getNewColumnName() {

17
main/src/com/google/refine/operations/column/ColumnMoveOperation.java
View File

@ -33,11 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.operations.column;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -46,7 +42,6 @@ import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.model.changes.ColumnMoveChange;
import com.google.refine.operations.OperationRegistry;
public class ColumnMoveOperation extends AbstractOperation {
final protected String _columnName;
@ -67,18 +62,6 @@ public class ColumnMoveOperation extends AbstractOperation {
_index = index;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value("Move column " + _columnName + " to position " + _index);
writer.key("columnName"); writer.value(_columnName);
writer.key("index"); writer.value(_index);
writer.endObject();
}
@JsonProperty("columnName")
public String getColumnName() {
return _columnName;

16
main/src/com/google/refine/operations/column/ColumnRemovalOperation.java
View File

@ -33,11 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.operations.column;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -47,7 +43,6 @@ import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.changes.ColumnRemovalChange;
import com.google.refine.operations.OperationRegistry;
public class ColumnRemovalOperation extends AbstractOperation {
final protected String _columnName;
@ -64,17 +59,6 @@ public class ColumnRemovalOperation extends AbstractOperation {
_columnName = columnName;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value("Remove column " + _columnName);
writer.key("columnName"); writer.value(_columnName);
writer.endObject();
}
@JsonProperty("columnName")
public String getColumnName() {
return _columnName;

17
main/src/com/google/refine/operations/column/ColumnRenameOperation.java
View File

@ -33,11 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.operations.column;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -46,7 +42,6 @@ import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.model.changes.ColumnRenameChange;
import com.google.refine.operations.OperationRegistry;
public class ColumnRenameOperation extends AbstractOperation {
final protected String _oldColumnName;
@ -67,18 +62,6 @@ public class ColumnRenameOperation extends AbstractOperation {
_newColumnName = newColumnName;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value("Rename column " + _oldColumnName + " to " + _newColumnName);
writer.key("oldColumnName"); writer.value(_oldColumnName);
writer.key("newColumnName"); writer.value(_newColumnName);
writer.endObject();
}
@JsonProperty("oldColumnName")
public String getOldColumnName() {
return _oldColumnName;

Some files were not shown because too many files have changed in this diff Show More