From a4fa1dca77cf2cdc627aaedfb9fe2ae3f34d60ea Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Wed, 21 Nov 2018 10:28:13 +0000 Subject: [PATCH] Update Wikidata extension after removal of org.json from core --- .../commands/SaveWikibaseSchemaCommand.java | 4 +-- .../PerformWikibaseEditsOperation.java | 21 +++++---------- .../SaveWikibaseSchemaOperation.java | 15 +++++------ .../wikidata/schema/WikibaseSchema.java | 21 +++------------ .../PreviewWikibaseSchemaCommandTest.java | 18 +++++-------- .../wikidata/commands/SchemaCommandTest.java | 4 ++- .../QuickStatementsExporterTest.java | 6 ++--- .../wikidata/operations/OperationTest.java | 8 +++--- .../PerformWikibaseEditsOperationTest.java | 9 +++---- .../SaveWikibaseSchemaOperationTest.java | 10 +++---- .../wikidata/schema/WikibaseSchemaTest.java | 26 +++++++++---------- .../wikidata/testing/TestingData.java | 4 +-- 12 files changed, 57 insertions(+), 89 deletions(-) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommand.java b/extensions/wikidata/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommand.java index 2dc44a39e..2470b9426 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommand.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommand.java @@ -33,7 +33,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.json.JSONException; -import org.json.JSONObject; import org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation; import org.openrefine.wikidata.schema.WikibaseSchema; @@ -58,8 +57,7 @@ public class SaveWikibaseSchemaCommand extends Command { return; } - JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); - WikibaseSchema schema = WikibaseSchema.reconstruct(json); + WikibaseSchema schema = ParsingUtilities.mapper.readValue(jsonString, WikibaseSchema.class); AbstractOperation op = new SaveWikibaseSchemaOperation(schema); Process process = op.createProcess(project, new Properties()); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java b/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java index 9ae696a7a..218d331ac 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java @@ -31,7 +31,6 @@ import java.util.Properties; import java.util.Random; import org.apache.commons.lang.Validate; -import org.json.JSONObject; import org.openrefine.wikidata.editing.ConnectionManager; import org.openrefine.wikidata.editing.EditBatchProcessor; import org.openrefine.wikidata.editing.NewItemLibrary; @@ -44,6 +43,7 @@ import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor; import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; @@ -51,7 +51,6 @@ import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.history.Change; import com.google.refine.history.HistoryEntry; -import com.google.refine.model.AbstractOperation; import com.google.refine.model.Project; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.process.LongRunningProcess; @@ -65,24 +64,18 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation { @JsonProperty("summary") private String summary; - public PerformWikibaseEditsOperation(EngineConfig engineConfig, String summary) { + @JsonCreator + public PerformWikibaseEditsOperation( + @JsonProperty("engineConfig") + EngineConfig engineConfig, + @JsonProperty("summary") + String summary) { super(engineConfig); Validate.notNull(summary, "An edit summary must be provided."); Validate.notEmpty(summary, "An edit summary must be provided."); this.summary = summary; } - static public AbstractOperation reconstruct(Project project, JSONObject obj) - throws Exception { - JSONObject engineConfig = obj.getJSONObject("engineConfig"); - String summary = null; - if (obj.has("summary")) { - summary = obj.getString("summary"); - } - return new PerformWikibaseEditsOperation( - EngineConfig.reconstruct(engineConfig), summary); - } - @Override protected String getBriefDescription(Project project) { return "Perform Wikibase edits"; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java b/extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java index 13e043bd1..d5f39166d 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java @@ -31,6 +31,7 @@ import java.util.Properties; import org.json.JSONObject; import org.openrefine.wikidata.schema.WikibaseSchema; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -48,16 +49,14 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation { @JsonProperty("schema") final protected WikibaseSchema _schema; - public SaveWikibaseSchemaOperation(WikibaseSchema schema) { + @JsonCreator + public SaveWikibaseSchemaOperation( + @JsonProperty("schema") + WikibaseSchema schema) { this._schema = schema; } - static public AbstractOperation reconstruct(Project project, JSONObject obj) - throws Exception { - return new SaveWikibaseSchemaOperation(WikibaseSchema.reconstruct(obj.getJSONObject("schema"))); - } - @Override protected String getBriefDescription(Project project) { return operationDescription; @@ -123,9 +122,9 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation { String value = line.substring(equal + 1); if ("oldSchema".equals(field) && value.length() > 0) { - oldSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); + oldSchema = ParsingUtilities.mapper.readValue(value, WikibaseSchema.class); } else if ("newSchema".equals(field) && value.length() > 0) { - newSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); + newSchema = ParsingUtilities.mapper.readValue(value, WikibaseSchema.class); } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java index 48a54d470..83e61e2bb 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java @@ -49,6 +49,7 @@ import com.google.refine.browsing.RowVisitor; import com.google.refine.model.OverlayModel; import com.google.refine.model.Project; import com.google.refine.model.Row; +import com.google.refine.util.ParsingUtilities; /** * Main class representing a skeleton of Wikibase edits with OpenRefine columns @@ -182,26 +183,12 @@ public class WikibaseSchema implements OverlayModel { ; } } - - static public WikibaseSchema reconstruct(JSONObject o) - throws JSONException { - return reconstruct(o.toString()); - } - static public WikibaseSchema reconstruct(String json) throws JSONException { - ObjectMapper mapper = new ObjectMapper(); - try { - return mapper.readValue(json, WikibaseSchema.class); - } catch (JsonParseException e) { - throw new JSONException(e.toString()); - } catch (JsonMappingException e) { - throw new JSONException(e.toString()); - } catch (IOException e) { - throw new JSONException(e.toString()); - } + static public WikibaseSchema reconstruct(String json) throws IOException { + return ParsingUtilities.mapper.readValue(json, WikibaseSchema.class); } - static public WikibaseSchema load(Project project, JSONObject obj) + static public WikibaseSchema load(Project project, String obj) throws Exception { return reconstruct(obj); } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommandTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommandTest.java index 81596105e..483b9519a 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommandTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommandTest.java @@ -31,35 +31,31 @@ import java.io.IOException; import javax.servlet.ServletException; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.junit.Assert; -import org.openrefine.wikidata.testing.TestingData; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.util.ParsingUtilities; public class PreviewWikibaseSchemaCommandTest extends SchemaCommandTest { @BeforeMethod - public void SetUp() - throws JSONException { + public void SetUp() { command = new PreviewWikibaseSchemaCommand(); } @Test public void testValidSchema() - throws JSONException, IOException, ServletException { + throws IOException, ServletException { String schemaJson = jsonFromFile("schema/inception.json").toString(); when(request.getParameter("schema")).thenReturn(schemaJson); command.doPost(request, response); - JSONObject response = ParsingUtilities.evaluateJsonStringToObject(writer.toString()); - JSONArray edits = response.getJSONArray("edits_preview"); - assertEquals(3, edits.length()); + ObjectNode response = ParsingUtilities.evaluateJsonStringToObjectNode(writer.toString()); + ArrayNode edits = (ArrayNode) response.get("edits_preview"); + assertEquals(3, edits.size()); } } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/SchemaCommandTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/SchemaCommandTest.java index e85f819c6..84e541d00 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/SchemaCommandTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/SchemaCommandTest.java @@ -32,6 +32,8 @@ import javax.servlet.ServletException; import org.testng.annotations.Test; +import com.google.refine.util.ParsingUtilities; + public abstract class SchemaCommandTest extends CommandTest { @Test @@ -48,6 +50,6 @@ public abstract class SchemaCommandTest extends CommandTest { when(request.getParameter("schema")).thenReturn("{bogus json"); command.doPost(request, response); - assertEquals("{\"code\":\"error\",\"message\":\"Wikibase schema could not be parsed.\"}", writer.toString()); + assertEquals("error", ParsingUtilities.mapper.readTree(writer.toString()).get("code").asText()); } } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java index b5d56fbbd..7c04de840 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java @@ -31,8 +31,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.Properties; -import org.json.JSONException; -import org.json.JSONObject; import org.openrefine.wikidata.schema.WikibaseSchema; import org.openrefine.wikidata.testing.TestingData; import org.openrefine.wikidata.updates.ItemUpdate; @@ -67,10 +65,10 @@ public class QuickStatementsExporterTest extends RefineTest { @Test public void testSimpleProject() - throws JSONException, IOException { + throws IOException { Project project = this.createCSVProject(TestingData.inceptionWithNewCsv); TestingData.reconcileInceptionCells(project); - JSONObject serialized = TestingData.jsonFromFile("schema/inception.json"); + String serialized = TestingData.jsonFromFile("schema/inception.json"); WikibaseSchema schema = WikibaseSchema.reconstruct(serialized); project.overlayModels.put("wikibaseSchema", schema); Engine engine = new Engine(project); diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java index a6969c0f0..6662ce0aa 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java @@ -32,8 +32,6 @@ import java.io.StringReader; import java.io.StringWriter; import java.util.Properties; -import org.json.JSONObject; -import org.openrefine.wikidata.testing.JacksonSerializationTest; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -69,17 +67,17 @@ public abstract class OperationTest extends RefineTest { public abstract AbstractOperation reconstruct() throws Exception; - public abstract JSONObject getJson() + public abstract String getJson() throws Exception; @Test public void testReconstruct() throws Exception { - JSONObject json = getJson(); + String json = getJson(); AbstractOperation op = reconstruct(); StringWriter writer = new StringWriter(); ParsingUtilities.defaultWriter.writeValue(writer, op); - TestUtils.assertEqualAsJson(json.toString(), writer.toString()); + TestUtils.assertEqualAsJson(json, writer.toString()); } protected LineNumberReader makeReader(String input) { diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperationTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperationTest.java index 828463780..7a0e10ab0 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperationTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperationTest.java @@ -27,7 +27,6 @@ import static org.junit.Assert.assertEquals; import java.io.LineNumberReader; -import org.json.JSONObject; import org.openrefine.wikidata.testing.TestingData; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -36,6 +35,7 @@ import com.google.refine.browsing.EngineConfig; import com.google.refine.history.Change; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Recon; +import com.google.refine.util.ParsingUtilities; public class PerformWikibaseEditsOperationTest extends OperationTest { @@ -47,19 +47,18 @@ public class PerformWikibaseEditsOperationTest extends OperationTest { @Override public AbstractOperation reconstruct() throws Exception { - JSONObject json = getJson(); - return PerformWikibaseEditsOperation.reconstruct(project, json); + return ParsingUtilities.mapper.readValue(getJson(), PerformWikibaseEditsOperation.class); } @Override - public JSONObject getJson() + public String getJson() throws Exception { return TestingData.jsonFromFile("operations/perform-edits.json"); } @Test(expectedExceptions=IllegalArgumentException.class) public void testConstructor() { - new PerformWikibaseEditsOperation(EngineConfig.reconstruct(new JSONObject("{}")), ""); + new PerformWikibaseEditsOperation(EngineConfig.reconstruct("{}"), ""); } @Test diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperationTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperationTest.java index 6c5e5db5a..8008cb282 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperationTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperationTest.java @@ -28,7 +28,6 @@ import static org.junit.Assert.assertNull; import java.io.LineNumberReader; -import org.json.JSONObject; import org.openrefine.wikidata.schema.WikibaseSchema; import org.openrefine.wikidata.testing.TestingData; import org.testng.annotations.BeforeMethod; @@ -36,6 +35,7 @@ import org.testng.annotations.Test; import com.google.refine.history.Change; import com.google.refine.model.AbstractOperation; +import com.google.refine.util.ParsingUtilities; public class SaveWikibaseSchemaOperationTest extends OperationTest { @@ -47,11 +47,11 @@ public class SaveWikibaseSchemaOperationTest extends OperationTest { @Override public AbstractOperation reconstruct() throws Exception { - return SaveWikibaseSchemaOperation.reconstruct(project, getJson()); + return ParsingUtilities.mapper.readValue(getJson(), SaveWikibaseSchemaOperation.class); } @Override - public JSONObject getJson() + public String getJson() throws Exception { return TestingData.jsonFromFile("operations/save-schema.json"); } @@ -59,8 +59,8 @@ public class SaveWikibaseSchemaOperationTest extends OperationTest { @Test public void testLoadChange() throws Exception { - JSONObject schemaJson = TestingData.jsonFromFile("schema/inception.json"); - String changeString = "newSchema=" + schemaJson.toString() + "\n" + "oldSchema=\n" + "/ec/"; + String schemaJson = TestingData.jsonFromFile("schema/inception.json"); + String changeString = "newSchema=" + schemaJson + "\n" + "oldSchema=\n" + "/ec/"; WikibaseSchema schema = WikibaseSchema.reconstruct(schemaJson); LineNumberReader reader = makeReader(changeString); diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java index 065136cb7..f342e4c73 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java @@ -31,8 +31,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import org.json.JSONException; -import org.json.JSONObject; import org.openrefine.wikidata.testing.TestingData; import org.openrefine.wikidata.updates.ItemUpdate; import org.openrefine.wikidata.updates.ItemUpdateBuilder; @@ -95,25 +93,25 @@ public class WikibaseSchemaTest extends RefineTest { @Test public void testSerialize() - throws JSONException, IOException { - JSONObject serialized = TestingData.jsonFromFile("schema/history_of_medicine.json"); + throws IOException { + String serialized = TestingData.jsonFromFile("schema/history_of_medicine.json"); WikibaseSchema parsed = WikibaseSchema.reconstruct(serialized); TestUtils.isSerializedTo(parsed, TestingData.jsonFromFile("schema/history_of_medicine_normalized.json").toString()); } @Test public void testDeserialize() - throws JSONException, IOException { + throws IOException { // this json file was generated by an earlier version of the software // it contains extra "type" fields that are now ignored. - JSONObject serialized = TestingData.jsonFromFile("schema/roarmap.json"); + String serialized = TestingData.jsonFromFile("schema/roarmap.json"); WikibaseSchema.reconstruct(serialized); } @Test public void testEvaluate() - throws JSONException, IOException { - JSONObject serialized = TestingData.jsonFromFile("schema/inception.json"); + throws IOException { + String serialized = TestingData.jsonFromFile("schema/inception.json"); WikibaseSchema schema = WikibaseSchema.reconstruct(serialized); Engine engine = new Engine(project); List updates = schema.evaluate(project, engine); @@ -125,8 +123,8 @@ public class WikibaseSchemaTest extends RefineTest { assertEquals(expected, updates); } - @Test(expectedExceptions = JSONException.class) - public void testDeserializeEmpty() throws JSONException { + @Test(expectedExceptions = IOException.class) + public void testDeserializeEmpty() throws IOException { String schemaJson = "{\"itemDocuments\":[{\"statementGroups\":[{\"statements\":[]}]," +"\"nameDescs\":[]}],\"wikibasePrefix\":\"http://www.wikidata.org/entity/\"}"; WikibaseSchema.reconstruct(schemaJson); @@ -134,11 +132,11 @@ public class WikibaseSchemaTest extends RefineTest { @Test public void testEvaluateRespectsFacets() - throws JSONException, IOException { - JSONObject serialized = TestingData.jsonFromFile("schema/inception.json"); + throws IOException { + String serialized = TestingData.jsonFromFile("schema/inception.json"); WikibaseSchema schema = WikibaseSchema.reconstruct(serialized); Engine engine = new Engine(project); - EngineConfig engineConfig = EngineConfig.reconstruct(new JSONObject("{\n" + EngineConfig engineConfig = EngineConfig.reconstruct("{\n" + " \"mode\": \"row-based\",\n" + " \"facets\": [\n" + " {\n" @@ -151,7 +149,7 @@ public class WikibaseSchemaTest extends RefineTest { + " \"columnName\": \"reference\"\n" + " }\n" + " ]\n" - + " }")); + + " }"); engine.initializeFromConfig(engineConfig); List updates = schema.evaluate(project, engine); List expected = new ArrayList<>(); diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/TestingData.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/TestingData.java index efa25756f..d5bd18bc7 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/TestingData.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/TestingData.java @@ -133,11 +133,11 @@ public class TestingData { return generateStatement(from, pid, to); } - public static JSONObject jsonFromFile(String filename) + public static String jsonFromFile(String filename) throws IOException, JSONException { InputStream f = TestingData.class.getClassLoader().getResourceAsStream(filename); String decoded = IOUtils.toString(f); - return ParsingUtilities.evaluateJsonStringToObject(decoded); + return decoded.trim(); } public static void reconcileInceptionCells(Project project) {