s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update Wikidata extension after removal of org.json from core

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-11-21 10:28:13 +00:00
parent 121661e8eb
commit a4fa1dca77
12 changed files with 57 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extensions/wikidata/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommand.java
View File

@ -33,7 +33,6 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONObject;
import org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation;
import org.openrefine.wikidata.schema.WikibaseSchema;
@ -58,8 +57,7 @@ public class SaveWikibaseSchemaCommand extends Command {
return;
}
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
WikibaseSchema schema = WikibaseSchema.reconstruct(json);
WikibaseSchema schema = ParsingUtilities.mapper.readValue(jsonString, WikibaseSchema.class);
AbstractOperation op = new SaveWikibaseSchemaOperation(schema);
Process process = op.createProcess(project, new Properties());

21
extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java
View File

@ -31,7 +31,6 @@ import java.util.Properties;
import java.util.Random;
import org.apache.commons.lang.Validate;
import org.json.JSONObject;
import org.openrefine.wikidata.editing.ConnectionManager;
import org.openrefine.wikidata.editing.EditBatchProcessor;
import org.openrefine.wikidata.editing.NewItemLibrary;
@ -44,6 +43,7 @@ import org.wikidata.wdtk.wikibaseapi.ApiConnection;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -51,7 +51,6 @@ import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.process.LongRunningProcess;
@ -65,24 +64,18 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation {
@JsonProperty("summary")
private String summary;
public PerformWikibaseEditsOperation(EngineConfig engineConfig, String summary) {
@JsonCreator
public PerformWikibaseEditsOperation(
@JsonProperty("engineConfig")
EngineConfig engineConfig,
@JsonProperty("summary")
String summary) {
super(engineConfig);
Validate.notNull(summary, "An edit summary must be provided.");
Validate.notEmpty(summary, "An edit summary must be provided.");
this.summary = summary;
}
static public AbstractOperation reconstruct(Project project, JSONObject obj)
throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
String summary = null;
if (obj.has("summary")) {
summary = obj.getString("summary");
}
return new PerformWikibaseEditsOperation(
EngineConfig.reconstruct(engineConfig), summary);
}
@Override
protected String getBriefDescription(Project project) {
return "Perform Wikibase edits";

15
extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java
View File

@ -31,6 +31,7 @@ import java.util.Properties;
import org.json.JSONObject;
import org.openrefine.wikidata.schema.WikibaseSchema;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -48,16 +49,14 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
@JsonProperty("schema")
final protected WikibaseSchema _schema;
public SaveWikibaseSchemaOperation(WikibaseSchema schema) {
@JsonCreator
public SaveWikibaseSchemaOperation(
@JsonProperty("schema")
WikibaseSchema schema) {
this._schema = schema;
}
static public AbstractOperation reconstruct(Project project, JSONObject obj)
throws Exception {
return new SaveWikibaseSchemaOperation(WikibaseSchema.reconstruct(obj.getJSONObject("schema")));
}
@Override
protected String getBriefDescription(Project project) {
return operationDescription;
@ -123,9 +122,9 @@ public class SaveWikibaseSchemaOperation extends AbstractOperation {
String value = line.substring(equal + 1);
if ("oldSchema".equals(field) && value.length() > 0) {
oldSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
oldSchema = ParsingUtilities.mapper.readValue(value, WikibaseSchema.class);
} else if ("newSchema".equals(field) && value.length() > 0) {
newSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
newSchema = ParsingUtilities.mapper.readValue(value, WikibaseSchema.class);
}
}

21
extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java
View File

@ -49,6 +49,7 @@ import com.google.refine.browsing.RowVisitor;
import com.google.refine.model.OverlayModel;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.util.ParsingUtilities;
/**
* Main class representing a skeleton of Wikibase edits with OpenRefine columns
@ -182,26 +183,12 @@ public class WikibaseSchema implements OverlayModel {
;
}
}
static public WikibaseSchema reconstruct(JSONObject o)
throws JSONException {
return reconstruct(o.toString());
}
static public WikibaseSchema reconstruct(String json) throws JSONException {
ObjectMapper mapper = new ObjectMapper();
try {
return mapper.readValue(json, WikibaseSchema.class);
} catch (JsonParseException e) {
throw new JSONException(e.toString());
} catch (JsonMappingException e) {
throw new JSONException(e.toString());
} catch (IOException e) {
throw new JSONException(e.toString());
}
static public WikibaseSchema reconstruct(String json) throws IOException {
return ParsingUtilities.mapper.readValue(json, WikibaseSchema.class);
}
static public WikibaseSchema load(Project project, JSONObject obj)
static public WikibaseSchema load(Project project, String obj)
throws Exception {
return reconstruct(obj);
}

18
extensions/wikidata/tests/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommandTest.java
View File

@ -31,35 +31,31 @@ import java.io.IOException;
import javax.servlet.ServletException;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.junit.Assert;
import org.openrefine.wikidata.testing.TestingData;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.util.ParsingUtilities;
public class PreviewWikibaseSchemaCommandTest extends SchemaCommandTest {
@BeforeMethod
public void SetUp()
throws JSONException {
public void SetUp() {
command = new PreviewWikibaseSchemaCommand();
}
@Test
public void testValidSchema()
throws JSONException, IOException, ServletException {
throws IOException, ServletException {
String schemaJson = jsonFromFile("schema/inception.json").toString();
when(request.getParameter("schema")).thenReturn(schemaJson);
command.doPost(request, response);
JSONObject response = ParsingUtilities.evaluateJsonStringToObject(writer.toString());
JSONArray edits = response.getJSONArray("edits_preview");
assertEquals(3, edits.length());
ObjectNode response = ParsingUtilities.evaluateJsonStringToObjectNode(writer.toString());
ArrayNode edits = (ArrayNode) response.get("edits_preview");
assertEquals(3, edits.size());
}
}

4
extensions/wikidata/tests/src/org/openrefine/wikidata/commands/SchemaCommandTest.java
View File

@ -32,6 +32,8 @@ import javax.servlet.ServletException;
import org.testng.annotations.Test;
import com.google.refine.util.ParsingUtilities;
public abstract class SchemaCommandTest extends CommandTest {
@Test
@ -48,6 +50,6 @@ public abstract class SchemaCommandTest extends CommandTest {
when(request.getParameter("schema")).thenReturn("{bogus json");
command.doPost(request, response);
assertEquals("{\"code\":\"error\",\"message\":\"Wikibase schema could not be parsed.\"}", writer.toString());
assertEquals("error", ParsingUtilities.mapper.readTree(writer.toString()).get("code").asText());
}
}

6
extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java
View File

@ -31,8 +31,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.testing.TestingData;
import org.openrefine.wikidata.updates.ItemUpdate;
@ -67,10 +65,10 @@ public class QuickStatementsExporterTest extends RefineTest {
@Test
public void testSimpleProject()
throws JSONException, IOException {
throws IOException {
Project project = this.createCSVProject(TestingData.inceptionWithNewCsv);
TestingData.reconcileInceptionCells(project);
JSONObject serialized = TestingData.jsonFromFile("schema/inception.json");
String serialized = TestingData.jsonFromFile("schema/inception.json");
WikibaseSchema schema = WikibaseSchema.reconstruct(serialized);
project.overlayModels.put("wikibaseSchema", schema);
Engine engine = new Engine(project);

8
extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java
View File

@ -32,8 +32,6 @@ import java.io.StringReader;
import java.io.StringWriter;
import java.util.Properties;
import org.json.JSONObject;
import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@ -69,17 +67,17 @@ public abstract class OperationTest extends RefineTest {
public abstract AbstractOperation reconstruct()
throws Exception;
public abstract JSONObject getJson()
public abstract String getJson()
throws Exception;
@Test
public void testReconstruct()
throws Exception {
JSONObject json = getJson();
String json = getJson();
AbstractOperation op = reconstruct();
StringWriter writer = new StringWriter();
ParsingUtilities.defaultWriter.writeValue(writer, op);
TestUtils.assertEqualAsJson(json.toString(), writer.toString());
TestUtils.assertEqualAsJson(json, writer.toString());
}
protected LineNumberReader makeReader(String input) {

9
extensions/wikidata/tests/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperationTest.java
View File

@ -27,7 +27,6 @@ import static org.junit.Assert.assertEquals;
import java.io.LineNumberReader;
import org.json.JSONObject;
import org.openrefine.wikidata.testing.TestingData;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@ -36,6 +35,7 @@ import com.google.refine.browsing.EngineConfig;
import com.google.refine.history.Change;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Recon;
import com.google.refine.util.ParsingUtilities;
public class PerformWikibaseEditsOperationTest extends OperationTest {
@ -47,19 +47,18 @@ public class PerformWikibaseEditsOperationTest extends OperationTest {
@Override
public AbstractOperation reconstruct()
throws Exception {
JSONObject json = getJson();
return PerformWikibaseEditsOperation.reconstruct(project, json);
return ParsingUtilities.mapper.readValue(getJson(), PerformWikibaseEditsOperation.class);
}
@Override
public JSONObject getJson()
public String getJson()
throws Exception {
return TestingData.jsonFromFile("operations/perform-edits.json");
}
@Test(expectedExceptions=IllegalArgumentException.class)
public void testConstructor() {
new PerformWikibaseEditsOperation(EngineConfig.reconstruct(new JSONObject("{}")), "");
new PerformWikibaseEditsOperation(EngineConfig.reconstruct("{}"), "");
}
@Test

10
extensions/wikidata/tests/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperationTest.java
View File

@ -28,7 +28,6 @@ import static org.junit.Assert.assertNull;
import java.io.LineNumberReader;
import org.json.JSONObject;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.testing.TestingData;
import org.testng.annotations.BeforeMethod;
@ -36,6 +35,7 @@ import org.testng.annotations.Test;
import com.google.refine.history.Change;
import com.google.refine.model.AbstractOperation;
import com.google.refine.util.ParsingUtilities;
public class SaveWikibaseSchemaOperationTest extends OperationTest {
@ -47,11 +47,11 @@ public class SaveWikibaseSchemaOperationTest extends OperationTest {
@Override
public AbstractOperation reconstruct()
throws Exception {
return SaveWikibaseSchemaOperation.reconstruct(project, getJson());
return ParsingUtilities.mapper.readValue(getJson(), SaveWikibaseSchemaOperation.class);
}
@Override
public JSONObject getJson()
public String getJson()
throws Exception {
return TestingData.jsonFromFile("operations/save-schema.json");
}
@ -59,8 +59,8 @@ public class SaveWikibaseSchemaOperationTest extends OperationTest {
@Test
public void testLoadChange()
throws Exception {
JSONObject schemaJson = TestingData.jsonFromFile("schema/inception.json");
String changeString = "newSchema=" + schemaJson.toString() + "\n" + "oldSchema=\n" + "/ec/";
String schemaJson = TestingData.jsonFromFile("schema/inception.json");
String changeString = "newSchema=" + schemaJson + "\n" + "oldSchema=\n" + "/ec/";
WikibaseSchema schema = WikibaseSchema.reconstruct(schemaJson);
LineNumberReader reader = makeReader(changeString);

26
extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java
View File

@ -31,8 +31,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.json.JSONException;
import org.json.JSONObject;
import org.openrefine.wikidata.testing.TestingData;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder;
@ -95,25 +93,25 @@ public class WikibaseSchemaTest extends RefineTest {
@Test
public void testSerialize()
throws JSONException, IOException {
JSONObject serialized = TestingData.jsonFromFile("schema/history_of_medicine.json");
throws IOException {
String serialized = TestingData.jsonFromFile("schema/history_of_medicine.json");
WikibaseSchema parsed = WikibaseSchema.reconstruct(serialized);
TestUtils.isSerializedTo(parsed, TestingData.jsonFromFile("schema/history_of_medicine_normalized.json").toString());
}
@Test
public void testDeserialize()
throws JSONException, IOException {
throws IOException {
// this json file was generated by an earlier version of the software
// it contains extra "type" fields that are now ignored.
JSONObject serialized = TestingData.jsonFromFile("schema/roarmap.json");
String serialized = TestingData.jsonFromFile("schema/roarmap.json");
WikibaseSchema.reconstruct(serialized);
}
@Test
public void testEvaluate()
throws JSONException, IOException {
JSONObject serialized = TestingData.jsonFromFile("schema/inception.json");
throws IOException {
String serialized = TestingData.jsonFromFile("schema/inception.json");
WikibaseSchema schema = WikibaseSchema.reconstruct(serialized);
Engine engine = new Engine(project);
List<ItemUpdate> updates = schema.evaluate(project, engine);
@ -125,8 +123,8 @@ public class WikibaseSchemaTest extends RefineTest {
assertEquals(expected, updates);
}
@Test(expectedExceptions = JSONException.class)
public void testDeserializeEmpty() throws JSONException {
@Test(expectedExceptions = IOException.class)
public void testDeserializeEmpty() throws IOException {
String schemaJson = "{\"itemDocuments\":[{\"statementGroups\":[{\"statements\":[]}],"
+"\"nameDescs\":[]}],\"wikibasePrefix\":\"http://www.wikidata.org/entity/\"}";
WikibaseSchema.reconstruct(schemaJson);
@ -134,11 +132,11 @@ public class WikibaseSchemaTest extends RefineTest {
@Test
public void testEvaluateRespectsFacets()
throws JSONException, IOException {
JSONObject serialized = TestingData.jsonFromFile("schema/inception.json");
throws IOException {
String serialized = TestingData.jsonFromFile("schema/inception.json");
WikibaseSchema schema = WikibaseSchema.reconstruct(serialized);
Engine engine = new Engine(project);
EngineConfig engineConfig = EngineConfig.reconstruct(new JSONObject("{\n"
EngineConfig engineConfig = EngineConfig.reconstruct("{\n"
+ " \"mode\": \"row-based\",\n"
+ " \"facets\": [\n"
+ " {\n"
@ -151,7 +149,7 @@ public class WikibaseSchemaTest extends RefineTest {
+ " \"columnName\": \"reference\"\n"
+ " }\n"
+ " ]\n"
+ " }"));
+ " }");
engine.initializeFromConfig(engineConfig);
List<ItemUpdate> updates = schema.evaluate(project, engine);
List<ItemUpdate> expected = new ArrayList<>();

4
extensions/wikidata/tests/src/org/openrefine/wikidata/testing/TestingData.java
View File

@ -133,11 +133,11 @@ public class TestingData {
return generateStatement(from, pid, to);
}
public static JSONObject jsonFromFile(String filename)
public static String jsonFromFile(String filename)
throws IOException, JSONException {
InputStream f = TestingData.class.getClassLoader().getResourceAsStream(filename);
String decoded = IOUtils.toString(f);
return ParsingUtilities.evaluateJsonStringToObject(decoded);
return decoded.trim();
}
public static void reconcileInceptionCells(Project project) {