From d8f081b9775480f31891f3901679ad24c946cd96 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Tue, 22 May 2018 23:09:51 +0200 Subject: [PATCH 1/2] Remove DataExtensions from change serialization --- .../model/changes/DataExtensionChange.java | 42 ++--------- .../tests/data/changes/data_extension_2.8.txt | 49 +++++++++++++ .../tests/data/changes/data_extension_3.0.txt | 29 ++++++++ .../changes/DataExtensionChangeTest.java | 72 +++++++++++++++++++ 4 files changed, 154 insertions(+), 38 deletions(-) create mode 100644 main/tests/data/changes/data_extension_2.8.txt create mode 100644 main/tests/data/changes/data_extension_3.0.txt create mode 100644 main/tests/server/src/com/google/refine/tests/model/changes/DataExtensionChangeTest.java diff --git a/main/src/com/google/refine/model/changes/DataExtensionChange.java b/main/src/com/google/refine/model/changes/DataExtensionChange.java index b7098a1e0..4497320d5 100644 --- a/main/src/com/google/refine/model/changes/DataExtensionChange.java +++ b/main/src/com/google/refine/model/changes/DataExtensionChange.java @@ -1,6 +1,6 @@ /* -Copyright 2010, Google Inc. +àCopyright 2010, Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -321,35 +321,6 @@ public class DataExtensionChange implements Change { for (Integer rowIndex : _rowIndices) { writer.write(rowIndex.toString()); writer.write('\n'); } - writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n'); - for (DataExtension dataExtension : _dataExtensions) { - if (dataExtension == null) { - writer.write('\n'); - continue; - } - - writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n'); - - for (Object[] values : dataExtension.data) { - for (Object value : values) { - if (value == null) { - writer.write("null"); - } else if (value instanceof ReconCandidate) { - try { - JSONWriter jsonWriter = new JSONWriter(writer); - ((ReconCandidate) value).write(jsonWriter, options); - } catch (JSONException e) { - // ??? - } - } else if (value instanceof String) { - writer.write(JSONObject.quote((String) value)); - } else { - writer.write(value.toString()); - } - writer.write('\n'); - } - } - } writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n'); @@ -435,6 +406,9 @@ public class DataExtensionChange implements Change { } } } else if ("dataExtensionCount".equals(field)) { + // kept for compatibility with 2.8, but the data + // deserialized here is not actually needed to apply/undo + // the change, so we ignore it. int count = Integer.parseInt(value); dataExtensions = new ArrayList(count); @@ -451,20 +425,12 @@ public class DataExtensionChange implements Change { } int rowCount = Integer.parseInt(line); - Object[][] data = new Object[rowCount][]; for (int r = 0; r < rowCount; r++) { - Object[] row = new Object[columnNames.size()]; for (int c = 0; c < columnNames.size(); c++) { line = reader.readLine(); - - row[c] = ReconCandidate.loadStreaming(line); } - - data[r] = row; } - - dataExtensions.add(new DataExtension(data)); } } else if ("oldRowCount".equals(field)) { int count = Integer.parseInt(value); diff --git a/main/tests/data/changes/data_extension_2.8.txt b/main/tests/data/changes/data_extension_2.8.txt new file mode 100644 index 000000000..e185f129c --- /dev/null +++ b/main/tests/data/changes/data_extension_2.8.txt @@ -0,0 +1,49 @@ +2.8 +com.google.refine.model.changes.DataExtensionChange +baseColumnName=item +service=https://tools.wmflabs.org/openrefine-wikidata/en/api +identifierSpace=http://www.wikidata.org/entity/ +schemaSpace=http://www.wikidata.org/prop/direct/ +columnInsertIndex=1 +columnNameCount=2 +instance of +inception +columnTypeCount=2 +{"id":"Q21503252","name":"instance of"} + +rowIndexCount=1 +0 +dataExtensionCount=1 +9 +{"id":"Q33120876","name":"Wikimedia content project","score":100,"types":[]} +2012-10-29T00:00Z +{"id":"Q638153","name":"semantic wiki","score":100,"types":[]} +null +{"id":"Q593744","name":"knowledge base","score":100,"types":[]} +null +{"id":"Q36509592","name":"wiki with script conversion","score":100,"types":[]} +null +{"id":"Q7094076","name":"online database","score":100,"types":[]} +null +{"id":"Q54837","name":"Semantic Web","score":100,"types":[]} +null +{"id":"Q14827288","name":"Wikimedia project","score":100,"types":[]} +null +{"id":"Q33002955","name":"knowledge graph","score":100,"types":[]} +null +{"id":"Q35127","name":"website","score":100,"types":[]} +null +firstNewCellIndex=1 +newRowCount=9 +{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"},{"v":"Wikimedia content project","r":"1527019713238399737"},{"v":"2012-10-29T00:00:00Z","t":"date"}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"semantic wiki","r":"1527019713239325739"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge base","r":"1527019713239978180"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"wiki with script conversion","r":"1527019713239859496"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"online database","r":"1527019713239103094"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"Semantic Web","r":"1527019713239599435"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"Wikimedia project","r":"1527019713239979343"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge graph","r":"1527019713239208298"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"website","r":"1527019713239148006"},{"v":null}]} +oldRowCount=1 +{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"}]} +/ec/ diff --git a/main/tests/data/changes/data_extension_3.0.txt b/main/tests/data/changes/data_extension_3.0.txt new file mode 100644 index 000000000..46b7fa4c0 --- /dev/null +++ b/main/tests/data/changes/data_extension_3.0.txt @@ -0,0 +1,29 @@ +2.8 +com.google.refine.model.changes.DataExtensionChange +baseColumnName=item +service=https://tools.wmflabs.org/openrefine-wikidata/en/api +identifierSpace=http://www.wikidata.org/entity/ +schemaSpace=http://www.wikidata.org/prop/direct/ +columnInsertIndex=1 +columnNameCount=2 +instance of +inception +columnTypeCount=2 +{"id":"Q21503252","name":"instance of"} + +rowIndexCount=1 +0 +firstNewCellIndex=1 +newRowCount=9 +{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"},{"v":"Wikimedia content project","r":"1527019713238399737"},{"v":"2012-10-29T00:00:00Z","t":"date"}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"semantic wiki","r":"1527019713239325739"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge base","r":"1527019713239978180"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"wiki with script conversion","r":"1527019713239859496"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"online database","r":"1527019713239103094"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"Semantic Web","r":"1527019713239599435"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"Wikimedia project","r":"1527019713239979343"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge graph","r":"1527019713239208298"},{"v":null}]} +{"flagged":false,"starred":false,"cells":[null,{"v":"website","r":"1527019713239148006"},{"v":null}]} +oldRowCount=1 +{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"}]} +/ec/ diff --git a/main/tests/server/src/com/google/refine/tests/model/changes/DataExtensionChangeTest.java b/main/tests/server/src/com/google/refine/tests/model/changes/DataExtensionChangeTest.java new file mode 100644 index 000000000..c3608622b --- /dev/null +++ b/main/tests/server/src/com/google/refine/tests/model/changes/DataExtensionChangeTest.java @@ -0,0 +1,72 @@ +package com.google.refine.tests.model.changes; + +import com.google.refine.model.ModelException; +import com.google.refine.model.Project; +import com.google.refine.model.changes.DataExtensionChange; +import com.google.refine.model.changes.MassChange; + +import static org.junit.Assert.assertEquals; + +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.io.Reader; + +import org.slf4j.LoggerFactory; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import com.google.refine.history.Change; +import com.google.refine.tests.RefineTest; +import com.google.refine.util.Pool; + + +public class DataExtensionChangeTest extends RefineTest { + + Project project; + + @Override + @BeforeTest + public void init() { + logger = LoggerFactory.getLogger(this.getClass()); + } + + @BeforeMethod + public void SetUp() + throws IOException, ModelException { + project = createCSVProject( + "reconciled\n"+ + "some item"); + } + + @Test + public void testApplyOldChange() throws Exception { + Pool pool = new Pool(); + InputStream in = this.getClass().getClassLoader() + .getResourceAsStream("changes/data_extension_2.8.txt"); + LineNumberReader lineReader = new LineNumberReader(new InputStreamReader(in)); + // skip the header + lineReader.readLine(); + lineReader.readLine(); + Change change = DataExtensionChange.load(lineReader, pool); + change.apply(project); + assertEquals("Wikimedia content project", project.rows.get(0).getCell(1).value); + } + + @Test + public void testApplyNewChange() throws Exception { + Pool pool = new Pool(); + InputStream in = this.getClass().getClassLoader() + .getResourceAsStream("changes/data_extension_3.0.txt"); + LineNumberReader lineReader = new LineNumberReader(new InputStreamReader(in)); + // skip the header + lineReader.readLine(); + lineReader.readLine(); + Change change = DataExtensionChange.load(lineReader, pool); + change.apply(project); + assertEquals("Wikimedia content project", project.rows.get(0).getCell(1).value); + } +} From 52989b78ce836804d5448c8eb8e2f15896a72e73 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Wed, 23 May 2018 07:39:04 +0200 Subject: [PATCH 2/2] Fix typo --- .../com/google/refine/model/changes/DataExtensionChange.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/src/com/google/refine/model/changes/DataExtensionChange.java b/main/src/com/google/refine/model/changes/DataExtensionChange.java index 4497320d5..a034c9ebc 100644 --- a/main/src/com/google/refine/model/changes/DataExtensionChange.java +++ b/main/src/com/google/refine/model/changes/DataExtensionChange.java @@ -1,6 +1,6 @@ /* -àCopyright 2010, Google Inc. +Copyright 2010, Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without