Merge pull request #1617 from OpenRefine/issue1600

Remove DataExtensions from change serialization

main/src/com/google/refine/model/changes/DataExtensionChange.java

@@ -321,35 +321,6 @@ public class DataExtensionChange implements Change {
         for (Integer rowIndex : _rowIndices) {
             writer.write(rowIndex.toString()); writer.write('\n');
         }
-        writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n');
-        for (DataExtension dataExtension : _dataExtensions) {
-            if (dataExtension == null) {
-                writer.write('\n');
-                continue;
-            }
-            
-            writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n');
-            
-            for (Object[] values : dataExtension.data) {
-                for (Object value : values) {
-                    if (value == null) {
-                        writer.write("null");
-                    } else if (value instanceof ReconCandidate) {
-                        try {
-                            JSONWriter jsonWriter = new JSONWriter(writer);
-                            ((ReconCandidate) value).write(jsonWriter, options);
-                        } catch (JSONException e) {
-                            // ???
-                        }
-                    } else if (value instanceof String) {
-                        writer.write(JSONObject.quote((String) value));
-                    } else {
-                        writer.write(value.toString());
-                    }
-                    writer.write('\n');
-                }
-            }
-        }
         
         writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
         
@@ -435,6 +406,9 @@ public class DataExtensionChange implements Change {
                     }
                 }
             } else if ("dataExtensionCount".equals(field)) {
+                // kept for compatibility with 2.8, but the data 
+                // deserialized here is not actually needed to apply/undo
+                // the change, so we ignore it.
                 int count = Integer.parseInt(value);
                 
                 dataExtensions = new ArrayList<DataExtension>(count);
@@ -451,20 +425,12 @@ public class DataExtensionChange implements Change {
                     }
                     
                     int rowCount = Integer.parseInt(line);
-                    Object[][] data = new Object[rowCount][];
                     
                     for (int r = 0; r < rowCount; r++) {
-                        Object[] row = new Object[columnNames.size()];
                         for (int c = 0; c < columnNames.size(); c++) {
                             line = reader.readLine();
-                            
-                            row[c] = ReconCandidate.loadStreaming(line);
                         }
-                        
-                        data[r] = row;
                     }
-                    
-                    dataExtensions.add(new DataExtension(data));
                 }
             } else if ("oldRowCount".equals(field)) {
                 int count = Integer.parseInt(value);

main/tests/data/changes/data_extension_2.8.txt

@@ -0,0 +1,49 @@
+2.8
+com.google.refine.model.changes.DataExtensionChange
+baseColumnName=item
+service=https://tools.wmflabs.org/openrefine-wikidata/en/api
+identifierSpace=http://www.wikidata.org/entity/
+schemaSpace=http://www.wikidata.org/prop/direct/
+columnInsertIndex=1
+columnNameCount=2
+instance of
+inception
+columnTypeCount=2
+{"id":"Q21503252","name":"instance of"}
+
+rowIndexCount=1
+0
+dataExtensionCount=1
+9
+{"id":"Q33120876","name":"Wikimedia content project","score":100,"types":[]}
+2012-10-29T00:00Z
+{"id":"Q638153","name":"semantic wiki","score":100,"types":[]}
+null
+{"id":"Q593744","name":"knowledge base","score":100,"types":[]}
+null
+{"id":"Q36509592","name":"wiki with script conversion","score":100,"types":[]}
+null
+{"id":"Q7094076","name":"online database","score":100,"types":[]}
+null
+{"id":"Q54837","name":"Semantic Web","score":100,"types":[]}
+null
+{"id":"Q14827288","name":"Wikimedia project","score":100,"types":[]}
+null
+{"id":"Q33002955","name":"knowledge graph","score":100,"types":[]}
+null
+{"id":"Q35127","name":"website","score":100,"types":[]}
+null
+firstNewCellIndex=1
+newRowCount=9
+{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"},{"v":"Wikimedia content project","r":"1527019713238399737"},{"v":"2012-10-29T00:00:00Z","t":"date"}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"semantic wiki","r":"1527019713239325739"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge base","r":"1527019713239978180"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"wiki with script conversion","r":"1527019713239859496"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"online database","r":"1527019713239103094"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"Semantic Web","r":"1527019713239599435"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"Wikimedia project","r":"1527019713239979343"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge graph","r":"1527019713239208298"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"website","r":"1527019713239148006"},{"v":null}]}
+oldRowCount=1
+{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"}]}
+/ec/

main/tests/data/changes/data_extension_3.0.txt

@@ -0,0 +1,29 @@
+2.8
+com.google.refine.model.changes.DataExtensionChange
+baseColumnName=item
+service=https://tools.wmflabs.org/openrefine-wikidata/en/api
+identifierSpace=http://www.wikidata.org/entity/
+schemaSpace=http://www.wikidata.org/prop/direct/
+columnInsertIndex=1
+columnNameCount=2
+instance of
+inception
+columnTypeCount=2
+{"id":"Q21503252","name":"instance of"}
+
+rowIndexCount=1
+0
+firstNewCellIndex=1
+newRowCount=9
+{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"},{"v":"Wikimedia content project","r":"1527019713238399737"},{"v":"2012-10-29T00:00:00Z","t":"date"}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"semantic wiki","r":"1527019713239325739"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge base","r":"1527019713239978180"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"wiki with script conversion","r":"1527019713239859496"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"online database","r":"1527019713239103094"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"Semantic Web","r":"1527019713239599435"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"Wikimedia project","r":"1527019713239979343"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge graph","r":"1527019713239208298"},{"v":null}]}
+{"flagged":false,"starred":false,"cells":[null,{"v":"website","r":"1527019713239148006"},{"v":null}]}
+oldRowCount=1
+{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"}]}
+/ec/

main/tests/server/src/com/google/refine/tests/model/changes/DataExtensionChangeTest.java

@@ -0,0 +1,72 @@
+package com.google.refine.tests.model.changes;
+
+import com.google.refine.model.ModelException;
+import com.google.refine.model.Project;
+import com.google.refine.model.changes.DataExtensionChange;
+import com.google.refine.model.changes.MassChange;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
+import java.io.Reader;
+
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import com.google.refine.history.Change;
+import com.google.refine.tests.RefineTest;
+import com.google.refine.util.Pool;
+
+
+public class DataExtensionChangeTest extends RefineTest {
+
+    Project project;
+    
+    @Override
+    @BeforeTest
+    public void init() {
+        logger = LoggerFactory.getLogger(this.getClass());
+    }
+
+    @BeforeMethod
+    public void SetUp()
+            throws IOException, ModelException {
+        project = createCSVProject(
+                "reconciled\n"+
+                "some item");
+    }
+    
+    @Test
+    public void testApplyOldChange() throws Exception {
+        Pool pool = new Pool();
+        InputStream in = this.getClass().getClassLoader()
+                .getResourceAsStream("changes/data_extension_2.8.txt");
+        LineNumberReader lineReader = new LineNumberReader(new InputStreamReader(in));
+        // skip the header
+        lineReader.readLine();
+        lineReader.readLine();
+        Change change = DataExtensionChange.load(lineReader, pool);
+        change.apply(project);
+        assertEquals("Wikimedia content project", project.rows.get(0).getCell(1).value);
+    }
+    
+    @Test
+    public void testApplyNewChange() throws Exception {
+        Pool pool = new Pool();
+        InputStream in = this.getClass().getClassLoader()
+                .getResourceAsStream("changes/data_extension_3.0.txt");
+        LineNumberReader lineReader = new LineNumberReader(new InputStreamReader(in));
+        // skip the header
+        lineReader.readLine();
+        lineReader.readLine();
+        Change change = DataExtensionChange.load(lineReader, pool);
+        change.apply(project);
+        assertEquals("Wikimedia content project", project.rows.get(0).getCell(1).value);
+    }
+}