s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove DataExtensions from change serialization

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-05-22 23:09:51 +02:00
parent 3132ce8b00
commit d8f081b977
4 changed files with 154 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
main/src/com/google/refine/model/changes/DataExtensionChange.java
View File

@ -1,6 +1,6 @@
/*
Copyright 2010, Google Inc.
àCopyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -321,35 +321,6 @@ public class DataExtensionChange implements Change {
for (Integer rowIndex : _rowIndices) {
writer.write(rowIndex.toString()); writer.write('\n');
}
writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n');
for (DataExtension dataExtension : _dataExtensions) {
if (dataExtension == null) {
writer.write('\n');
continue;
}
writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n');
for (Object[] values : dataExtension.data) {
for (Object value : values) {
if (value == null) {
writer.write("null");
} else if (value instanceof ReconCandidate) {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
((ReconCandidate) value).write(jsonWriter, options);
} catch (JSONException e) {
// ???
}
} else if (value instanceof String) {
writer.write(JSONObject.quote((String) value));
} else {
writer.write(value.toString());
}
writer.write('\n');
}
}
}
writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
@ -435,6 +406,9 @@ public class DataExtensionChange implements Change {
}
}
} else if ("dataExtensionCount".equals(field)) {
// kept for compatibility with 2.8, but the data
// deserialized here is not actually needed to apply/undo
// the change, so we ignore it.
int count = Integer.parseInt(value);
dataExtensions = new ArrayList<DataExtension>(count);
@ -451,20 +425,12 @@ public class DataExtensionChange implements Change {
}
int rowCount = Integer.parseInt(line);
Object[][] data = new Object[rowCount][];
for (int r = 0; r < rowCount; r++) {
Object[] row = new Object[columnNames.size()];
for (int c = 0; c < columnNames.size(); c++) {
line = reader.readLine();
row[c] = ReconCandidate.loadStreaming(line);
}
data[r] = row;
}
dataExtensions.add(new DataExtension(data));
}
} else if ("oldRowCount".equals(field)) {
int count = Integer.parseInt(value);

49
main/tests/data/changes/data_extension_2.8.txt Normal file
View File

@ -0,0 +1,49 @@
2.8
com.google.refine.model.changes.DataExtensionChange
baseColumnName=item
service=https://tools.wmflabs.org/openrefine-wikidata/en/api
identifierSpace=http://www.wikidata.org/entity/
schemaSpace=http://www.wikidata.org/prop/direct/
columnInsertIndex=1
columnNameCount=2
instance of
inception
columnTypeCount=2
{"id":"Q21503252","name":"instance of"}
rowIndexCount=1
0
dataExtensionCount=1
9
{"id":"Q33120876","name":"Wikimedia content project","score":100,"types":[]}
2012-10-29T00:00Z
{"id":"Q638153","name":"semantic wiki","score":100,"types":[]}
null
{"id":"Q593744","name":"knowledge base","score":100,"types":[]}
null
{"id":"Q36509592","name":"wiki with script conversion","score":100,"types":[]}
null
{"id":"Q7094076","name":"online database","score":100,"types":[]}
null
{"id":"Q54837","name":"Semantic Web","score":100,"types":[]}
null
{"id":"Q14827288","name":"Wikimedia project","score":100,"types":[]}
null
{"id":"Q33002955","name":"knowledge graph","score":100,"types":[]}
null
{"id":"Q35127","name":"website","score":100,"types":[]}
null
firstNewCellIndex=1
newRowCount=9
{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"},{"v":"Wikimedia content project","r":"1527019713238399737"},{"v":"2012-10-29T00:00:00Z","t":"date"}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"semantic wiki","r":"1527019713239325739"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge base","r":"1527019713239978180"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"wiki with script conversion","r":"1527019713239859496"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"online database","r":"1527019713239103094"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"Semantic Web","r":"1527019713239599435"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"Wikimedia project","r":"1527019713239979343"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge graph","r":"1527019713239208298"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"website","r":"1527019713239148006"},{"v":null}]}
oldRowCount=1
{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"}]}
/ec/

29
main/tests/data/changes/data_extension_3.0.txt Normal file
View File

@ -0,0 +1,29 @@
2.8
com.google.refine.model.changes.DataExtensionChange
baseColumnName=item
service=https://tools.wmflabs.org/openrefine-wikidata/en/api
identifierSpace=http://www.wikidata.org/entity/
schemaSpace=http://www.wikidata.org/prop/direct/
columnInsertIndex=1
columnNameCount=2
instance of
inception
columnTypeCount=2
{"id":"Q21503252","name":"instance of"}
rowIndexCount=1
0
firstNewCellIndex=1
newRowCount=9
{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"},{"v":"Wikimedia content project","r":"1527019713238399737"},{"v":"2012-10-29T00:00:00Z","t":"date"}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"semantic wiki","r":"1527019713239325739"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge base","r":"1527019713239978180"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"wiki with script conversion","r":"1527019713239859496"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"online database","r":"1527019713239103094"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"Semantic Web","r":"1527019713239599435"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"Wikimedia project","r":"1527019713239979343"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"knowledge graph","r":"1527019713239208298"},{"v":null}]}
{"flagged":false,"starred":false,"cells":[null,{"v":"website","r":"1527019713239148006"},{"v":null}]}
oldRowCount=1
{"flagged":false,"starred":false,"cells":[{"v":"Wikidata","r":"1526743824662484326"}]}
/ec/

72
main/tests/server/src/com/google/refine/tests/model/changes/DataExtensionChangeTest.java Normal file
View File

@ -0,0 +1,72 @@
package com.google.refine.tests.model.changes;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.changes.DataExtensionChange;
import com.google.refine.model.changes.MassChange;
import static org.junit.Assert.assertEquals;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.io.Reader;
import org.slf4j.LoggerFactory;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.history.Change;
import com.google.refine.tests.RefineTest;
import com.google.refine.util.Pool;
public class DataExtensionChangeTest extends RefineTest {
Project project;
@Override
@BeforeTest
public void init() {
logger = LoggerFactory.getLogger(this.getClass());
}
@BeforeMethod
public void SetUp()
throws IOException, ModelException {
project = createCSVProject(
"reconciled\n"+
"some item");
}
@Test
public void testApplyOldChange() throws Exception {
Pool pool = new Pool();
InputStream in = this.getClass().getClassLoader()
.getResourceAsStream("changes/data_extension_2.8.txt");
LineNumberReader lineReader = new LineNumberReader(new InputStreamReader(in));
// skip the header
lineReader.readLine();
lineReader.readLine();
Change change = DataExtensionChange.load(lineReader, pool);
change.apply(project);
assertEquals("Wikimedia content project", project.rows.get(0).getCell(1).value);
}
@Test
public void testApplyNewChange() throws Exception {
Pool pool = new Pool();
InputStream in = this.getClass().getClassLoader()
.getResourceAsStream("changes/data_extension_3.0.txt");
LineNumberReader lineReader = new LineNumberReader(new InputStreamReader(in));
// skip the header
lineReader.readLine();
lineReader.readLine();
Change change = DataExtensionChange.load(lineReader, pool);
change.apply(project);
assertEquals("Wikimedia content project", project.rows.get(0).getCell(1).value);
}
}