s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Migrate ProjectMetadata to Jackson

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-11-19 15:57:24 +00:00
parent 55dc752144
commit b569490eef
11 changed files with 173 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
main/src/com/google/refine/ProjectManager.java
View File

@ -47,18 +47,20 @@ import java.util.Map.Entry;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.tools.tar.TarOutputStream;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.history.HistoryEntryManager;
import com.google.refine.model.Project;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.preference.TopList;
import com.google.refine.util.ParsingUtilities;
/**
* ProjectManager is responsible for loading and saving the workspace and projects.
@ -399,7 +401,7 @@ public abstract class ProjectManager {
* @param placeHolderJsonObj
* @return
*/
private boolean isValidUserMetadataDefinition(JSONObject placeHolderJsonObj) {
private boolean isValidUserMetadataDefinition(ObjectNode placeHolderJsonObj) {
return (placeHolderJsonObj != null &&
placeHolderJsonObj.has("name") &&
placeHolderJsonObj.has("display"));
@ -410,9 +412,9 @@ public abstract class ProjectManager {
return;
// place holder
JSONArray userMetadataPreference = null;
ArrayNode userMetadataPreference = null;
// actual metadata for project
JSONArray jsonObjArray = metadata.getUserMetadata();
ArrayNode jsonObjArray = metadata.getUserMetadata();
initDisplay(jsonObjArray);
@ -420,41 +422,41 @@ public abstract class ProjectManager {
String userMeta = (String)_preferenceStore.get(PreferenceStore.USER_METADATA_KEY);
if (userMeta == null)
return;
userMetadataPreference = new JSONArray(userMeta);
userMetadataPreference = ParsingUtilities.mapper.createArrayNode();
} catch (JSONException e1) {
logger.warn("wrong definition of userMetadata format. Please use form [{\"name\": \"client name\", \"display\":true}, {\"name\": \"progress\", \"display\":false}]");
logger.error(ExceptionUtils.getFullStackTrace(e1));
}
for (int index = 0; index < userMetadataPreference.length(); index++) {
try {
boolean found = false;
JSONObject placeHolderJsonObj = userMetadataPreference.getJSONObject(index);
if (!isValidUserMetadataDefinition(placeHolderJsonObj)) {
logger.warn("Skipped invalid user metadata definition" + placeHolderJsonObj.toString());
continue;
}
for (int i = 0; i < jsonObjArray.length(); i++) {
JSONObject jsonObj = jsonObjArray.getJSONObject(i);
if (jsonObj.getString("name").equals(placeHolderJsonObj.getString("name"))) {
found = true;
jsonObj.put("display", placeHolderJsonObj.get("display"));
break;
}
}
if (!found) {
placeHolderJsonObj.put("value", "");
metadata.getUserMetadata().put(placeHolderJsonObj);
logger.info("Put the placeholder {} for project {}",
placeHolderJsonObj.getString("name"),
metadata.getName());
}
} catch (JSONException e) {
logger.warn("Exception when mergeEmptyUserMetadata",e);
for (int index = 0; index < userMetadataPreference.size(); index++) {
boolean found = false;
ObjectNode placeHolderJsonObj = (ObjectNode) userMetadataPreference.get(index);
if (!isValidUserMetadataDefinition(placeHolderJsonObj)) {
logger.warn("Skipped invalid user metadata definition" + placeHolderJsonObj.toString());
continue;
}
for (int i = 0; i < jsonObjArray.size(); i++) {
JsonNode jsonObj = jsonObjArray.get(i);
if (!(jsonObj instanceof ObjectNode)) {
continue;
}
ObjectNode node = (ObjectNode)jsonObj;
if (node.get("name").asText("").equals(placeHolderJsonObj.get("name").asText(""))) {
found = true;
node.put("display", placeHolderJsonObj.get("display"));
break;
}
}
if (!found) {
placeHolderJsonObj.put("value", "");
metadata.getUserMetadata().add(placeHolderJsonObj);
logger.info("Put the placeholder {} for project {}",
placeHolderJsonObj.get("name").asText(""),
metadata.getName());
}
}
}
@ -462,13 +464,11 @@ public abstract class ProjectManager {
* honor the meta data preference
* @param jsonObjArray
*/
private void initDisplay(JSONArray jsonObjArray) {
for (int index = 0; index < jsonObjArray.length(); index++) {
try {
JSONObject projectMetaJsonObj = jsonObjArray.getJSONObject(index);
private void initDisplay(ArrayNode jsonObjArray) {
for (int index = 0; index < jsonObjArray.size(); index++) {
if (jsonObjArray.get(index) instanceof ObjectNode) {
ObjectNode projectMetaJsonObj = (ObjectNode) jsonObjArray.get(index);
projectMetaJsonObj.put("display", false);
} catch (JSONException e) {
logger.error(ExceptionUtils.getFullStackTrace(e));
}
}
}

125
main/src/com/google/refine/ProjectMetadata.java
View File

@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.lang.reflect.Field;
import java.time.LocalDateTime;
@ -43,19 +40,20 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.util.JsonViews;
import com.google.refine.util.ParsingUtilities;
public class ProjectMetadata {
@ -63,30 +61,62 @@ public class ProjectMetadata {
public final static String TEMP_FILE_NAME = "metadata.temp.json";
public final static String OLD_FILE_NAME = "metadata.old.json";
@JsonProperty("created")
private final LocalDateTime _created;
@JsonProperty("modified")
private LocalDateTime _modified;
@JsonIgnore
private LocalDateTime written = null;
@JsonProperty("name")
private String _name = "";
@JsonProperty("password")
@JsonView(JsonViews.SaveMode.class)
private String _password = "";
@JsonProperty("encoding")
@JsonView(JsonViews.SaveMode.class)
private String _encoding = "";
@JsonProperty("encodingConfidence")
@JsonView(JsonViews.SaveMode.class)
private int _encodingConfidence;
@JsonProperty("tags")
private String[] _tags = new String[0];
@JsonProperty("creator")
private String _creator = "";
@JsonProperty("contributors")
private String _contributors = "";
@JsonProperty("subject")
private String _subject = ""; // Several refine projects may be linked
@JsonProperty("description")
private String _description = ""; // free form of comment
@JsonProperty("rowCount")
private int _rowCount; // at the creation. Essential for cleaning old projects too heavy
@JsonProperty("title")
private String _title = "";
@JsonProperty("version")
private String _version = "";
@JsonProperty("license")
private String license = "";
@JsonProperty("homepage")
private String homepage = "";
@JsonProperty("image")
private String image = "";
// import options is an array for 1-n data sources
private JSONArray _importOptionMetadata = new JSONArray();
@JsonProperty("importOptionMetadata")
private ArrayNode _importOptionMetadata = ParsingUtilities.mapper.createArrayNode();
// user metadata
private JSONArray _userMetadata = new JSONArray();
@JsonIgnore
private ArrayNode _userMetadata = ParsingUtilities.mapper.createArrayNode();
@JsonProperty("customMetadata")
private Map<String, Serializable> _customMetadata = new HashMap<String, Serializable>();
@JsonProperty("preferences")
@JsonView(JsonViews.SaveMode.class)
private PreferenceStore _preferenceStore = new PreferenceStore();
private final static Logger logger = LoggerFactory.getLogger("project_metadata");
@ -106,11 +136,8 @@ public class ProjectMetadata {
_modified = modified;
_name = name;
}
private boolean isSaveMode(Properties options) {
return "save".equals(options.getProperty("mode"));
}
@JsonIgnore
public boolean isDirty() {
return written == null || _modified.isAfter(written);
}
@ -120,43 +147,52 @@ public class ProjectMetadata {
// Any project specific preferences?
}
@JsonIgnore
public LocalDateTime getCreated() {
return _created;
}
@JsonIgnore
public void setName(String name) {
this._name = name;
updateModified();
}
@JsonIgnore
public String getName() {
return _name;
}
@JsonIgnore
public void setEncoding(String encoding) {
this._encoding = encoding;
updateModified();
}
@JsonIgnore
public String getEncoding() {
return _encoding;
}
@JsonIgnore
public void setEncodingConfidence(int confidence) {
this._encodingConfidence = confidence;
updateModified();
}
@JsonIgnore
public void setEncodingConfidence(String confidence) {
if (confidence != null) {
this.setEncodingConfidence(Integer.parseInt(confidence));
}
}
@JsonIgnore
public int getEncodingConfidence() {
return _encodingConfidence;
}
@JsonIgnore
public void setTags(String[] tags) {
if (tags != null) {
List<String> tmpTags = new ArrayList<String>(tags.length);
@ -177,32 +213,39 @@ public class ProjectMetadata {
updateModified();
}
@JsonIgnore
public String[] getTags() {
if (_tags == null) this._tags = new String[0];
return _tags;
}
@JsonIgnore
public void setPassword(String password) {
this._password = password;
updateModified();
}
@JsonIgnore
public String getPassword() {
return _password;
}
@JsonIgnore
public LocalDateTime getModified() {
return _modified;
}
@JsonIgnore
public void updateModified() {
_modified = LocalDateTime.now();
}
@JsonIgnore
public PreferenceStore getPreferenceStore() {
return _preferenceStore;
}
@JsonIgnore
public Serializable getCustomMetadata(String key) {
return _customMetadata.get(key);
}
@ -216,82 +259,101 @@ public class ProjectMetadata {
updateModified();
}
public JSONArray getImportOptionMetadata() {
@JsonIgnore
public ArrayNode getImportOptionMetadata() {
return _importOptionMetadata;
}
public void setImportOptionMetadata(JSONArray jsonArray) {
@JsonIgnore
public void setImportOptionMetadata(ArrayNode jsonArray) {
_importOptionMetadata = jsonArray;
updateModified();
}
public void appendImportOptionMetadata(ObjectNode options) {
_importOptionMetadata.put(options);
_importOptionMetadata.add(options);
updateModified();
}
@JsonIgnore
public String getCreator() {
return _creator;
}
@JsonIgnore
public void setCreator(String creator) {
this._creator = creator;
updateModified();
}
@JsonIgnore
public String getContributors() {
return _contributors;
}
@JsonIgnore
public void setContributors(String contributors) {
this._contributors = contributors;
updateModified();
}
@JsonIgnore
public String getSubject() {
return _subject;
}
@JsonIgnore
public void setSubject(String subject) {
this._subject = subject;
updateModified();
}
@JsonIgnore
public String getDescription() {
return _description;
}
@JsonIgnore
public void setDescription(String description) {
this._description = description;
updateModified();
}
@JsonIgnore
public int getRowCount() {
return _rowCount;
}
@JsonIgnore
public void setRowCount(int rowCount) {
this._rowCount = rowCount;
updateModified();
}
public JSONArray getUserMetadata() {
@JsonIgnore
public ArrayNode getUserMetadata() {
return _userMetadata;
}
@JsonProperty("userMetadata")
@JsonInclude(Include.NON_NULL)
public ArrayNode getUserMetadataJson() {
if (_userMetadata != null && _userMetadata.size() > 0) {
return _userMetadata;
}
return null;
}
public void setUserMetadata(JSONArray userMetadata) {
@JsonIgnore
public void setUserMetadata(ArrayNode userMetadata) {
this._userMetadata = userMetadata;
}
private void updateUserMetadata(String metaName, String valueString) {
for (int i = 0; i < _userMetadata.length(); i++) {
try {
JSONObject obj = _userMetadata.getJSONObject(i);
if (obj.getString("name").equals(metaName)) {
obj.put("value", valueString);
}
} catch (JSONException e) {
logger.error(ExceptionUtils.getFullStackTrace(e));
for (int i = 0; i < _userMetadata.size(); i++) {
ObjectNode obj = (ObjectNode)_userMetadata.get(i);
if (obj.get("name").asText("").equals(metaName)) {
obj.put("value", valueString);
}
}
}
@ -311,9 +373,4 @@ public class ProjectMetadata {
logger.error(ExceptionUtils.getFullStackTrace(e));
}
}
public static ProjectMetadata loadFromStream(InputStream f) throws IOException {
return ParsingUtilities.mapper.readValue(f, ProjectMetadata.class);
}
}

3
main/src/com/google/refine/io/ProjectMetadataUtilities.java
View File

@ -162,7 +162,6 @@ public class ProjectMetadataUtilities {
static protected ProjectMetadata loadFromFile(File metadataFile) throws Exception {
FileReader reader = new FileReader(metadataFile);
return ProjectMetadata.loadFromStream(reader);
return ParsingUtilities.mapper.readValue(reader, ProjectMetadata.class);
}
}

2
main/src/com/google/refine/model/Column.java
View File

@ -66,7 +66,7 @@ public class Column {
// from data package metadata Field.java:
private String type = "";
private String format = "";
private String format = "default";
private String title = "";
private String description = "";
private Map<String, Object> constraints = Collections.emptyMap();

1
main/src/com/google/refine/util/ParsingUtilities.java
View File

@ -84,6 +84,7 @@ public class ParsingUtilities {
module.addSerializer(double.class, new SerializationFilters.DoubleSerializer());
module.addSerializer(OffsetDateTime.class, new SerializationFilters.OffsetDateSerializer());
module.addSerializer(LocalDateTime.class, new SerializationFilters.LocalDateSerializer());
module.addDeserializer(LocalDateTime.class, new SerializationFilters.LocalDateDeserializer());
mapper.registerModule(module);
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);

18
main/src/com/google/refine/util/SerializationFilters.java
View File

@ -5,7 +5,11 @@ import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
import com.fasterxml.jackson.databind.ser.BeanPropertyWriter;
import com.fasterxml.jackson.databind.ser.PropertyFilter;
import com.fasterxml.jackson.databind.ser.PropertyWriter;
@ -120,4 +124,18 @@ public class SerializationFilters {
gen.writeString(ParsingUtilities.localDateToString(arg0));
}
}
public static class LocalDateDeserializer extends StdDeserializer<LocalDateTime> {
private static final long serialVersionUID = 93872874L;
public LocalDateDeserializer() {
super(LocalDateTime.class);
}
@Override
public LocalDateTime deserialize(JsonParser p, DeserializationContext ctxt)
throws IOException, JsonProcessingException {
return ParsingUtilities.stringToLocalDate(p.getValueAsString());
}
}
}

2
main/tests/data/example_project_metadata.json
View File

@ -1 +1 @@
{"_customMetadata":{},"created":"2018-10-12T10:24:28Z","tags":[],"importOptionMetadata":[{"guessCellValueTypes":false,"projectTags":[""],"ignoreLines":-1,"processQuotes":true,"fileSource":"query.tsv","encoding":"","separator":"\\t","storeBlankCellsAsNulls":true,"storeBlankRows":true,"skipDataLines":0,"includeFileSources":false,"headerLines":1,"limit":-1,"quoteCharacter":"\"","projectName":"CPVs"}],"creator":"","contributors":"","subject":"","rowCount":289,"preferenceStore":{"entries":{"scripting.starred-expressions":{"top":2147483647,"list":[],"class":"com.google.refine.preference.TopList"},"scripting.expressions":{"top":100,"list":[],"class":"com.google.refine.preference.TopList"}}},"modified":"2018-10-12T10:51:41Z","description":"","name":"CPVs","dirty":false,"title":"","homepage":"","image":"","license":"","version":""}
{"name":"numeric facet test","tags":[],"created":"2018-09-04T16:07:31Z","modified":"2018-09-04T17:02:31Z","creator":"","contributors":"","subject":"","description":"","rowCount":4,"title":"","homepage":"","image":"","license":"","version":"","customMetadata":{},"importOptionMetadata":[{"guessCellValueTypes":false,"projectTags":[""],"ignoreLines":-1,"processQuotes":true,"fileSource":"(clipboard)","encoding":"","separator":"\\t","storeBlankCellsAsNulls":true,"storeBlankRows":true,"skipDataLines":0,"includeFileSources":false,"headerLines":1,"limit":-1,"quoteCharacter":"\"","projectName":"numeric facet test"}]}

1
main/tests/data/example_project_metadata_save_mode.json Normal file
View File

@ -0,0 +1 @@
{"name":"numeric facet test","tags":[],"created":"2018-09-04T16:07:31Z","modified":"2018-09-04T17:02:31Z","creator":"","contributors":"","subject":"","description":"","rowCount":4,"title":"","homepage":"","image":"","license":"","version":"","customMetadata":{},"importOptionMetadata":[{"guessCellValueTypes":false,"projectTags":[""],"ignoreLines":-1,"processQuotes":true,"fileSource":"(clipboard)","encoding":"","separator":"\\t","storeBlankCellsAsNulls":true,"storeBlankRows":true,"skipDataLines":0,"includeFileSources":false,"headerLines":1,"limit":-1,"quoteCharacter":"\"","projectName":"numeric facet test"}],"password":"","encoding":"UTF-8","encodingConfidence":0,"preferences":{"entries":{"scripting.starred-expressions":{"class":"com.google.refine.preference.TopList","top":2147483647,"list":[]},"scripting.expressions":{"class":"com.google.refine.preference.TopList","top":100,"list":[]}}}}

20
main/tests/server/src/com/google/refine/tests/commands/project/SetProjectMetadataCommandTests.java
View File

@ -33,7 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.tests.commands.project;
import static org.mockito.Matchers.anyLong;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@ -46,9 +46,6 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
@ -56,11 +53,14 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.commands.project.SetProjectMetadataCommand;
import com.google.refine.model.Project;
import com.google.refine.tests.RefineTest;
import com.google.refine.util.ParsingUtilities;
public class SetProjectMetadataCommandTests extends RefineTest {
@ -86,7 +86,7 @@ public class SetProjectMetadataCommandTests extends RefineTest {
PrintWriter pw = null;
@BeforeMethod
public void SetUp() throws JSONException {
public void SetUp() throws IOException {
projMan = mock(ProjectManager.class);
ProjectManager.singleton = projMan;
proj = mock(Project.class);
@ -97,7 +97,7 @@ public class SetProjectMetadataCommandTests extends RefineTest {
SUT = new SetProjectMetadataCommand();
ProjectMetadata metadata = new ProjectMetadata();
metadata.setUserMetadata(new JSONArray("[ {name: clientID, display: true} ]"));
metadata.setUserMetadata((ArrayNode) ParsingUtilities.mapper.readTree("[ {name: \"clientID\", display: true} ]"));
// mock dependencies
when(request.getParameter("project")).thenReturn(PROJECT_ID);
@ -162,7 +162,7 @@ public class SetProjectMetadataCommandTests extends RefineTest {
* @throws JSONException
*/
@Test
public void setUserMetadataFieldTest() throws JSONException {
public void setUserMetadataFieldTest() {
when(request.getParameter("name")).thenReturn("clientID");
when(request.getParameter("value")).thenReturn("IBM");
@ -189,9 +189,9 @@ public class SetProjectMetadataCommandTests extends RefineTest {
}
verify(pw, times(1)).write("{ \"code\" : \"ok\" }");
JSONObject obj = (JSONObject) proj.getMetadata().getUserMetadata().get(0);
Assert.assertEquals(obj.get("name"), "clientID");
Assert.assertEquals(obj.get("value"), "IBM");
ObjectNode obj = (ObjectNode) proj.getMetadata().getUserMetadata().get(0);
Assert.assertEquals(obj.get("name").asText(), "clientID");
Assert.assertEquals(obj.get("value").asText(), "IBM");
}
@Test

2
main/tests/server/src/com/google/refine/tests/importers/ImporterTest.java
View File

@ -25,7 +25,7 @@ import com.google.refine.tests.RefineServletStub;
import com.google.refine.tests.RefineTest;
import com.google.refine.util.ParsingUtilities;
abstract class ImporterTest extends RefineTest {
public abstract class ImporterTest extends RefineTest {
//mock dependencies
protected Project project;
protected ProjectMetadata metadata;

13
main/tests/server/src/com/google/refine/tests/io/ProjectMetadataTests.java
View File

@ -5,18 +5,21 @@ import java.io.InputStream;
import org.apache.commons.io.IOUtils;
import org.testng.annotations.Test;
import com.google.refine.model.metadata.ProjectMetadata;
import com.google.refine.tests.util.TestUtils;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.ProjectMetadata;
public class ProjectMetadataTests {
@Test
public void serializeProjectMetadata() throws IOException {
ProjectMetadata metadata = new ProjectMetadata();
InputStream f = ProjectMetadataTests.class.getClassLoader().getResourceAsStream("example_project_metadata.json");
String json = IOUtils.toString(f);
f = ProjectMetadataTests.class.getClassLoader().getResourceAsStream("example_project_metadata.json");
metadata.loadFromStream(f);
f = ProjectMetadataTests.class.getClassLoader().getResourceAsStream("example_project_metadata_save_mode.json");
String fullJson = IOUtils.toString(f);
f = ProjectMetadataTests.class.getClassLoader().getResourceAsStream("example_project_metadata_save_mode.json");
ProjectMetadata metadata = ParsingUtilities.mapper.readValue(f, ProjectMetadata.class);
TestUtils.isSerializedTo(metadata, json);
TestUtils.isSerializedTo(metadata, fullJson, true);
}
}