Add a lot of tests in the schema

This commit is contained in:
Antonin Delpeuch 2018-02-27 18:44:49 +00:00
parent 7cae9455da
commit 6f9636176b
31 changed files with 840 additions and 148 deletions

View File

@ -5,8 +5,10 @@ import java.util.Date;
import java.util.Map;
import java.util.Map.Entry;
import java.text.ParseException;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
@ -16,9 +18,19 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableMap;
/**
* A constant for a time value, accepting a number of formats
* which determine the precision of the parsed value.
*
* @author Antonin Delpeuch
*
*/
public class WbDateConstant implements WbExpression<TimeValue> {
/**
* Map of formats accepted by the parser. Each format is associated
* to the time precision it induces (an integer according to Wikibase's data model).
*/
public static Map<SimpleDateFormat,Integer> acceptedFormats = ImmutableMap.<SimpleDateFormat,Integer>builder()
.put(new SimpleDateFormat("yyyy"), 9)
.put(new SimpleDateFormat("yyyy-MM"), 10)
@ -31,30 +43,49 @@ public class WbDateConstant implements WbExpression<TimeValue> {
private TimeValue parsed;
private String origDatestamp;
/**
* Constructor. Used for deserialization from JSON.
* The object will be constructed even if the time cannot
* be parsed (it will evaluate to null) in {@link evaluate}.
*
* @param origDatestamp
* the date value as a string
*/
@JsonCreator
public WbDateConstant(
@JsonProperty("value") String origDatestamp) {
Validate.notNull(origDatestamp);
this.setOrigDatestamp(origDatestamp);
}
@Override
public TimeValue evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
if (parsed == null) {
throw new SkipSchemaExpressionException();
}
return parsed;
}
/**
* Parses a timestamp into a Wikibase {@link TimeValue}. The
* precision is automatically inferred from the format.
*
* @param datestamp
* the time to parse
* @return
* @throws ParseException
* if the time cannot be parsed
*/
public static TimeValue parse(String datestamp) throws ParseException {
Date date = null;
int precision = 9; // default precision (will be overridden)
for(Entry<SimpleDateFormat,Integer> entry : acceptedFormats.entrySet()) {
try {
date = entry.getKey().parse(datestamp);
ParsePosition position = new ParsePosition(0);
String trimmedDatestamp = datestamp.trim();
date = entry.getKey().parse(trimmedDatestamp, position);
// Ignore parses which failed or do not consume all the input
if (date != null && position.getIndex() == trimmedDatestamp.length()) {
precision = entry.getValue();
} catch (ParseException e) {
continue;
break;
}
}
if (date == null) {
@ -71,13 +102,16 @@ public class WbDateConstant implements WbExpression<TimeValue> {
(byte) calendar.get(Calendar.MINUTE),
(byte) calendar.get(Calendar.SECOND),
(byte) precision,
1,
0,
1,
calendar.getTimeZone().getRawOffset()/3600000,
TimeValue.CM_GREGORIAN_PRO);
}
}
/**
* @return the original datestamp
*/
@JsonProperty("value")
public String getOrigDatestamp() {
return origDatestamp;
@ -88,9 +122,17 @@ public class WbDateConstant implements WbExpression<TimeValue> {
try {
this.parsed = parse(origDatestamp);
} catch(ParseException e) {
this.parsed = null;
throw new IllegalArgumentException("Invalid datestamp provided: "+origDatestamp);
}
}
@Override
public boolean equals(Object other) {
if(other == null || !WbDateConstant.class.isInstance(other)) {
return false;
}
WbDateConstant otherConstant = (WbDateConstant)other;
return origDatestamp.equals(otherConstant.getOrigDatestamp());
}
}

View File

@ -3,11 +3,18 @@ package org.openrefine.wikidata.schema;
import java.text.ParseException;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import com.google.refine.model.Cell;
/**
* An expression that represents a time value, extracted from a string.
* A number of formats are recognized, see {@link WbDateConstant} for details.
*
* @author Antonin Delpeuch
*
*/
public class WbDateVariable extends WbVariableExpr<TimeValue> {
@Override
@ -20,4 +27,9 @@ public class WbDateVariable extends WbVariableExpr<TimeValue> {
throw new SkipSchemaExpressionException();
}
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbDateVariable.class);
}
}

View File

@ -1,5 +1,6 @@
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.entityvalues.SuggestedItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
@ -19,7 +20,9 @@ public class WbItemConstant implements WbExpression<ItemIdValue> {
public WbItemConstant(
@JsonProperty("qid") String qid,
@JsonProperty("label") String label) {
Validate.notNull(qid);
this.qid = qid;
Validate.notNull(label);
this.label = label;
}
@ -40,4 +43,13 @@ public class WbItemConstant implements WbExpression<ItemIdValue> {
public String getLabel() {
return label;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbItemConstant.class.isInstance(other)) {
return false;
}
WbItemConstant otherConstant = (WbItemConstant)other;
return (qid.equals(otherConstant.getQid()) && label.equals(otherConstant.getLabel()));
}
}

View File

@ -5,17 +5,35 @@ import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
import com.google.refine.model.Recon.Judgment;
/**
* An item that depends on a reconciled value in a column.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class WbItemVariable extends WbVariableExpr<ItemIdValue> {
@JsonCreator
public WbItemVariable() {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly
* used as a convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from
*/
public WbItemVariable(String columnName) {
setColumnName(columnName);
}
@Override
public ItemIdValue fromCell(Cell cell, ExpressionContext ctxt) throws SkipSchemaExpressionException {
if (cell.recon != null
@ -25,4 +43,9 @@ public class WbItemVariable extends WbVariableExpr<ItemIdValue> {
}
throw new SkipSchemaExpressionException();
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbItemVariable.class);
}
}

View File

@ -1,6 +1,8 @@
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.interfaces.WikimediaLanguageCodes;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -8,9 +10,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A constant that represents a Wikimedia language code.
*
* TODO: migrate to a class more specific than String, with validation.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class WbLanguageConstant implements WbExpression<String> {
@ -22,22 +22,58 @@ public class WbLanguageConstant implements WbExpression<String> {
public WbLanguageConstant(
@JsonProperty("id") String langId,
@JsonProperty("label") String langLabel) {
_langId = langId;
_langId = normalizeLanguageCode(langId);
Validate.notNull(_langId, "A valid language code must be provided.");
Validate.notNull(langLabel);
_langLabel = langLabel;
}
/**
* Checks that a language code is valid and returns its preferred
* version (converting deprecated language codes to their better values).
*
* @param lang
* a Wikimedia language code
* @return
* the normalized code, or null if the code is invalid.
*/
public static String normalizeLanguageCode(String lang) {
try {
WikimediaLanguageCodes.getLanguageCode(lang);
return WikimediaLanguageCodes.fixLanguageCodeIfDeprecated(lang);
} catch(IllegalArgumentException e) {
return null;
}
}
@Override
public String evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException {
return _langId;
}
/**
* @return the language code for this language
*/
@JsonProperty("id")
public String getLang() {
return _langId;
}
/**
* @return the name of the language in itself
*/
@JsonProperty("label")
public String getLabel() {
return _langLabel;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbLanguageConstant.class.isInstance(other)) {
return false;
}
WbLanguageConstant otherConstant = (WbLanguageConstant)other;
return _langId.equals(otherConstant.getLang()) && _langLabel.equals(otherConstant.getLabel());
}
}

View File

@ -3,26 +3,47 @@ package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.interfaces.WikimediaLanguageCodes;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
/**
* A language variable generates a language code from a cell.
* It checks its values against a known list of valid language codes
* and fixes on the fly the deprecated ones (see {@link WbLanguageConstant}).
*/
public class WbLanguageVariable extends WbVariableExpr<String> {
@JsonCreator
public WbLanguageVariable() {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly
* used as a convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from
*/
public WbLanguageVariable(String columnName) {
setColumnName(columnName);
}
@Override
public String fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException {
if (cell.value != null && !cell.value.toString().isEmpty()) {
String code = cell.value.toString().trim();
try {
// this just checks that the language code is known
WikimediaLanguageCodes.getLanguageCode(code);
return cell.value.toString();
} catch(IllegalArgumentException e) {
;
String normalized = WbLanguageConstant.normalizeLanguageCode(code);
if (normalized != null) {
return normalized;
}
}
throw new SkipSchemaExpressionException();
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbLanguageVariable.class);
}
}

View File

@ -2,6 +2,7 @@ package org.openrefine.wikidata.schema;
import java.text.ParseException;
import org.apache.commons.lang.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
@ -9,7 +10,12 @@ import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A constant for a geographical location. The accepted format is lat,lng or lat/lng.
*
* @author Antonin Delpeuch
*
*/
public class WbLocationConstant implements WbExpression<GlobeCoordinatesValue> {
public static final double defaultPrecision = GlobeCoordinatesValue.PREC_TEN_MICRO_DEGREE;
@ -19,15 +25,22 @@ public class WbLocationConstant implements WbExpression<GlobeCoordinatesValue> {
@JsonCreator
public WbLocationConstant(
@JsonProperty("value") String origValue) {
@JsonProperty("value") String origValue) throws ParseException {
this.value = origValue;
try {
Validate.notNull(origValue);
this.parsed = parse(origValue);
} catch (ParseException e) {
this.parsed = null;
}
Validate.notNull(this.parsed);
}
/**
* Parses a string to a location.
*
* @param expr
* the string to parse
* @return
* the parsed location
* @throws ParseException
*/
public static GlobeCoordinatesValue parse(String expr) throws ParseException {
double lat = 0;
double lng = 0;
@ -52,13 +65,28 @@ public class WbLocationConstant implements WbExpression<GlobeCoordinatesValue> {
@Override
public GlobeCoordinatesValue evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
if (parsed == null)
throw new SkipSchemaExpressionException();
return parsed;
}
/**
* @return the original value as a string.
*/
@JsonProperty("value")
public String getValue() {
return value;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbLocationConstant.class.isInstance(other)) {
return false;
}
WbLocationConstant otherConstant = (WbLocationConstant)other;
return value.equals(otherConstant.getValue());
}
@Override
public int hashCode() {
return value.hashCode();
}
}

View File

@ -20,4 +20,9 @@ public class WbLocationVariable extends WbVariableExpr<GlobeCoordinatesValue> {
throw new SkipSchemaExpressionException();
}
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbLocationVariable.class);
}
}

View File

@ -27,10 +27,11 @@ public class WbMonolingualExpr implements WbExpression<MonolingualTextValue> {
public MonolingualTextValue evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
String text = getValueExpr().evaluate(ctxt).getString();
if (text.isEmpty())
throw new SkipSchemaExpressionException();
try {
String lang = getLanguageExpr().evaluate(ctxt);
if (lang.isEmpty()) {
return Datamodel.makeMonolingualTextValue(text, lang);
} catch(SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning(
"monolingual-text-without-language",
null,
@ -40,9 +41,6 @@ public class WbMonolingualExpr implements WbExpression<MonolingualTextValue> {
ctxt.addWarning(warning);
throw new SkipSchemaExpressionException();
}
return Datamodel.makeMonolingualTextValue(text, lang);
}
@JsonProperty("language")

View File

@ -1,5 +1,6 @@
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
@ -7,6 +8,14 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* An expression that represent a term (label, description or alias).
* The structure is slightly different from other expressions because
* we need to call different methods on {@link ItemUpdate}.
*
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class WbNameDescExpr {
@ -23,7 +32,9 @@ public class WbNameDescExpr {
public WbNameDescExpr(
@JsonProperty("name_type") NameDescrType type,
@JsonProperty("value") WbMonolingualExpr value) {
Validate.notNull(type);
this.type = type;
Validate.notNull(value);
this.value = value;
}

View File

@ -9,7 +9,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A constant property, that does not change depending on the row
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class WbPropConstant implements WbExpression<PropertyIdValue> {
@ -48,4 +48,13 @@ public class WbPropConstant implements WbExpression<PropertyIdValue> {
return datatype;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbPropConstant.class.isInstance(other)) {
return false;
}
WbPropConstant otherConstant = (WbPropConstant)other;
return pid.equals(otherConstant.getPid()) && label.equals(otherConstant.getLabel()) && datatype.equals(otherConstant.getDatatype());
}
}

View File

@ -40,9 +40,7 @@ public class WbQuantityExpr implements WbExpression<QuantityValue> {
public QuantityValue evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
StringValue amount = getLanguageExpr().evaluate(ctxt);
if (amount == null || amount.getString().isEmpty()) {
throw new SkipSchemaExpressionException();
}
// we know the amount is nonnull, nonempty here
BigDecimal parsedAmount = null;
try {

View File

@ -4,16 +4,33 @@ import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
/**
* A variable that returns a simple string value.
*
* @author antonin
* @author Antonin Delpeuch
*
*/
public class WbStringVariable extends WbVariableExpr<StringValue> {
@JsonCreator
public WbStringVariable() {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly
* used as a convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from
*/
public WbStringVariable(String columnName) {
setColumnName(columnName);
}
@Override
public StringValue fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException {
@ -22,4 +39,9 @@ public class WbStringVariable extends WbVariableExpr<StringValue> {
}
throw new SkipSchemaExpressionException();
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbStringVariable.class);
}
}

View File

@ -20,18 +20,6 @@ public abstract class WbVariableExpr<T> implements WbExpression<T> {
private String columnName;
/**
* Constructs a variable expression from a column name.
*
* @param columnName
* the name of the column the expression should draw its value from.
*/
@JsonCreator
public WbVariableExpr(
@JsonProperty("columnName") String columnName) {
this.columnName = columnName;
}
/**
* Constructs a variable without setting the column name yet.
*/
@ -87,4 +75,28 @@ public abstract class WbVariableExpr<T> implements WbExpression<T> {
* the corresponding Wikibase value
*/
public abstract T fromCell(Cell cell, ExpressionContext ctxt) throws SkipSchemaExpressionException;
/**
* Helper for equality methods of subclasses.
*
* @param other
* the object to compare
* @param columnName
* the column name to compare to
* @param targetClass
* the target class for equality
* @return
*/
protected boolean equalAsVariables(Object other, Class<? extends WbVariableExpr<?>> targetClass) {
if(other == null || !targetClass.isInstance(other)) {
return false;
}
return columnName.equals(targetClass.cast(other).getColumnName());
}
@Override
public int hashCode() {
return columnName.hashCode();
}
}

View File

@ -0,0 +1,46 @@
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
public class WbDateConstantTest extends WbExpressionTest<TimeValue> {
private WbDateConstant year = new WbDateConstant("2018");
private WbDateConstant month = new WbDateConstant("2018-02");
private WbDateConstant day = new WbDateConstant("2018-02-27");
private WbDateConstant whitespace = new WbDateConstant(" 2018-02-27 ");
private WbDateConstant hour = new WbDateConstant("2018-02-27T13");
@Test
public void testSerialize() {
JacksonSerializationTest.canonicalSerialization(WbExpression.class, year,
"{\"type\":\"wbdateconstant\",\"value\":\"2018\"}");
JacksonSerializationTest.canonicalSerialization(WbExpression.class, day,
"{\"type\":\"wbdateconstant\",\"value\":\"2018-02-27\"}");
}
@Test
public void testEvaluate() {
evaluatesTo(Datamodel.makeTimeValue(2018, (byte)1, (byte)1, (byte)0, (byte)0,
(byte)0, (byte)9, 0, 1, 0, TimeValue.CM_GREGORIAN_PRO), year);
evaluatesTo(Datamodel.makeTimeValue(2018, (byte)2, (byte)1, (byte)0, (byte)0,
(byte)0, (byte)10, 0, 1, 0, TimeValue.CM_GREGORIAN_PRO), month);
evaluatesTo(Datamodel.makeTimeValue(2018, (byte)2, (byte)27, TimeValue.CM_GREGORIAN_PRO), day);
evaluatesTo(Datamodel.makeTimeValue(2018, (byte)2, (byte)27, (byte)13, (byte)0,
(byte)0, (byte)12, 0, 1, 0, TimeValue.CM_GREGORIAN_PRO), hour);
evaluatesTo(Datamodel.makeTimeValue(2018, (byte)2, (byte)27, TimeValue.CM_GREGORIAN_PRO), whitespace);
}
@Test(expectedExceptions=IllegalArgumentException.class)
public void testInvalid() {
new WbDateConstant("invalid format");
}
@Test(expectedExceptions=IllegalArgumentException.class)
public void testPartlyValid() {
new WbDateConstant("2018-partly valid");
}
}

View File

@ -0,0 +1,45 @@
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
public class WbDateVariableTest extends WbVariableTest<TimeValue> {
private TimeValue year = Datamodel.makeTimeValue(2018, (byte)1, (byte)1, (byte)0, (byte)0,
(byte)0, (byte)9, 0, 1, 0, TimeValue.CM_GREGORIAN_PRO);
private TimeValue day = Datamodel.makeTimeValue(2018, (byte)2, (byte)27, TimeValue.CM_GREGORIAN_PRO);
@Override
public WbVariableExpr<TimeValue> initVariableExpr() {
return new WbDateVariable();
}
@Test
public void testValidFormat() {
evaluatesTo(year, "2018");
evaluatesTo(day, "2018-02-27");
}
@Test
public void testWhitespace() {
evaluatesTo(year, " 2018");
evaluatesTo(day, "2018-02-27 ");
}
@Test
public void testSkipped() {
isSkipped(" 2018-XX");
isSkipped("invalid format");
}
// TODO accept parsed dates with default precision
@Test
public void testSerialize() {
JacksonSerializationTest.canonicalSerialization(WbExpression.class, variable,
"{\"type\":\"wbdatevariable\",\"columnName\":\"column A\"}");
}
}

View File

@ -0,0 +1,103 @@
package org.openrefine.wikidata.schema;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.io.Serializable;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.testing.TestingDataGenerator;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import com.google.refine.model.Cell;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Row;
import com.google.refine.tests.RefineTest;
public class WbExpressionTest<T> extends RefineTest {
protected Project project;
protected Row row;
protected ExpressionContext ctxt;
protected QAWarningStore warningStore;
@BeforeMethod
public void createProject() throws IOException, ModelException {
project = createCSVProject("Wikidata variable test project", "column A,column B,column C,column D\n"+
"value A,value B,value C,value D");
warningStore = new QAWarningStore();
row = project.rows.get(0);
ctxt = new ExpressionContext("http://www.wikidata.org/entity/", 0,
row, project.columnModel, warningStore);
}
/**
* Test that a particular expression evaluates to some object.
*
* @param expected
* the expected evaluation of the value
* @param expression
* the expression to evaluate
*/
public void evaluatesTo(T expected, WbExpression<T> expression) {
try {
T result = expression.evaluate(ctxt);
Assert.assertEquals(expected, result);
} catch (SkipSchemaExpressionException e) {
Assert.fail("Value was skipped by evaluator");
}
}
/**
* Test that a particular expression is skipped.
*
* @param expected
* the expected evaluation of the value
* @param expression
* the expression to evaluate
*/
public void isSkipped(WbExpression<T> expression) {
try {
expression.evaluate(ctxt);
Assert.fail("Value was not skipped by evaluator");
} catch (SkipSchemaExpressionException e) {
return;
}
}
/**
* Sets the context to a row with the given values.
*
* @param rowValues
* the list of row values. They can be cells or cell values.
*/
public void setRow(Object... rowValues) {
Row row = new Row(rowValues.length);
for(int i = 0; i != rowValues.length; i++) {
Object val = rowValues[i];
if(Cell.class.isInstance(val)) {
row.cells.add((Cell)val);
} else {
Cell cell = new Cell((Serializable)val, (Recon)null);
row.cells.add(cell);
}
}
ctxt = new ExpressionContext("http://www.wikidata.org/entity/", 0,
row, project.columnModel, warningStore);
}
/**
* Creates a make-shift reconciled cell for a given Qid.
*
* @param qid
* @return
* a cell for use in setRow
*/
public Cell recon(String qid) {
return TestingDataGenerator.makeMatchedCell(qid, qid);
}
}

View File

@ -0,0 +1,22 @@
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
public class WbItemConstantTest extends WbExpressionTest<ItemIdValue> {
private WbItemConstant constant = new WbItemConstant("Q42", "Douglas Adams");
@Test
public void testSerialize() {
JacksonSerializationTest.canonicalSerialization(WbExpression.class, constant,
"{\"type\":\"wbitemconstant\",\"qid\":\"Q42\",\"label\":\"Douglas Adams\"}");
}
@Test
public void testEvaluate() {
evaluatesTo(Datamodel.makeWikidataItemIdValue("Q42"), constant);
}
}

View File

@ -3,6 +3,7 @@ package org.openrefine.wikidata.schema;
import java.util.Collections;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
@ -48,4 +49,10 @@ public class WbItemVariableTest extends WbVariableTest<ItemIdValue> {
public void testUnreconciledCell() {
isSkipped("some value");
}
@Test
public void testSerialize() {
JacksonSerializationTest.canonicalSerialization(WbExpression.class, variable,
"{\"type\":\"wbitemvariable\",\"columnName\":\"column A\"}");
}
}

View File

@ -0,0 +1,33 @@
package org.openrefine.wikidata.schema;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.Test;
public class WbLanguageConstantTest extends WbExpressionTest<String> {
private WbLanguageConstant constant = new WbLanguageConstant("de", "Deutsch");
@Test
public void testEvaluation() {
evaluatesTo("de", constant);
}
@Test
public void testSerialization() {
JacksonSerializationTest.canonicalSerialization(WbExpression.class, constant,
"{\"type\":\"wblanguageconstant\",\"id\":\"de\",\"label\":\"Deutsch\"}");
}
@Test
public void testNormalizeLanguageCode() {
assertEquals("ku-latn", WbLanguageConstant.normalizeLanguageCode("ku-latn"));
assertEquals("de", WbLanguageConstant.normalizeLanguageCode("de"));
assertEquals("nb", WbLanguageConstant.normalizeLanguageCode("no"));
assertEquals("nb", WbLanguageConstant.normalizeLanguageCode("nb"));
assertNull(WbLanguageConstant.normalizeLanguageCode("non-existent language code"));
assertNull(WbLanguageConstant.normalizeLanguageCode(null));
}
}

View File

@ -1,5 +1,6 @@
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.Test;
public class WbLanguageVariableTest extends WbVariableTest<String> {
@ -11,7 +12,21 @@ public class WbLanguageVariableTest extends WbVariableTest<String> {
@Test
public void testValidLanguageCode() {
// TODO
evaluatesTo("en", "en");
evaluatesTo("nb", "no");
evaluatesTo("de", " de ");
}
@Test
public void testInvalidLanguageCode() {
isSkipped("unknown language code");
isSkipped((String)null);
isSkipped("");
}
@Test
public void testSerialize() {
JacksonSerializationTest.canonicalSerialization(WbExpression.class, variable,
"{\"type\":\"wblanguagevariable\",\"columnName\":\"column A\"}");
}
}

View File

@ -0,0 +1,61 @@
package org.openrefine.wikidata.schema;
import static org.junit.Assert.assertEquals;
import java.text.ParseException;
import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
public class WbLocationConstantTest extends WbExpressionTest<GlobeCoordinatesValue> {
private GlobeCoordinatesValue loc = Datamodel.makeGlobeCoordinatesValue(1.2345, 6.7890,
GlobeCoordinatesValue.PREC_TEN_MICRO_DEGREE, GlobeCoordinatesValue.GLOBE_EARTH);
private GlobeCoordinatesValue locWithPrecision = Datamodel.makeGlobeCoordinatesValue(1.2345, 6.7890,
0.1, GlobeCoordinatesValue.GLOBE_EARTH);
private String input = "1.2345,6.7890";
private String inputWithPrecision = "1.2345,6.7890,0.1";
public void testParseValid() throws ParseException {
assertEquals(loc, WbLocationConstant.parse(input));
assertEquals(locWithPrecision, WbLocationConstant.parse(inputWithPrecision));
}
@Test(expectedExceptions=ParseException.class)
public void testParseInvalid() throws ParseException {
WbLocationConstant.parse("some bad value");
}
@Test
public void testEvaluate() throws ParseException {
evaluatesTo(loc, new WbLocationConstant(input));
}
@Test
public void testEvaluateWithPrecision() throws ParseException {
evaluatesTo(locWithPrecision, new WbLocationConstant(inputWithPrecision));
}
@Test(expectedExceptions=ParseException.class)
public void constructInvalid() throws ParseException {
new WbLocationConstant("some bad value");
}
@Test(expectedExceptions=ParseException.class)
public void constructNotNumber() throws ParseException {
new WbLocationConstant("lat,lng");
}
@Test(expectedExceptions=ParseException.class)
public void constructtooManyParts() throws ParseException {
new WbLocationConstant("0.1,2.3,4.5,6.7");
}
@Test
public void testSerialization() throws ParseException {
JacksonSerializationTest.canonicalSerialization(WbExpression.class, new WbLocationConstant(input),
"{\"type\":\"wblocationconstant\",\"value\":\""+input+"\"}");
}
}

View File

@ -1,5 +1,6 @@
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
@ -39,4 +40,10 @@ public class WbLocationVariableTest extends WbVariableTest<GlobeCoordinatesValue
public void testEmpty() {
isSkipped("");
}
@Test
public void testSerialize() {
JacksonSerializationTest.canonicalSerialization(WbExpression.class, variable,
"{\"type\":\"wblocationvariable\",\"columnName\":\"column A\"}");
}
}

View File

@ -0,0 +1,44 @@
package org.openrefine.wikidata.schema;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
public class WbMonolingualExprTest extends WbExpressionTest<MonolingualTextValue> {
private WbMonolingualExpr expr = new WbMonolingualExpr(
new WbLanguageVariable("column A"),
new WbStringVariable("column B"));
@Test
public void testEvaluateConstant() {
evaluatesTo(Datamodel.makeMonolingualTextValue("hello", "en"),
new WbMonolingualExpr(new WbLanguageConstant("en", "English"),
new WbStringConstant("hello")));
}
@Test
public void testEvaluateVariable() {
setRow("en", "hello");
evaluatesTo(Datamodel.makeMonolingualTextValue("hello", "en"), expr);
}
@Test
public void testInvalidLanguageCode() {
setRow("ueuue", "my label");
isSkipped(expr);
}
@Test
public void testEmptyLanguageCode() {
setRow("", "my label");
isSkipped(expr);
}
@Test
public void testEmptyText() {
setRow("en", "");
isSkipped(expr);
}
}

View File

@ -0,0 +1,49 @@
package org.openrefine.wikidata.schema;
import static org.junit.Assert.assertEquals;
import java.util.Collections;
import org.openrefine.wikidata.testing.TestingDataGenerator;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
public class WbNameDescrExprTest extends WbExpressionTest<MonolingualTextValue> {
@Test
public void testContributeToLabel() {
WbNameDescExpr labelExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.LABEL,
TestingDataGenerator.getTestMonolingualExpr("fr", "français", "le croissant magnifique"));
ItemUpdate update = new ItemUpdate(Datamodel.makeWikidataItemIdValue("Q56"));
labelExpr.contributeTo(update, ctxt);
assertEquals(Collections.singletonList(Datamodel.makeMonolingualTextValue("le croissant magnifique", "fr")), update.getLabels());
}
@Test
public void testContributeToDescription() {
WbNameDescExpr descriptionExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.DESCRIPTION,
TestingDataGenerator.getTestMonolingualExpr("de", "Deutsch", "wunderschön"));
ItemUpdate update = new ItemUpdate(Datamodel.makeWikidataItemIdValue("Q56"));
descriptionExpr.contributeTo(update, ctxt);
assertEquals(Collections.singletonList(Datamodel.makeMonolingualTextValue("wunderschön", "de")), update.getDescriptions());
}
@Test
public void testContributeToAlias() {
WbNameDescExpr aliasExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS,
TestingDataGenerator.getTestMonolingualExpr("en", "English", "snack"));
ItemUpdate update = new ItemUpdate(Datamodel.makeWikidataItemIdValue("Q56"));
aliasExpr.contributeTo(update, ctxt);
assertEquals(Collections.singletonList(Datamodel.makeMonolingualTextValue("snack", "en")), update.getAliases());
}
@Test
public void testGetters() {
WbMonolingualExpr monolingualExpr = TestingDataGenerator.getTestMonolingualExpr("en", "English", "not sure what");
WbNameDescExpr aliasExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS,
monolingualExpr);
assertEquals(WbNameDescExpr.NameDescrType.ALIAS, aliasExpr.getType());
assertEquals(monolingualExpr, aliasExpr.getValue());
}
}

View File

@ -0,0 +1,22 @@
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
public class WbPropConstantTest extends WbExpressionTest<PropertyIdValue> {
private WbPropConstant constant = new WbPropConstant("P48", "my ID", "external-id");
@Test
public void testEvaluate() {
evaluatesTo(Datamodel.makeWikidataPropertyIdValue("P48"), constant);
}
@Test
public void testSerialize() {
JacksonSerializationTest.canonicalSerialization(WbExpression.class, constant,
"{\"type\":\"wbpropconstant\",\"pid\":\"P48\",\"label\":\"my ID\",\"datatype\":\"external-id\"}");
}
}

View File

@ -0,0 +1,44 @@
package org.openrefine.wikidata.schema;
import java.math.BigDecimal;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
public class WbQuantityExprTest extends WbExpressionTest<QuantityValue> {
private WbQuantityExpr exprWithUnit = new WbQuantityExpr(new WbStringVariable("column A"),
new WbItemVariable("column B"));
private WbQuantityExpr exprWithoutUnit = new WbQuantityExpr(new WbStringVariable("column A"),
null);
@Test
public void testWithoutUnit() throws SkipSchemaExpressionException {
setRow("4.00");
evaluatesTo(Datamodel.makeQuantityValue(new BigDecimal("4.00"), null, null, "1"), exprWithoutUnit);
}
public void testInvalidAmountWithoutUnit() {
setRow("hello");
isSkipped(exprWithoutUnit);
}
@Test
public void testWithUnit() throws SkipSchemaExpressionException {
setRow("56.094", recon("Q42"));
evaluatesTo(Datamodel.makeQuantityValue(new BigDecimal("56.094"), null, null, "http://www.wikidata.org/entity/Q42"), exprWithUnit);
}
public void testInvalidAmountWithUnit() throws SkipSchemaExpressionException {
setRow("invalid", recon("Q42"));
isSkipped(exprWithUnit);
}
public void testInvalidUnitWithAmount() throws SkipSchemaExpressionException {
setRow("56.094", "not reconciled");
isSkipped(exprWithUnit);
}
}

View File

@ -1,68 +0,0 @@
package org.openrefine.wikidata.schema;
import java.io.IOException;
import java.math.BigDecimal;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.tests.RefineTest;
public class WbQuantityValueTest extends RefineTest {
protected Project project;
protected Row row;
protected ExpressionContext ctxt;
protected QAWarningStore warningStore;
@BeforeMethod
public void createProject() throws IOException, ModelException {
project = createCSVProject("Wikidata quantity value test project",
"column A\nrow1");
warningStore = new QAWarningStore();
row = project.rows.get(0);
ctxt = new ExpressionContext("http://www.wikidata.org/entity/", 0,
row, project.columnModel, warningStore);
}
@Test
public void testWithoutUnit() throws SkipSchemaExpressionException {
WbQuantityExpr expr = new WbQuantityExpr(new WbStringConstant("4.00"), null);
Assert.assertEquals(Datamodel.makeQuantityValue(new BigDecimal("4.00"), null, null, "1"), expr.evaluate(ctxt));
}
@Test(expectedExceptions = SkipSchemaExpressionException.class)
public void testInvalidAmountWithoutUnit() throws SkipSchemaExpressionException {
WbQuantityExpr expr = new WbQuantityExpr(new WbStringConstant("hello"), null);
expr.evaluate(ctxt);
}
@Test
public void testWithUnit() throws SkipSchemaExpressionException {
WbItemConstant item = new WbItemConstant("Q42", "label");
WbQuantityExpr expr = new WbQuantityExpr(new WbStringConstant("56.094"), item);
Assert.assertEquals(Datamodel.makeQuantityValue(new BigDecimal("56.094"), null, null, "http://www.wikidata.org/entity/Q42"), expr.evaluate(ctxt));
}
@Test(expectedExceptions = SkipSchemaExpressionException.class)
public void testInvalidAmountWithUnit() throws SkipSchemaExpressionException {
WbItemConstant item = new WbItemConstant("Q42", "label");
WbQuantityExpr expr = new WbQuantityExpr(new WbStringConstant("invalid"), item);
expr.evaluate(ctxt);
}
@Test(expectedExceptions = SkipSchemaExpressionException.class)
public void testInvalidUnitWithAmount() throws SkipSchemaExpressionException {
WbItemVariable item = new WbItemVariable();
item.setColumnName("column A");
WbQuantityExpr expr = new WbQuantityExpr(new WbStringConstant("56.094"), item);
expr.evaluate(ctxt);
}
}

View File

@ -13,13 +13,9 @@ import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.tests.RefineTest;
public abstract class WbVariableTest<T> extends RefineTest {
public abstract class WbVariableTest<T> extends WbExpressionTest<T> {
protected WbVariableExpr<T> variable;
protected Project project;
protected Row row;
protected ExpressionContext ctxt;
protected QAWarningStore warningStore;
/**
* This should return a variable expression, to be tested with the helpers below.
@ -28,12 +24,7 @@ public abstract class WbVariableTest<T> extends RefineTest {
public abstract WbVariableExpr<T> initVariableExpr();
@BeforeMethod
public void createProject() throws IOException, ModelException {
project = createCSVProject("Wikidata variable test project", "column A\nrow1");
warningStore = new QAWarningStore();
row = project.rows.get(0);
ctxt = new ExpressionContext("http://www.wikidata.org/entity/", 0,
row, project.columnModel, warningStore);
public void setupVariable() throws IOException, ModelException {
variable = initVariableExpr();
variable.setColumnName("column A");
}
@ -59,12 +50,7 @@ public abstract class WbVariableTest<T> extends RefineTest {
*/
public void evaluatesTo(T expected, Cell cell) {
row.setCell(0, cell);
try {
T result = variable.evaluate(ctxt);
Assert.assertEquals(expected, result);
} catch (SkipSchemaExpressionException e) {
Assert.fail("Value was skipped by evaluator");
}
evaluatesTo(expected, variable);
}
/**
@ -83,11 +69,6 @@ public abstract class WbVariableTest<T> extends RefineTest {
*/
protected void isSkipped(Cell cell) {
row.setCell(0, cell);
try {
variable.evaluate(ctxt);
Assert.fail("Value was not skipped by evaluator");
} catch (SkipSchemaExpressionException e) {
return;
}
isSkipped(variable);
}
}

View File

@ -0,0 +1,45 @@
package org.openrefine.wikidata.testing;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import org.testng.Assert;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
public class JacksonSerializationTest {
private static ObjectMapper mapper = new ObjectMapper();
public static void testSerialize(Object pojo, String expectedJson) {
// Test that the pojo is correctly serialized
try {
JsonNode parsedExpectedJson = mapper.readValue(expectedJson, JsonNode.class);
String actualJson = mapper.writeValueAsString(pojo);
JsonNode parsedActualJson = mapper.readValue(actualJson, JsonNode.class);
assertEquals(parsedExpectedJson, parsedActualJson);
} catch (JsonProcessingException e) {
e.printStackTrace();
Assert.fail("Failed to serialize object");
} catch (IOException e) {
e.printStackTrace();
Assert.fail("Invalid test JSON provided");
}
}
public static void testDeserialize(Class targetClass, Object pojo, String inputJson) {
try {
assertEquals(pojo, mapper.readValue(inputJson, targetClass));
} catch (IOException e) {
e.printStackTrace();
Assert.fail("Failed to deserialize object");
}
}
public static void canonicalSerialization(Class targetClass, Object pojo, String json) {
testSerialize(pojo, json);
testDeserialize(targetClass, pojo, json);
}
}

View File

@ -2,6 +2,9 @@ package org.openrefine.wikidata.testing;
import java.util.Collections;
import org.openrefine.wikidata.schema.WbLanguageConstant;
import org.openrefine.wikidata.schema.WbMonolingualExpr;
import org.openrefine.wikidata.schema.WbStringConstant;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.schema.entityvalues.ReconPropertyIdValue;
@ -54,4 +57,8 @@ public class TestingDataGenerator {
public static ReconEntityIdValue makeMatchedPropertyIdValue(String pid, String name) {
return new ReconPropertyIdValue(makeMatchedRecon(pid, name), name);
}
public static WbMonolingualExpr getTestMonolingualExpr(String langCode, String langLabel, String text) {
return new WbMonolingualExpr(new WbLanguageConstant(langCode, langLabel), new WbStringConstant(text));
}
}