From e0cdb915207be20bf79e293114254f1867e9a666 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Thu, 1 Mar 2018 00:31:30 +0000 Subject: [PATCH] Better testing for the schema package --- .../wikidata/exporters/QSValuePrinter.java | 1 - .../exporters/QuickStatementsExporter.java | 11 +- .../PerformWikibaseEditsOperation.java | 31 ++- .../wikidata/schema/ItemUpdate.java | 197 ++++++++++++++---- .../wikidata/schema/WbDateConstant.java | 5 + .../wikidata/schema/WbDateVariable.java | 12 +- .../wikidata/schema/WbItemConstant.java | 5 + .../wikidata/schema/WbItemDocumentExpr.java | 25 ++- .../wikidata/schema/WbLanguageConstant.java | 5 + .../wikidata/schema/WbLanguageVariable.java | 1 - .../wikidata/schema/WbLocationVariable.java | 11 + .../wikidata/schema/WbMonolingualExpr.java | 15 ++ .../wikidata/schema/WbNameDescExpr.java | 23 ++ .../wikidata/schema/WbPropConstant.java | 4 + .../wikidata/schema/WbQuantityExpr.java | 1 - .../wikidata/schema/WbReferenceExpr.java | 24 ++- .../wikidata/schema/WbSnakExpr.java | 25 ++- .../wikidata/schema/WbStatementExpr.java | 25 +++ .../wikidata/schema/WbStatementGroupExpr.java | 20 ++ .../wikidata/schema/WikibaseSchema.java | 68 +++--- .../entityvalues/ReconEntityIdValue.java | 51 +++-- .../history_of_medicine_normalized.json | 1 + .../wikidata/tests/data/schema/inception.json | 1 + .../exporters/QSValuePrinterTest.java | 3 - .../wikidata/schema/ItemUpdateTest.java | 139 ++++++++++++ .../schema/SchemaSerializationTest.java | 32 --- .../wikidata/schema/WbExpressionTest.java | 6 +- .../schema/WbItemDocumentExprTest.java | 74 +++++++ .../schema/WbLocationConstantTest.java | 2 +- .../wikidata/schema/WbNameDescExprTest.java | 75 +++++++ .../wikidata/schema/WbNameDescrExprTest.java | 49 ----- .../wikidata/schema/WbReferenceExprTest.java | 62 ++++++ .../wikidata/schema/WbSnakExprTest.java | 33 +++ .../wikidata/schema/WbStatementExprTest.java | 122 ++++++++++- .../schema/WbStatementGroupExprTest.java | 72 +++++++ .../wikidata/schema/WbVariableTest.java | 6 - .../wikidata/schema/WikibaseSchemaTest.java | 112 ++++++++++ .../entityvalues/ReconEntityIdValueTest.java | 6 + .../testing/JacksonSerializationTest.java | 6 +- .../testing/TestingDataGenerator.java | 10 +- 40 files changed, 1150 insertions(+), 221 deletions(-) create mode 100644 extensions/wikidata/tests/data/schema/history_of_medicine_normalized.json create mode 100644 extensions/wikidata/tests/data/schema/inception.json create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ItemUpdateTest.java delete mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/SchemaSerializationTest.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemDocumentExprTest.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescExprTest.java delete mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescrExprTest.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbReferenceExprTest.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbSnakExprTest.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementGroupExprTest.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java diff --git a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java index 2f5f7c695..f5b5ae3b6 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java @@ -7,7 +7,6 @@ import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; -import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.datamodel.interfaces.QuantityValue; import org.wikidata.wdtk.datamodel.interfaces.StringValue; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java index ea68f973c..14291930f 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java @@ -2,10 +2,9 @@ package org.openrefine.wikidata.exporters; import java.io.IOException; import java.io.Writer; -import java.math.BigDecimal; import java.util.List; -import java.util.Locale; import java.util.Properties; +import java.util.Set; import com.google.refine.browsing.Engine; import com.google.refine.exporters.WriterExporter; @@ -16,18 +15,12 @@ import org.openrefine.wikidata.schema.WikibaseSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.wikidata.wdtk.datamodel.interfaces.Claim; -import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue; -import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; -import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; -import org.wikidata.wdtk.datamodel.interfaces.QuantityValue; import org.wikidata.wdtk.datamodel.interfaces.Reference; import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; -import org.wikidata.wdtk.datamodel.interfaces.StringValue; -import org.wikidata.wdtk.datamodel.interfaces.TimeValue; import org.wikidata.wdtk.datamodel.interfaces.Value; import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor; @@ -77,7 +70,7 @@ public class QuickStatementsExporter implements WriterExporter { } } - protected void translateNameDescr(String qid, List values, String prefix, ItemIdValue id, Writer writer) throws IOException { + protected void translateNameDescr(String qid, Set values, String prefix, ItemIdValue id, Writer writer) throws IOException { for (MonolingualTextValue value : values) { writer.write(qid+"\t"); writer.write(prefix); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java b/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java index 5b18b5d42..5467ea5c4 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java @@ -6,8 +6,10 @@ import java.io.Writer; import java.util.HashMap; import java.util.List; import java.util.ArrayList; +import java.util.Collections; import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; import org.json.JSONException; import org.json.JSONObject; @@ -199,7 +201,6 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation { @Override public void run() { - // TODO Auto-generated method stub WebResourceFetcherImpl.setUserAgent("OpenRefine Wikidata extension"); ConnectionManager manager = ConnectionManager.getInstance(); @@ -208,8 +209,8 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation { } ApiConnection connection = manager.getConnection(); - WikibaseDataFetcher wbdf = new WikibaseDataFetcher(connection, _schema.getBaseUri()); - WikibaseDataEditor wbde = new WikibaseDataEditor(connection, _schema.getBaseUri()); + WikibaseDataFetcher wbdf = new WikibaseDataFetcher(connection, _schema.getBaseIri()); + WikibaseDataEditor wbde = new WikibaseDataEditor(connection, _schema.getBaseIri()); wbde.setEditAsBot(true); //wbde.disableEditing(); @@ -284,9 +285,9 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation { ItemDocument itemDocument = factory.getItemDocument( update.getItemId(), - update.getLabels(), - update.getDescriptions(), - update.getAliases(), + update.getLabels().stream().collect(Collectors.toList()), + update.getDescriptions().stream().collect(Collectors.toList()), + update.getAliases().stream().collect(Collectors.toList()), update.getAddedStatementGroups(), new HashMap(), 0L); @@ -296,20 +297,14 @@ public class PerformWikibaseEditsOperation extends EngineDependentOperation { } else { // Existing item ItemDocument currentDocument = (ItemDocument)currentDocs.get(update.getItemId().getId()); - TermStatementUpdate itemUpdate = new TermStatementUpdate( currentDocument, update.getAddedStatements(), - update.getDeletedStatements(), update.getLabels(), - update.getDescriptions(), - update.getAliases(), - new ArrayList() - ); - System.out.println(itemUpdate.getJsonUpdateString()); wbde.updateTermsStatements(currentDocument, - update.getLabels(), - update.getDescriptions(), - update.getAliases(), + update.getLabels().stream().collect(Collectors.toList()), + update.getDescriptions().stream().collect(Collectors.toList()), + update.getAliases().stream().collect(Collectors.toList()), new ArrayList(), - update.getAddedStatements(), - update.getDeletedStatements(), _summary); + update.getAddedStatements().stream().collect(Collectors.toList()), + update.getDeletedStatements().stream().collect(Collectors.toList()), + _summary); } } catch (MediaWikiApiErrorException e) { // TODO Auto-generated catch block diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/ItemUpdate.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/ItemUpdate.java index edf863b63..797344eb5 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/ItemUpdate.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/ItemUpdate.java @@ -3,8 +3,12 @@ package org.openrefine.wikidata.schema; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import java.util.HashMap; +import java.util.HashSet; +import org.jsoup.helper.Validate; import org.wikidata.wdtk.datamodel.implementation.StatementGroupImpl; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; @@ -19,58 +23,104 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; * but before fetching the current content of the item (this is why it does not * extend StatementsUpdate). * - * @author antonin + * @author Antonin Delpeuch */ public class ItemUpdate { private ItemIdValue qid; - private List addedStatements; - private List deletedStatements; - private List labels; - private List descriptions; - private List aliases; + private Set addedStatements; + private Set deletedStatements; + private Set labels; + private Set descriptions; + private Set aliases; + /** + * Constructor. + * + * @param qid + * the subject of the document. It can be a reconciled item value for new items. + */ public ItemUpdate(ItemIdValue qid) { + Validate.notNull(qid); this.qid = qid; - this.addedStatements = new ArrayList(); - this.deletedStatements = new ArrayList(); - this.labels = new ArrayList(); - this.descriptions = new ArrayList(); - this.aliases = new ArrayList(); + this.addedStatements = new HashSet<>(); + this.deletedStatements = new HashSet(); + this.labels = new HashSet(); + this.descriptions = new HashSet(); + this.aliases = new HashSet(); } - public void addStatement(Statement s) { - addedStatements.add(s); + /** + * Mark a statement for insertion. If it matches an existing + * statement, it will update the statement instead. + * + * @param statement + * the statement to add or update + */ + public void addStatement(Statement statement) { + addedStatements.add(statement); } - public void deleteStatement(Statement s) { - deletedStatements.add(s); + /** + * Mark a statement for deletion. If no such statement exists, + * nothing will be deleted. + * + * @param statement + * the statement to delete + */ + public void deleteStatement(Statement statement) { + deletedStatements.add(statement); } - public void addStatements(List l) { - addedStatements.addAll(l); + /** + * Add a list of statement, as in {@link addStatement}. + * + * @param statements + * the statements to add + */ + public void addStatements(Set statements) { + addedStatements.addAll(statements); } - public void deleteStatements(List l) { - deletedStatements.addAll(l); + /** + * Delete a list of statements, as in {@link deleteStatement}. + * + * @param statements + * the statements to delete + */ + public void deleteStatements(Set statements) { + deletedStatements.addAll(statements); } + /** + * @return the subject of the item + */ public ItemIdValue getItemId() { return qid; } - public List getAddedStatements() { + /** + * @return the set of all added statements + */ + public Set getAddedStatements() { return addedStatements; } - public List getDeletedStatements() { + /** + * @return the list of all deleted statements + */ + public Set getDeletedStatements() { return deletedStatements; } /** * Merges all the changes in other into this instance. - * @param other: the other change that should be merged + * Both updates should have the same subject. + * + * @param other + * the other change that should be merged */ public void merge(ItemUpdate other) { + Validate.isTrue(qid.equals(other.getItemId())); addStatements(other.getAddedStatements()); deleteStatements(other.getDeletedStatements()); labels.addAll(other.getLabels()); @@ -78,6 +128,10 @@ public class ItemUpdate { aliases.addAll(other.getAliases()); } + /** + * @return true when this change is empty + * (no statements or terms changed) + */ public boolean isNull() { return (addedStatements.isEmpty() && deletedStatements.isEmpty() @@ -86,27 +140,57 @@ public class ItemUpdate { && aliases.isEmpty()); } - public void addLabel(MonolingualTextValue val) { - labels.add(val); + /** + * Adds a label to the item. It will override any + * existing label in this language. + * + * @param label + * the label to add + */ + public void addLabel(MonolingualTextValue label) { + labels.add(label); } - public void addDescription(MonolingualTextValue val) { - descriptions.add(val); + /** + * Adds a description to the item. It will override any existing + * description in this language. + * + * @param description + * the description to add + */ + public void addDescription(MonolingualTextValue description) { + descriptions.add(description); } - public void addAlias(MonolingualTextValue val) { - aliases.add(val); + /** + * Adds an alias to the item. It will be added to any existing + * aliases in that language. + * + * @param alias + * the alias to add + */ + public void addAlias(MonolingualTextValue alias) { + aliases.add(alias); } - public List getLabels() { + /** + * @return the list of updated labels + */ + public Set getLabels() { return labels; } - public List getDescriptions() { + /** + * @return the list of updated descriptions + */ + public Set getDescriptions() { return descriptions; } - public List getAliases() { + /** + * @return the list of updated aliases + */ + public Set getAliases() { return aliases; } @@ -164,12 +248,12 @@ public class ItemUpdate { */ public void normalizeLabelsAndAliases() { // Ensure that we are only adding aliases with labels - List labelLanguages = new ArrayList(); - for(MonolingualTextValue label : labels) { - labelLanguages.add(label.getLanguageCode()); - } + Set labelLanguages = labels.stream() + .map(l -> l.getLanguageCode()) + .collect(Collectors.toSet()); + System.out.println(labelLanguages); - List filteredAliases = new ArrayList<>(); + Set filteredAliases = new HashSet<>(); for(MonolingualTextValue alias : aliases) { if(!labelLanguages.contains(alias.getLanguageCode())) { labelLanguages.add(alias.getLanguageCode()); @@ -182,9 +266,48 @@ public class ItemUpdate { } /** - * is this update about a new item? + * Is this update about a new item? */ public boolean isNew() { return "Q0".equals(getItemId().getId()); } + + @Override + public boolean equals(Object other) { + if(other == null || !ItemUpdate.class.isInstance(other)) { + return false; + } + ItemUpdate otherUpdate = (ItemUpdate)other; + return qid.equals(otherUpdate.getItemId())&& + addedStatements.equals(otherUpdate.getAddedStatements()) && + deletedStatements.equals(otherUpdate.getDeletedStatements()) && + labels.equals(otherUpdate.getLabels()) && + descriptions.equals(otherUpdate.getDescriptions()) && + aliases.equals(otherUpdate.getAliases()); + } + + @Override + public int hashCode() { + return qid.hashCode() + addedStatements.hashCode() + deletedStatements.hashCode() + + labels.hashCode() + descriptions.hashCode() + aliases.hashCode(); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append(""); + return builder.toString(); + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java index adffe4cef..85337d9bc 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java @@ -134,5 +134,10 @@ public class WbDateConstant implements WbExpression { WbDateConstant otherConstant = (WbDateConstant)other; return origDatestamp.equals(otherConstant.getOrigDatestamp()); } + + @Override + public int hashCode() { + return origDatestamp.hashCode(); + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java index ff80ab486..fbd92ebd3 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java @@ -3,9 +3,10 @@ package org.openrefine.wikidata.schema; import java.text.ParseException; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; -import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.TimeValue; +import com.fasterxml.jackson.annotation.JsonCreator; + import com.google.refine.model.Cell; /** @@ -16,6 +17,15 @@ import com.google.refine.model.Cell; * */ public class WbDateVariable extends WbVariableExpr { + + @JsonCreator + public WbDateVariable() { + + } + + public WbDateVariable(String columnName) { + setColumnName(columnName); + } @Override public TimeValue fromCell(Cell cell, ExpressionContext ctxt) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java index 669373813..df200938b 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java @@ -52,4 +52,9 @@ public class WbItemConstant implements WbExpression { WbItemConstant otherConstant = (WbItemConstant)other; return (qid.equals(otherConstant.getQid()) && label.equals(otherConstant.getLabel())); } + + @Override + public int hashCode() { + return qid.hashCode() + label.hashCode(); + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java index 7b9d2419d..b38afe56d 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java @@ -8,16 +8,20 @@ import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.Statement; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; /** * The representation of an item document, which can contain * variables both for its own id and in its contents. * - * @author antonin + * @author Antonin Delpeuch * */ -public class WbItemDocumentExpr extends JacksonJsonizable { +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use=JsonTypeInfo.Id.NONE) +public class WbItemDocumentExpr extends JacksonJsonizable implements WbExpression { private WbExpression subject; private List nameDescs; @@ -33,6 +37,7 @@ public class WbItemDocumentExpr extends JacksonJsonizable { this.statementGroups = statementGroupExprs; } + @Override public ItemUpdate evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException { ItemIdValue subjectId = getSubject().evaluate(ctxt); ItemUpdate update = new ItemUpdate(subjectId); @@ -65,4 +70,20 @@ public class WbItemDocumentExpr extends JacksonJsonizable { public List getStatementGroups() { return statementGroups; } + + @Override + public boolean equals(Object other) { + if(other == null || !WbItemDocumentExpr.class.isInstance(other)) { + return false; + } + WbItemDocumentExpr otherExpr = (WbItemDocumentExpr)other; + return subject.equals(otherExpr.getSubject()) && + nameDescs.equals(otherExpr.getNameDescs()) && + statementGroups.equals(otherExpr.getStatementGroups()); + } + + @Override + public int hashCode() { + return subject.hashCode() + nameDescs.hashCode() + statementGroups.hashCode(); + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageConstant.java index 4dfb0c171..c49c7d724 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageConstant.java @@ -76,4 +76,9 @@ public class WbLanguageConstant implements WbExpression { return _langId.equals(otherConstant.getLang()) && _langLabel.equals(otherConstant.getLabel()); } + @Override + public int hashCode() { + return _langId.hashCode(); + } + } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageVariable.java index b22c99e71..063939a18 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageVariable.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageVariable.java @@ -1,7 +1,6 @@ package org.openrefine.wikidata.schema; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; -import org.wikidata.wdtk.datamodel.interfaces.WikimediaLanguageCodes; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java index f22893a38..be99e1c25 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java @@ -5,10 +5,21 @@ import java.text.ParseException; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; +import com.fasterxml.jackson.annotation.JsonCreator; + import com.google.refine.model.Cell; public class WbLocationVariable extends WbVariableExpr { + + @JsonCreator + public WbLocationVariable() { + + } + + public WbLocationVariable(String columnName) { + setColumnName(columnName); + } @Override public GlobeCoordinatesValue fromCell(Cell cell, ExpressionContext ctxt) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbMonolingualExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbMonolingualExpr.java index 9c532113e..b3b352e99 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbMonolingualExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbMonolingualExpr.java @@ -52,4 +52,19 @@ public class WbMonolingualExpr implements WbExpression { public WbExpression getValueExpr() { return valueExpr; } + + @Override + public boolean equals(Object other) { + if(other == null || !WbMonolingualExpr.class.isInstance(other)) { + return false; + } + WbMonolingualExpr otherExpr = (WbMonolingualExpr)other; + return languageExpr.equals(otherExpr.getLanguageExpr()) && + valueExpr.equals(otherExpr.getValueExpr()); + } + + @Override + public int hashCode() { + return languageExpr.hashCode() + valueExpr.hashCode(); + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java index 310187940..d31c040a6 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java @@ -38,6 +38,14 @@ public class WbNameDescExpr { this.value = value; } + /** + * Evaluates the expression and adds the result to the item update. + * + * @param item + * the item update where the term should be stored + * @param ctxt + * the evaluation context for the expression + */ public void contributeTo(ItemUpdate item, ExpressionContext ctxt) { try { MonolingualTextValue val = getValue().evaluate(ctxt); @@ -66,4 +74,19 @@ public class WbNameDescExpr { public WbMonolingualExpr getValue() { return value; } + + @Override + public boolean equals(Object other) { + if(other == null || !WbNameDescExpr.class.isInstance(other)) { + return false; + } + WbNameDescExpr otherExpr = (WbNameDescExpr)other; + return type.equals(otherExpr.getType()) && + value.equals(otherExpr.getValue()); + } + + @Override + public int hashCode() { + return type.hashCode() + value.hashCode(); + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java index 30c17cd5d..8856a45cc 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java @@ -57,4 +57,8 @@ public class WbPropConstant implements WbExpression { return pid.equals(otherConstant.getPid()) && label.equals(otherConstant.getLabel()) && datatype.equals(otherConstant.getDatatype()); } + @Override + public int hashCode() { + return pid.hashCode() + label.hashCode(); + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java index 6415cd8df..6437faf1f 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java @@ -1,7 +1,6 @@ package org.openrefine.wikidata.schema; import java.math.BigDecimal; -import java.util.regex.Pattern; import org.apache.commons.lang.Validate; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java index f4e970ab6..8344a9305 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java @@ -12,9 +12,17 @@ import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +/** + * An expression for a reference (list of reference snaks). + * + * @author Antonin Delpeuch + * + */ @JsonIgnoreProperties(ignoreUnknown = true) -public class WbReferenceExpr { +@JsonTypeInfo(use=JsonTypeInfo.Id.NONE) +public class WbReferenceExpr implements WbExpression { private List snakExprs; @JsonCreator @@ -23,6 +31,7 @@ public class WbReferenceExpr { this.snakExprs = snakExprs; } + @Override public Reference evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException { List snakGroups = new ArrayList(); for (WbSnakExpr expr : getSnaks()) { @@ -46,4 +55,17 @@ public class WbReferenceExpr { return snakExprs; } + @Override + public boolean equals(Object other) { + if(other == null || !WbReferenceExpr.class.isInstance(other)) { + return false; + } + WbReferenceExpr otherExpr = (WbReferenceExpr)other; + return snakExprs.equals(otherExpr.getSnaks()); + } + + @Override + public int hashCode() { + return snakExprs.hashCode(); + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java index bc4199be1..efd20e8ff 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java @@ -9,9 +9,17 @@ import org.wikidata.wdtk.datamodel.interfaces.Value; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +/** + * An expression for a snak (pair of property and value). + * + * @author Antonin Delpeuch + * + */ @JsonIgnoreProperties(ignoreUnknown = true) -public class WbSnakExpr { +@JsonTypeInfo(use=JsonTypeInfo.Id.NONE) +public class WbSnakExpr implements WbExpression { private WbExpression prop; private WbExpression value; @@ -24,6 +32,7 @@ public class WbSnakExpr { this.value = valueExpr; } + @Override public Snak evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException { PropertyIdValue propertyId = getProp().evaluate(ctxt); Value evaluatedValue = value.evaluate(ctxt); @@ -39,4 +48,18 @@ public class WbSnakExpr { public WbExpression getValue() { return value; } + + @Override + public boolean equals(Object other) { + if (other == null || !WbSnakExpr.class.isInstance(other)) { + return false; + } + WbSnakExpr otherExpr = (WbSnakExpr) other; + return prop.equals(otherExpr.getProp()) && value.equals(otherExpr.getValue()); + } + + @Override + public int hashCode() { + return prop.hashCode() + value.hashCode(); + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java index 347ba70bb..c4aba1862 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java @@ -1,8 +1,10 @@ package org.openrefine.wikidata.schema; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import org.jsoup.helper.Validate; import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.wikidata.wdtk.datamodel.helpers.Datamodel; @@ -32,8 +34,15 @@ public class WbStatementExpr { @JsonProperty("value") WbExpression mainSnakValueExpr, @JsonProperty("qualifiers") List qualifierExprs, @JsonProperty("references") List referenceExprs) { + Validate.notNull(mainSnakValueExpr); this.mainSnakValueExpr = mainSnakValueExpr; + if (qualifierExprs == null) { + qualifierExprs = Collections.emptyList(); + } this.qualifierExprs = qualifierExprs; + if (referenceExprs == null) { + referenceExprs = Collections.emptyList(); + } this.referenceExprs = referenceExprs; } @@ -106,4 +115,20 @@ public class WbStatementExpr { public List getReferences() { return referenceExprs; } + + @Override + public boolean equals(Object other) { + if (other == null || !WbStatementExpr.class.isInstance(other)) { + return false; + } + WbStatementExpr otherExpr = (WbStatementExpr)other; + return mainSnakValueExpr.equals(otherExpr.getMainsnak()) && + qualifierExprs.equals(otherExpr.getQualifiers()) && + referenceExprs.equals(otherExpr.getReferences()); + } + + @Override + public int hashCode() { + return mainSnakValueExpr.hashCode() + qualifierExprs.hashCode() + referenceExprs.hashCode(); + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java index 9144a582e..98230233e 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java @@ -3,6 +3,7 @@ package org.openrefine.wikidata.schema; import java.util.ArrayList; import java.util.List; +import org.jsoup.helper.Validate; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; @@ -13,6 +14,7 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonIgnoreProperties(ignoreUnknown = true) public class WbStatementGroupExpr { @@ -24,7 +26,10 @@ public class WbStatementGroupExpr { public WbStatementGroupExpr( @JsonProperty("property") WbExpression propertyExpr, @JsonProperty("statements") List claimExprs) { + Validate.notNull(propertyExpr); this.propertyExpr = propertyExpr; + Validate.notNull(claimExprs); + Validate.isTrue(!claimExprs.isEmpty()); this.statementExprs = claimExprs; } @@ -54,4 +59,19 @@ public class WbStatementGroupExpr { public List getStatements() { return statementExprs; } + + @Override + public boolean equals(Object other) { + if (other == null || !WbStatementGroupExpr.class.isInstance(other)) { + return false; + } + WbStatementGroupExpr otherExpr = (WbStatementGroupExpr)other; + return propertyExpr.equals(otherExpr.getProperty()) && + statementExprs.equals(otherExpr.getStatements()); + } + + @Override + public int hashCode() { + return propertyExpr.hashCode() + statementExprs.hashCode(); + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java index b578f8dfa..83bead6be 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java @@ -23,40 +23,38 @@ import org.openrefine.wikidata.qa.QAWarningStore; import org.openrefine.wikidata.schema.ExpressionContext; import org.openrefine.wikidata.utils.JacksonJsonizable; +/** + * Main class representing a skeleton of Wikibase edits with + * OpenRefine columns as variables. + * + * @author Antonin Delpeuch + * + */ public class WikibaseSchema implements OverlayModel { final static Logger logger = LoggerFactory.getLogger("RdfSchema"); protected List itemDocumentExprs = new ArrayList(); - protected String baseUri = "http://www.wikidata.org/entity/"; - - @Override - public void onBeforeSave(Project project) { - } - - @Override - public void onAfterSave(Project project) { - } - - @Override - public void dispose(Project project) { - - } - - public void setBaseUri(String baseUri) { - this.baseUri = baseUri; - } + protected String baseIri = "http://www.wikidata.org/entity/"; + /** + * Constructor. + */ public WikibaseSchema() { } - - public String getBaseUri() { - return baseUri; + /** + * @return the site IRI of the Wikibase instance referenced by this schema + */ + public String getBaseIri() { + return baseIri; } + /** + * @return the list of document expressions for this schema + */ public List getItemDocumentExpressions() { return itemDocumentExprs; } @@ -67,7 +65,11 @@ public class WikibaseSchema implements OverlayModel { /** * Evaluates all item documents in a particular expression context. + * This specifies, among others, a row where the values of the variables + * will be read. + * * @param ctxt + * the context in which the schema should be evaluated. * @return */ public List evaluateItemDocuments(ExpressionContext ctxt) { @@ -92,9 +94,12 @@ public class WikibaseSchema implements OverlayModel { * for dates). Issues detected on candidate statements (such as constraint * violations) are not included at this stage. * - * @param project: the project on which the schema should be evaluated - * @param engine: the engine, which gives access to the current facets - * @param warningStore: a store in which issues will be emitted + * @param project + * the project on which the schema should be evaluated + * @param engine + * the engine, which gives access to the current facets + * @param warningStore + * a store in which issues will be emitted * @return item updates are stored in their * generating order (not merged yet). */ @@ -128,7 +133,7 @@ public class WikibaseSchema implements OverlayModel { @Override public boolean visit(Project project, int rowIndex, Row row) { ExpressionContext ctxt = new ExpressionContext( - baseUri, + baseIri, rowIndex, row, project.columnModel, @@ -170,4 +175,17 @@ public class WikibaseSchema implements OverlayModel { static public WikibaseSchema load(Project project, JSONObject obj) throws Exception { return reconstruct(obj); } + + @Override + public void onBeforeSave(Project project) { + } + + @Override + public void onAfterSave(Project project) { + } + + @Override + public void dispose(Project project) { + + } } \ No newline at end of file diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconEntityIdValue.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconEntityIdValue.java index a67f313c1..9b7320f68 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconEntityIdValue.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconEntityIdValue.java @@ -65,6 +65,28 @@ public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue { @Override public abstract String getEntityType(); + /** + * Returns the integer used internally in OpenRefine to identify the new + * item. + * + * @return + * the judgment history entry id of the reconciled cell + */ + public long getReconInternalId() { + return getRecon().judgmentHistoryEntry; + } + + + /** + * Returns the reconciliation object corresponding to this entity. + * + * @return + * the full reconciliation metadata of the corresponding cell + */ + protected Recon getRecon() { + return _recon; + } + /** * Returns the id of the reconciled item */ @@ -94,6 +116,7 @@ public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue { public T accept(ValueVisitor valueVisitor) { return valueVisitor.visit(this); } + /** * Equality check is important when we gather @@ -126,17 +149,6 @@ public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue { return getIri().equals(otherNew.getIri()); } - /** - * Returns the integer used internally in OpenRefine to identify the new - * item. - * - * @return - * the judgment history entry id of the reconciled cell - */ - public long getReconInternalId() { - return getRecon().judgmentHistoryEntry; - } - @Override public int hashCode() { if (isMatched()) { @@ -145,14 +157,13 @@ public abstract class ReconEntityIdValue implements PrefetchedEntityIdValue { return (int) getReconInternalId(); } } - - /** - * Returns the reconciliation object corresponding to this entity. - * - * @return - * the full reconciliation metadata of the corresponding cell - */ - protected Recon getRecon() { - return _recon; + + @Override + public String toString() { + if(isNew()) { + return "new item (reconciled from " + getReconInternalId() +")"; + } else { + return getIri() + " (reconciled from " + getReconInternalId()+")"; + } } } diff --git a/extensions/wikidata/tests/data/schema/history_of_medicine_normalized.json b/extensions/wikidata/tests/data/schema/history_of_medicine_normalized.json new file mode 100644 index 000000000..8e729cd11 --- /dev/null +++ b/extensions/wikidata/tests/data/schema/history_of_medicine_normalized.json @@ -0,0 +1 @@ +{"itemDocuments":[{"subject":{"type":"wbitemvariable","columnName":"name"},"statementGroups":[{"property":{"datatype":"wikibase-item","pid":"P39","label":"position held","type":"wbpropconstant"},"statements":[{"references":[{"snaks":[{"prop":{"datatype":"wikibase-item","pid":"P248","label":"stated in","type":"wbpropconstant"},"value":{"label":"The History of The Royal Society of Medicine","type":"wbitemconstant","qid":"Q42036099"}},{"prop":{"datatype":"string","pid":"P304","label":"page","type":"wbpropconstant"},"value":{"type":"wbstringconstant","value":"330-333"}}]}],"qualifiers":[{"prop":{"datatype":"wikibase-item","pid":"P642","label":"of","type":"wbpropconstant"},"value":{"label":"History of Medicine Society","type":"wbitemconstant","qid":"Q37461404"}},{"prop":{"datatype":"time","pid":"P580","label":"start date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"start"}},{"prop":{"datatype":"time","pid":"P582","label":"end date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"end"}}],"value":{"label":"president","type":"wbitemconstant","qid":"Q30461"}}]}],"nameDescs":[]},{"subject":{"label":"History of Medicine Society","type":"wbitemconstant","qid":"Q37461404"},"statementGroups":[{"property":{"datatype":"wikibase-item","pid":"P488","label":"chairperson","type":"wbpropconstant"},"statements":[{"references":[{"snaks":[{"prop":{"datatype":"wikibase-item","pid":"P248","label":"stated in","type":"wbpropconstant"},"value":{"label":"The History of The Royal Society of Medicine","type":"wbitemconstant","qid":"Q42036099"}},{"prop":{"datatype":"string","pid":"P304","label":"page","type":"wbpropconstant"},"value":{"type":"wbstringconstant","value":"330-333"}}]}],"qualifiers":[{"prop":{"datatype":"time","pid":"P580","label":"start date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"start"}},{"prop":{"datatype":"time","pid":"P582","label":"end date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"end"}}],"value":{"type":"wbitemvariable","columnName":"name"}}]}],"nameDescs":[]}]} diff --git a/extensions/wikidata/tests/data/schema/inception.json b/extensions/wikidata/tests/data/schema/inception.json new file mode 100644 index 000000000..a7d90cdc8 --- /dev/null +++ b/extensions/wikidata/tests/data/schema/inception.json @@ -0,0 +1 @@ +{"itemDocuments":[{"subject":{"type":"wbitemvariable","columnName":"subject"},"statementGroups":[{"property":{"type":"wbpropconstant","pid":"P571","label":"inception","datatype":"time"},"statements":[{"value":{"type":"wbdatevariable","columnName":"inception"},"qualifiers":[],"references":[{"snaks":[{"prop":{"type":"wbpropconstant","pid":"P854","label":"reference URL","datatype":"url"},"value":{"type":"wbstringvariable","columnName":"reference"}},{"prop":{"type":"wbpropconstant","pid":"P813","label":"retrieved","datatype":"time"},"value":{"type":"wbdateconstant","value":"2018-02-28"}}]}]}]}],"nameDescs":[]}],"wikibasePrefix":"http://www.wikidata.org/entity/"} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSValuePrinterTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSValuePrinterTest.java index 89f23e327..72b79ac1f 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSValuePrinterTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSValuePrinterTest.java @@ -13,9 +13,6 @@ import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; import org.wikidata.wdtk.datamodel.interfaces.TimeValue; import org.wikidata.wdtk.datamodel.interfaces.Value; -import com.google.refine.model.Recon; -import com.google.refine.model.ReconCandidate; - public class QSValuePrinterTest { private QSValuePrinter printer; diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ItemUpdateTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ItemUpdateTest.java new file mode 100644 index 000000000..265074905 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ItemUpdateTest.java @@ -0,0 +1,139 @@ +package org.openrefine.wikidata.schema; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.openrefine.wikidata.testing.TestingDataGenerator; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.Claim; +import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; +import org.wikidata.wdtk.datamodel.interfaces.StatementRank; + +public class ItemUpdateTest { + + private ItemIdValue existingSubject = Datamodel.makeWikidataItemIdValue("Q34"); + private ItemIdValue newSubject = TestingDataGenerator.makeNewItemIdValue(1234L, "new item"); + private ItemIdValue sameNewSubject = TestingDataGenerator.makeNewItemIdValue(1234L, "other new item"); + private ItemIdValue matchedSubject = TestingDataGenerator.makeMatchedItemIdValue("Q78", "well known item"); + private ItemUpdate update = new ItemUpdate(existingSubject); + + private PropertyIdValue pid1 = Datamodel.makeWikidataPropertyIdValue("P348"); + private PropertyIdValue pid2 = Datamodel.makeWikidataPropertyIdValue("P52"); + private Claim claim1 = Datamodel.makeClaim(existingSubject, + Datamodel.makeNoValueSnak(pid1), Collections.emptyList()); + private Claim claim2 = Datamodel.makeClaim(existingSubject, + Datamodel.makeValueSnak(pid2, newSubject), Collections.emptyList()); + private Statement statement1 = Datamodel.makeStatement(claim1, + Collections.emptyList(), StatementRank.NORMAL, ""); + private Statement statement2 = Datamodel.makeStatement(claim2, + Collections.emptyList(), StatementRank.NORMAL, ""); + private MonolingualTextValue label = Datamodel.makeMonolingualTextValue("this is a label", "en"); + + private Set statementGroups; + + public ItemUpdateTest() { + statementGroups = new HashSet<>(); + statementGroups.add(Datamodel.makeStatementGroup(Collections.singletonList(statement1))); + statementGroups.add(Datamodel.makeStatementGroup(Collections.singletonList(statement2))); + } + + @Test(expectedExceptions=IllegalArgumentException.class) + public void testCreateWithoutSubject() { + new ItemUpdate(null); + } + + @Test + public void testIsNull() { + assertTrue(update.isNull()); + } + + @Test + public void testIsNew() { + ItemUpdate newUpdate = new ItemUpdate(newSubject); + assertTrue(newUpdate.isNew()); + assertFalse(update.isNew()); + } + + @Test + public void testAddStatements() { + ItemUpdate update = new ItemUpdate(existingSubject); + update.addStatement(statement1); + update.addStatement(statement2); + assertEquals(Arrays.asList(statement1, statement2).stream().collect(Collectors.toSet()), + update.getAddedStatements()); + assertEquals(statementGroups, update.getAddedStatementGroups().stream().collect(Collectors.toSet())); + } + + @Test + public void testDeleteStatements() { + ItemUpdate update = new ItemUpdate(existingSubject); + update.deleteStatement(statement1); + update.deleteStatement(statement2); + assertEquals(Arrays.asList(statement1, statement2).stream().collect(Collectors.toSet()), + update.getDeletedStatements()); + } + + @Test + public void testMerge() { + ItemUpdate updateA = new ItemUpdate(existingSubject); + updateA.addStatement(statement1); + ItemUpdate updateB = new ItemUpdate(existingSubject); + updateB.addStatement(statement2); + assertNotEquals(updateA, updateB); + updateA.merge(updateB); + assertEquals(statementGroups, + updateA.getAddedStatementGroups().stream().collect(Collectors.toSet())); + } + + @Test + public void testGroupBySubject() { + ItemUpdate updateA = new ItemUpdate(newSubject); + updateA.addStatement(statement1); + ItemUpdate updateB = new ItemUpdate(sameNewSubject); + updateB.addStatement(statement2); + ItemUpdate updateC = new ItemUpdate(existingSubject); + updateC.addLabel(label); + ItemUpdate updateD = new ItemUpdate(matchedSubject); + Map grouped = ItemUpdate.groupBySubject( + Arrays.asList(updateA, updateB, updateC, updateD)); + ItemUpdate mergedUpdate = new ItemUpdate(newSubject); + mergedUpdate.addStatement(statement1); + mergedUpdate.addStatement(statement2); + Map expected = new HashMap<>(); + expected.put(newSubject, mergedUpdate); + expected.put(existingSubject, updateC); + assertEquals(expected, grouped); + } + + @Test + public void testNormalizeTerms() { + MonolingualTextValue aliasEn = Datamodel.makeMonolingualTextValue("alias", "en"); + MonolingualTextValue aliasFr = Datamodel.makeMonolingualTextValue("coucou", "fr"); + ItemUpdate updateA = new ItemUpdate(newSubject); + updateA.addLabel(label); + updateA.addAlias(aliasEn); + updateA.addAlias(aliasFr); + updateA.normalizeLabelsAndAliases(); + ItemUpdate expectedUpdate = new ItemUpdate(newSubject); + expectedUpdate.addLabel(label); + expectedUpdate.addAlias(aliasEn); + expectedUpdate.addLabel(aliasFr); + assertEquals(expectedUpdate, updateA); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/SchemaSerializationTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/SchemaSerializationTest.java deleted file mode 100644 index 6419b2e61..000000000 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/SchemaSerializationTest.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.openrefine.wikidata.schema; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; - -import org.json.JSONException; -import org.json.JSONObject; -import org.testng.annotations.Test; - -import com.google.refine.util.ParsingUtilities; - -public class SchemaSerializationTest { - - static JSONObject jsonFromFile(String filename) throws IOException, JSONException { - byte[] contents = Files.readAllBytes(Paths.get(filename)); - String decoded = new String(contents, "utf-8"); - return ParsingUtilities.evaluateJsonStringToObject(decoded); - } - - @Test - public void testDeserializeHistoryOfMedicine() throws JSONException, IOException { - JSONObject serialized = jsonFromFile("data/schema/history_of_medicine.json"); - WikibaseSchema schema = WikibaseSchema.reconstruct(serialized); - } - - @Test - public void testDeserializeROARMAP() throws JSONException, IOException { - JSONObject serialized = jsonFromFile("data/schema/roarmap.json"); - WikibaseSchema schema = WikibaseSchema.reconstruct(serialized); - } -} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbExpressionTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbExpressionTest.java index 404a755d1..5711b7c29 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbExpressionTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbExpressionTest.java @@ -1,8 +1,6 @@ package org.openrefine.wikidata.schema; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.io.Serializable; import org.openrefine.wikidata.qa.QAWarningStore; @@ -27,8 +25,8 @@ public class WbExpressionTest extends RefineTest { @BeforeMethod public void createProject() throws IOException, ModelException { - project = createCSVProject("Wikidata variable test project", "column A,column B,column C,column D\n"+ - "value A,value B,value C,value D"); + project = createCSVProject("Wikidata variable test project", "column A,column B,column C,column D,column E\n"+ + "value A,value B,value C,value D,value E"); warningStore = new QAWarningStore(); row = project.rows.get(0); ctxt = new ExpressionContext("http://www.wikidata.org/entity/", 0, diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemDocumentExprTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemDocumentExprTest.java new file mode 100644 index 000000000..93d9d3f8f --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemDocumentExprTest.java @@ -0,0 +1,74 @@ +package org.openrefine.wikidata.schema; + +import java.util.Collections; + +import org.openrefine.wikidata.testing.JacksonSerializationTest; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; +import org.wikidata.wdtk.datamodel.interfaces.Statement; + +public class WbItemDocumentExprTest extends WbExpressionTest { + + public WbItemDocumentExpr expr; + ItemIdValue subject = Datamodel.makeWikidataItemIdValue("Q23"); + MonolingualTextValue alias = Datamodel.makeMonolingualTextValue("my alias", "en"); + Statement fullStatement; + + public String jsonRepresentation; + + public WbItemDocumentExprTest() { + WbStatementGroupExprTest sgt = new WbStatementGroupExprTest(); + WbNameDescExpr nde = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS, + new WbMonolingualExpr(new WbLanguageConstant("en", "English"), + new WbStringVariable("column D"))); + WbItemVariable subjectExpr = new WbItemVariable("column E"); + expr = new WbItemDocumentExpr(subjectExpr, + Collections.singletonList(nde), + Collections.singletonList(sgt.expr)); + fullStatement = sgt.statementGroup.getStatements().get(0); + + jsonRepresentation = "{\"subject\":{\"type\":\"wbitemvariable\",\"columnName\":\"column E\"},"+ + "\"nameDescs\":[{\"name_type\":\"ALIAS\",\"value\":{\"type\":\"wbmonolingualexpr\",\"language\":"+ + "{\"type\":\"wblanguageconstant\",\"id\":\"en\",\"label\":\"English\"},"+ + "\"value\":{\"type\":\"wbstringvariable\",\"columnName\":\"column D\"}}}"+ + "],\"statementGroups\":["+sgt.jsonRepresentation+"]}"; + } + + @Test + public void testEvaluate() { + setRow(recon("Q3434"), "2010-07-23", "3.898,4.389", "my alias", recon("Q23")); + ItemUpdate result = new ItemUpdate(subject); + result.addAlias(alias); + result.addStatement(fullStatement); + evaluatesTo(result, expr); + } + + @Test + public void testSubjectSkipped() { + setRow(recon("Q3434"), "2010-07-23", "3.898,4.389", "my alias", "not reconciled"); + isSkipped(expr); + } + + @Test + public void testStatementSkipped() { + setRow(recon("Q3434"), "2010-07-23", "3.898,invalid4.389", "my alias", recon("Q23")); + ItemUpdate result = new ItemUpdate(subject); + result.addAlias(alias); + evaluatesTo(result, expr); + } + + @Test + public void testAliasSkipped() { + setRow(recon("Q3434"), "2010-07-23", "3.898,4.389", "", recon("Q23")); + ItemUpdate result = new ItemUpdate(subject); + result.addStatement(fullStatement); + evaluatesTo(result, expr); + } + + @Test + public void testSerialize() { + JacksonSerializationTest.canonicalSerialization(WbItemDocumentExpr.class, expr, jsonRepresentation); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLocationConstantTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLocationConstantTest.java index 8007d0bfc..dd5159770 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLocationConstantTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLocationConstantTest.java @@ -12,7 +12,7 @@ import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; public class WbLocationConstantTest extends WbExpressionTest { private GlobeCoordinatesValue loc = Datamodel.makeGlobeCoordinatesValue(1.2345, 6.7890, - GlobeCoordinatesValue.PREC_TEN_MICRO_DEGREE, GlobeCoordinatesValue.GLOBE_EARTH); + WbLocationConstant.defaultPrecision, GlobeCoordinatesValue.GLOBE_EARTH); private GlobeCoordinatesValue locWithPrecision = Datamodel.makeGlobeCoordinatesValue(1.2345, 6.7890, 0.1, GlobeCoordinatesValue.GLOBE_EARTH); private String input = "1.2345,6.7890"; diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescExprTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescExprTest.java new file mode 100644 index 000000000..f263fc84c --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescExprTest.java @@ -0,0 +1,75 @@ +package org.openrefine.wikidata.schema; + +import static org.junit.Assert.assertEquals; + +import java.util.Collections; + +import org.openrefine.wikidata.testing.JacksonSerializationTest; +import org.openrefine.wikidata.testing.TestingDataGenerator; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; + +public class WbNameDescExprTest extends WbExpressionTest { + private ItemIdValue subject = Datamodel.makeWikidataItemIdValue("Q56"); + public WbNameDescExpr expr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS, + new WbMonolingualExpr(new WbLanguageConstant("en", "English"), + new WbStringVariable("column A"))); + + public String jsonRepresentation = "{\"name_type\":\"ALIAS\",\"value\":{\"type\":\"wbmonolingualexpr\",\"language\":"+ + "{\"type\":\"wblanguageconstant\",\"id\":\"en\",\"label\":\"English\"},\"value\":"+ + "{\"type\":\"wbstringvariable\",\"columnName\":\"column A\"}}}"; + + @Test + public void testContributeToLabel() { + WbNameDescExpr labelExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.LABEL, + TestingDataGenerator.getTestMonolingualExpr("fr", "français", "le croissant magnifique")); + ItemUpdate update = new ItemUpdate(subject); + labelExpr.contributeTo(update, ctxt); + assertEquals(Collections.singleton(Datamodel.makeMonolingualTextValue("le croissant magnifique", "fr")), + update.getLabels()); + } + + @Test + public void testContributeToDescription() { + WbNameDescExpr descriptionExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.DESCRIPTION, + TestingDataGenerator.getTestMonolingualExpr("de", "Deutsch", "wunderschön")); + ItemUpdate update = new ItemUpdate(subject); + descriptionExpr.contributeTo(update, ctxt); + assertEquals(Collections.singleton(Datamodel.makeMonolingualTextValue("wunderschön", "de")), + update.getDescriptions()); + } + + @Test + public void testContributeToAlias() { + WbNameDescExpr aliasExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS, + TestingDataGenerator.getTestMonolingualExpr("en", "English", "snack")); + ItemUpdate update = new ItemUpdate(subject); + aliasExpr.contributeTo(update, ctxt); + assertEquals(Collections.singleton(Datamodel.makeMonolingualTextValue("snack", "en")), + update.getAliases()); + } + + @Test + public void testSkipped() { + ItemUpdate update = new ItemUpdate(subject); + setRow(""); + expr.contributeTo(update, ctxt); + assertEquals(new ItemUpdate(subject), update); + } + + @Test + public void testGetters() { + WbMonolingualExpr monolingualExpr = TestingDataGenerator.getTestMonolingualExpr("en", "English", "not sure what"); + WbNameDescExpr aliasExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS, + monolingualExpr); + assertEquals(WbNameDescExpr.NameDescrType.ALIAS, aliasExpr.getType()); + assertEquals(monolingualExpr, aliasExpr.getValue()); + } + + @Test + public void testSerialization() { + JacksonSerializationTest.canonicalSerialization(WbNameDescExpr.class, expr, jsonRepresentation); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescrExprTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescrExprTest.java deleted file mode 100644 index 2f4974582..000000000 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescrExprTest.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.openrefine.wikidata.schema; - -import static org.junit.Assert.assertEquals; - -import java.util.Collections; - -import org.openrefine.wikidata.testing.TestingDataGenerator; -import org.testng.annotations.Test; -import org.wikidata.wdtk.datamodel.helpers.Datamodel; -import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; - -public class WbNameDescrExprTest extends WbExpressionTest { - - @Test - public void testContributeToLabel() { - WbNameDescExpr labelExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.LABEL, - TestingDataGenerator.getTestMonolingualExpr("fr", "français", "le croissant magnifique")); - ItemUpdate update = new ItemUpdate(Datamodel.makeWikidataItemIdValue("Q56")); - labelExpr.contributeTo(update, ctxt); - assertEquals(Collections.singletonList(Datamodel.makeMonolingualTextValue("le croissant magnifique", "fr")), update.getLabels()); - } - - @Test - public void testContributeToDescription() { - WbNameDescExpr descriptionExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.DESCRIPTION, - TestingDataGenerator.getTestMonolingualExpr("de", "Deutsch", "wunderschön")); - ItemUpdate update = new ItemUpdate(Datamodel.makeWikidataItemIdValue("Q56")); - descriptionExpr.contributeTo(update, ctxt); - assertEquals(Collections.singletonList(Datamodel.makeMonolingualTextValue("wunderschön", "de")), update.getDescriptions()); - } - - @Test - public void testContributeToAlias() { - WbNameDescExpr aliasExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS, - TestingDataGenerator.getTestMonolingualExpr("en", "English", "snack")); - ItemUpdate update = new ItemUpdate(Datamodel.makeWikidataItemIdValue("Q56")); - aliasExpr.contributeTo(update, ctxt); - assertEquals(Collections.singletonList(Datamodel.makeMonolingualTextValue("snack", "en")), update.getAliases()); - } - - @Test - public void testGetters() { - WbMonolingualExpr monolingualExpr = TestingDataGenerator.getTestMonolingualExpr("en", "English", "not sure what"); - WbNameDescExpr aliasExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS, - monolingualExpr); - assertEquals(WbNameDescExpr.NameDescrType.ALIAS, aliasExpr.getType()); - assertEquals(monolingualExpr, aliasExpr.getValue()); - } -} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbReferenceExprTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbReferenceExprTest.java new file mode 100644 index 000000000..e7a5ee6a8 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbReferenceExprTest.java @@ -0,0 +1,62 @@ +package org.openrefine.wikidata.schema; + +import java.util.Arrays; +import java.util.Collections; + +import org.openrefine.wikidata.testing.JacksonSerializationTest; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.Reference; +import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.TimeValue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class WbReferenceExprTest extends WbExpressionTest { + + private WbReferenceExpr expr = new WbReferenceExpr( + Arrays.asList( + new WbSnakExpr(new WbPropConstant("P87","retrieved","time"), + new WbDateVariable("column A")), + new WbSnakExpr(new WbPropConstant("P347","reference URL","url"), + new WbStringVariable("column B"))) + ); + + private Snak snak1 = Datamodel.makeValueSnak(Datamodel.makeWikidataPropertyIdValue("P87"), + Datamodel.makeTimeValue(2018, (byte)3, (byte)28, TimeValue.CM_GREGORIAN_PRO)); + private Snak snak2 = Datamodel.makeValueSnak(Datamodel.makeWikidataPropertyIdValue("P347"), + Datamodel.makeStringValue("http://gnu.org/")); + + private String jsonRepresentation = "{\"snaks\":[{\"prop\":{\"type\":\"wbpropconstant\",\"pid\":\"P87\"," + +"\"label\":\"retrieved\",\"datatype\":\"time\"},\"value\":{\"type\":\"wbdatevariable\"," + +"\"columnName\":\"column A\"}},{\"prop\":{\"type\":\"wbpropconstant\",\"pid\":\"P347\"," + +"\"label\":\"reference URL\",\"datatype\":\"url\"},\"value\":{\"type\":\"wbstringvariable\"," + +"\"columnName\":\"column B\"}}]}"; + + @Test + public void testEvaluate() { + setRow("2018-03-28", "http://gnu.org/"); + evaluatesTo(Datamodel.makeReference(Arrays.asList( + Datamodel.makeSnakGroup(Collections.singletonList(snak1)), + Datamodel.makeSnakGroup(Collections.singletonList(snak2)))), expr); + } + + @Test + public void testEvaluateWithOneSkip() { + setRow("invalid date", "http://gnu.org/"); + evaluatesTo(Datamodel.makeReference(Arrays.asList( + Datamodel.makeSnakGroup(Collections.singletonList(snak2)))), expr); + } + + @Test + public void testNoValidSnak() { + setRow("invalid date", ""); + isSkipped(expr); + } + + @Test + public void testSerialize() throws JsonProcessingException { + JacksonSerializationTest.canonicalSerialization(WbReferenceExpr.class, expr, jsonRepresentation); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbSnakExprTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbSnakExprTest.java new file mode 100644 index 000000000..bc3a07d41 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbSnakExprTest.java @@ -0,0 +1,33 @@ +package org.openrefine.wikidata.schema; + +import org.openrefine.wikidata.testing.JacksonSerializationTest; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class WbSnakExprTest extends WbExpressionTest { + private PropertyIdValue propStringId = Datamodel.makeWikidataPropertyIdValue("P89"); + private WbPropConstant propStringExpr = new WbPropConstant("P89", "prop label", "string"); + private WbSnakExpr expr = new WbSnakExpr(propStringExpr, new WbStringVariable("column A")); + + public String jsonRepresentation = "{\"prop\":{\"type\":\"wbpropconstant\",\"pid\":\"P89\"," + +"\"label\":\"prop label\",\"datatype\":\"string\"},\"value\":" + +"{\"type\":\"wbstringvariable\",\"columnName\":\"column A\"}}"; + + @Test + public void testEvaluate() { + setRow("cinema"); + evaluatesTo(Datamodel.makeValueSnak(propStringId, Datamodel.makeStringValue("cinema")), expr); + } + + @Test + public void testSerialize() throws JsonProcessingException { + JacksonSerializationTest.canonicalSerialization(WbSnakExpr.class, expr, jsonRepresentation); + } + + // TODO check that the datatype of the property matches that of the datavalue (important when we introduce property variables) +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementExprTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementExprTest.java index 96e1a4047..a7ecb4a9c 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementExprTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementExprTest.java @@ -1,13 +1,131 @@ package org.openrefine.wikidata.schema; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; +import org.openrefine.wikidata.testing.JacksonSerializationTest; import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.Claim; +import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Reference; +import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.StatementRank; +import org.wikidata.wdtk.datamodel.interfaces.TimeValue; -public class WbStatementExprTest { +import com.fasterxml.jackson.core.JsonGenerationException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class WbStatementExprTest extends WbExpressionTest { + + private WbReferenceExpr refExpr = new WbReferenceExpr( + Arrays.asList( + new WbSnakExpr(new WbPropConstant("P43","imported from","wikibase-item"), + new WbItemVariable("column A"))) + ); + private WbSnakExpr qualifierExpr = new WbSnakExpr( + new WbPropConstant("P897", "point in time", "time"), + new WbDateVariable("column B") + ); + private WbLocationVariable mainValueExpr = new WbLocationVariable("column C"); + public WbStatementExpr statementExpr = new WbStatementExpr(mainValueExpr, + Collections.singletonList(qualifierExpr), + Collections.singletonList(refExpr)); + + public ItemIdValue subject = Datamodel.makeWikidataItemIdValue("Q23"); + private PropertyIdValue property = Datamodel.makeWikidataPropertyIdValue("P908"); + private Reference reference = Datamodel.makeReference(Collections.singletonList( + Datamodel.makeSnakGroup(Collections.singletonList( + Datamodel.makeValueSnak(Datamodel.makeWikidataPropertyIdValue("P43"), + Datamodel.makeWikidataItemIdValue("Q3434")))) + )); + private Snak qualifier = Datamodel.makeValueSnak( + Datamodel.makeWikidataPropertyIdValue("P897"), + Datamodel.makeTimeValue(2010, (byte)7, (byte)23, TimeValue.CM_GREGORIAN_PRO) + ); + private Snak mainsnak = Datamodel.makeValueSnak( + property, Datamodel.makeGlobeCoordinatesValue(3.898, 4.389, + WbLocationConstant.defaultPrecision, GlobeCoordinatesValue.GLOBE_EARTH)); + private Claim fullClaim = Datamodel.makeClaim(subject, mainsnak, Collections.singletonList( + Datamodel.makeSnakGroup(Collections.singletonList(qualifier)))); + public Statement fullStatement = Datamodel.makeStatement(fullClaim, Collections.singletonList(reference), + StatementRank.NORMAL, ""); + + class Wrapper implements WbExpression { + public WbStatementExpr expr; + + public Wrapper(WbStatementExpr e) { + expr = e; + } + + @Override + public Statement evaluate(ExpressionContext ctxt) + throws SkipSchemaExpressionException { + return expr.evaluate(ctxt, subject, property); + } + } + + public String jsonRepresentation = "{\"value\":{\"type\":\"wblocationvariable\",\"columnName\":\"column C\"}," + +"\"qualifiers\":[{\"prop\":{\"type\":\"wbpropconstant\",\"pid\":\"P897\",\"label\":\"point in time\"," + +"\"datatype\":\"time\"},\"value\":{\"type\":\"wbdatevariable\",\"columnName\":\"column B\"}}]," + +"\"references\":[{\"snaks\":[{\"prop\":{\"type\":\"wbpropconstant\",\"pid\":\"P43\"," + +"\"label\":\"imported from\",\"datatype\":\"wikibase-item\"},\"value\":" + +"{\"type\":\"wbitemvariable\",\"columnName\":\"column A\"}}]}]}"; + @Test public void testCreation() { WbItemConstant q5 = new WbItemConstant("Q5", "human"); - new WbStatementExpr(q5, Collections.emptyList(), Collections.emptyList()); + WbStatementExpr empty = new WbStatementExpr(q5, Collections.emptyList(), Collections.emptyList()); + WbStatementExpr withNulls = new WbStatementExpr(q5, null, null); + assertEquals(empty, withNulls); + } + + @Test + public void testEvaluate() { + setRow(recon("Q3434"), "2010-07-23", "3.898,4.389"); + evaluatesTo(fullStatement, new Wrapper(statementExpr)); + } + + @Test + public void testEvaluateWithoutReference() { + setRow("not reconciled", "2010-07-23", "3.898,4.389"); + evaluatesTo(Datamodel.makeStatement(fullClaim, Collections.emptyList(), + StatementRank.NORMAL, ""), + new Wrapper(statementExpr)); + } + + @Test + public void testEvaluateWithoutQualifier() { + setRow(recon("Q3434"), "2010-invalid", "3.898,4.389"); + evaluatesTo(Datamodel.makeStatement(Datamodel.makeClaim(subject, mainsnak, + Collections.emptyList()), Collections.singletonList(reference), StatementRank.NORMAL, ""), + new Wrapper(statementExpr)); + } + + @Test + public void testEvaluateWithoutQualifierAndReference() { + setRow("invalid", "2010-invalid", "3.898,4.389"); + evaluatesTo(Datamodel.makeStatement(Datamodel.makeClaim(subject, mainsnak, + Collections.emptyList()), Collections.emptyList(), StatementRank.NORMAL, ""), + new Wrapper(statementExpr)); + } + + @Test + public void testSkip() throws JsonGenerationException, JsonMappingException, IOException { + setRow(recon("Q3434"), "2010-07-23", "3.898,invalid"); + isSkipped(new Wrapper(statementExpr)); + } + + @Test + public void testSerialize() { + JacksonSerializationTest.canonicalSerialization(WbStatementExpr.class, statementExpr, jsonRepresentation); } } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementGroupExprTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementGroupExprTest.java new file mode 100644 index 000000000..e89741e02 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementGroupExprTest.java @@ -0,0 +1,72 @@ +package org.openrefine.wikidata.schema; + +import java.util.Collections; + +import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; +import org.openrefine.wikidata.testing.JacksonSerializationTest; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; + +import com.fasterxml.jackson.core.JsonProcessingException; + +public class WbStatementGroupExprTest extends WbExpressionTest { + + private WbPropConstant propertyExpr = new WbPropConstant("P908", "myprop", "time"); + public WbStatementGroupExpr expr; + + private ItemIdValue subject; + public StatementGroup statementGroup; + + public String jsonRepresentation; + + + class Wrapper implements WbExpression { + public WbStatementGroupExpr expr; + + public Wrapper(WbStatementGroupExpr e) { + expr = e; + } + + @Override + public StatementGroup evaluate(ExpressionContext ctxt) + throws SkipSchemaExpressionException { + return expr.evaluate(ctxt, subject); + } + } + + + public WbStatementGroupExprTest() { + WbStatementExprTest statementTest = new WbStatementExprTest(); + expr = new WbStatementGroupExpr( + propertyExpr, + Collections.singletonList(statementTest.statementExpr)); + subject = statementTest.subject; + statementGroup = Datamodel.makeStatementGroup(Collections.singletonList(statementTest.fullStatement)); + jsonRepresentation = "{\"property\":{\"type\":\"wbpropconstant\",\"pid\":\"P908\",\"label\":\"myprop\",\"datatype\":\"time\"}," + +"\"statements\":["+statementTest.jsonRepresentation+"]}"; + } + + @Test(expectedExceptions=IllegalArgumentException.class) + public void testCreate() { + new WbStatementGroupExpr(propertyExpr, Collections.emptyList()); + } + + @Test + public void testEvaluate() { + setRow(recon("Q3434"), "2010-07-23", "3.898,4.389"); + evaluatesTo(statementGroup, new Wrapper(expr)); + } + + @Test + public void testSkip() { + setRow(recon("Q3434"), "2010-07-23", "3.898,invalid"); + isSkipped(new Wrapper(expr)); + } + + @Test + public void testSerialize() throws JsonProcessingException { + JacksonSerializationTest.canonicalSerialization(WbStatementGroupExpr.class, expr, jsonRepresentation); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbVariableTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbVariableTest.java index 53f09047c..055906bb7 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbVariableTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbVariableTest.java @@ -2,16 +2,10 @@ package org.openrefine.wikidata.schema; import java.io.IOException; -import org.openrefine.wikidata.qa.QAWarningStore; -import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; -import org.testng.Assert; import org.testng.annotations.BeforeMethod; import com.google.refine.model.Cell; import com.google.refine.model.ModelException; -import com.google.refine.model.Project; -import com.google.refine.model.Row; -import com.google.refine.tests.RefineTest; public abstract class WbVariableTest extends WbExpressionTest { diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java new file mode 100644 index 000000000..71423e2e4 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java @@ -0,0 +1,112 @@ +package org.openrefine.wikidata.schema; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.openrefine.wikidata.testing.TestingDataGenerator; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.Claim; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; +import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.StatementRank; +import org.wikidata.wdtk.datamodel.interfaces.StringValue; +import org.wikidata.wdtk.datamodel.interfaces.TimeValue; + +import com.google.refine.browsing.Engine; +import com.google.refine.model.Project; +import com.google.refine.tests.RefineTest; +import com.google.refine.util.ParsingUtilities; + +public class WikibaseSchemaTest extends RefineTest { + + private ItemIdValue qid1 = Datamodel.makeWikidataItemIdValue("Q1377"); + private ItemIdValue qid2 = Datamodel.makeWikidataItemIdValue("Q865528"); + private TimeValue date1 = Datamodel.makeTimeValue(1919, (byte)1, (byte)1, + (byte)0, (byte)0, (byte)0, (byte)9, (byte)0, (byte)1, (byte)0, TimeValue.CM_GREGORIAN_PRO); + private TimeValue date2 = Datamodel.makeTimeValue(1965, (byte)1, (byte)1, + (byte)0, (byte)0, (byte)0, (byte)9, (byte)0, (byte)1, (byte)0, TimeValue.CM_GREGORIAN_PRO); + private StringValue url = Datamodel.makeStringValue("http://www.ljubljana-slovenia.com/university-ljubljana"); + private PropertyIdValue inceptionPid = Datamodel.makeWikidataPropertyIdValue("P571"); + private PropertyIdValue refPid = Datamodel.makeWikidataPropertyIdValue("P854"); + private PropertyIdValue retrievedPid = Datamodel.makeWikidataPropertyIdValue("P813"); + private Snak refSnak = Datamodel.makeValueSnak(refPid, url); + private Snak retrievedSnak = Datamodel.makeValueSnak(retrievedPid, + Datamodel.makeTimeValue(2018, (byte) 2, (byte) 28, TimeValue.CM_GREGORIAN_PRO)); + private Snak mainSnak1 = Datamodel.makeValueSnak(inceptionPid, date1); + private Snak mainSnak2 = Datamodel.makeValueSnak(inceptionPid, date2); + private Claim claim1 = Datamodel.makeClaim(qid1, mainSnak1, Collections.emptyList()); + private Claim claim2 = Datamodel.makeClaim(qid2, mainSnak2, Collections.emptyList()); + private SnakGroup refSnakGroup = Datamodel.makeSnakGroup(Collections.singletonList(refSnak)); + private SnakGroup retrievedSnakGroup = Datamodel.makeSnakGroup(Collections.singletonList(retrievedSnak)); + private Statement statement1 = Datamodel.makeStatement(claim1, + Collections.singletonList(Datamodel.makeReference(Arrays.asList(refSnakGroup, retrievedSnakGroup))), + StatementRank.NORMAL, ""); + private Statement statement2 = Datamodel.makeStatement(claim2, + Collections.singletonList(Datamodel.makeReference(Collections.singletonList(retrievedSnakGroup))), + StatementRank.NORMAL, ""); + + static JSONObject jsonFromFile(String filename) throws IOException, JSONException { + byte[] contents = Files.readAllBytes(Paths.get(filename)); + String decoded = new String(contents, "utf-8"); + return ParsingUtilities.evaluateJsonStringToObject(decoded); + } + + @Test + public void testSerialize() throws JSONException, IOException { + JSONObject serialized = jsonFromFile("data/schema/history_of_medicine.json"); + WikibaseSchema parsed = WikibaseSchema.reconstruct(serialized); + StringWriter writer = new StringWriter(); + JSONWriter jsonWriter = new JSONWriter(writer); + parsed.write(jsonWriter, new Properties()); + writer.close(); + JSONObject newSerialized = ParsingUtilities.evaluateJsonStringToObject(writer.toString()); + // toString because it looks like JSONObject equality isn't great… + assertEquals(jsonFromFile("data/schema/history_of_medicine_normalized.json").toString(), newSerialized.toString()); + } + + @Test + public void testDeserialize() throws JSONException, IOException { + // this json file was generated by an earlier version of the software + // it contains extra "type" fields that are now ignored. + JSONObject serialized = jsonFromFile("data/schema/roarmap.json"); + WikibaseSchema.reconstruct(serialized); + } + + @Test + public void testEvaluate() throws JSONException, IOException { + JSONObject serialized = jsonFromFile("data/schema/inception.json"); + WikibaseSchema schema = WikibaseSchema.reconstruct(serialized); + Project project = this.createCSVProject( + "subject,inception,reference\n"+ + "Q1377,1919,http://www.ljubljana-slovenia.com/university-ljubljana\n"+ + "Q865528,1965,"); + project.rows.get(0).cells.set(0, TestingDataGenerator.makeMatchedCell("Q1377", "University of Ljubljana")); + project.rows.get(1).cells.set(0, TestingDataGenerator.makeMatchedCell("Q865528", "University of Warwick")); + Engine engine = new Engine(project); + List updates = schema.evaluate(project, engine); + List expected = new ArrayList<>(); + ItemUpdate update1 = new ItemUpdate(qid1); + update1.addStatement(statement1); + expected.add(update1); + ItemUpdate update2 = new ItemUpdate(qid2); + update2.addStatement(statement2); + expected.add(update2); + assertEquals(expected, updates); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/entityvalues/ReconEntityIdValueTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/entityvalues/ReconEntityIdValueTest.java index 564d621e7..6d306f851 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/entityvalues/ReconEntityIdValueTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/entityvalues/ReconEntityIdValueTest.java @@ -94,4 +94,10 @@ public class ReconEntityIdValueTest { public void testGetRecon() { assertEquals(newItem.getReconInternalId(), newItem.getRecon().judgmentHistoryEntry); } + + @Test + public void testToString() { + assertTrue(existingItem.toString().contains("Q42")); + assertTrue(newItem.toString().contains("new")); + } } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/JacksonSerializationTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/JacksonSerializationTest.java index f7df954f8..8c2dc6dc7 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/JacksonSerializationTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/JacksonSerializationTest.java @@ -29,15 +29,19 @@ public class JacksonSerializationTest { } } + @SuppressWarnings({ "rawtypes", "unchecked" }) public static void testDeserialize(Class targetClass, Object pojo, String inputJson) { try { - assertEquals(pojo, mapper.readValue(inputJson, targetClass)); + Object deserialized = mapper.readValue(inputJson, targetClass); + assertEquals(pojo, deserialized); + assertEquals(pojo.hashCode(), deserialized.hashCode()); } catch (IOException e) { e.printStackTrace(); Assert.fail("Failed to deserialize object"); } } + @SuppressWarnings("rawtypes") public static void canonicalSerialization(Class targetClass, Object pojo, String json) { testSerialize(pojo, json); testDeserialize(targetClass, pojo, json); diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/TestingDataGenerator.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/TestingDataGenerator.java index 1c34a0ede..c6999ff3a 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/TestingDataGenerator.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/TestingDataGenerator.java @@ -5,10 +5,8 @@ import java.util.Collections; import org.openrefine.wikidata.schema.WbLanguageConstant; import org.openrefine.wikidata.schema.WbMonolingualExpr; import org.openrefine.wikidata.schema.WbStringConstant; -import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue; import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue; import org.openrefine.wikidata.schema.entityvalues.ReconPropertyIdValue; -import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import com.google.refine.model.Cell; import com.google.refine.model.Recon; @@ -42,19 +40,19 @@ public class TestingDataGenerator { return new Cell(name, makeMatchedRecon(qid, name)); } - public static ReconEntityIdValue makeNewItemIdValue(long judgementId, String name) { + public static ReconItemIdValue makeNewItemIdValue(long judgementId, String name) { return new ReconItemIdValue(makeNewItemRecon(judgementId), name); } - public static ReconEntityIdValue makeMatchedItemIdValue(String qid, String name) { + public static ReconItemIdValue makeMatchedItemIdValue(String qid, String name) { return new ReconItemIdValue(makeMatchedRecon(qid, name), name); } - public static ReconEntityIdValue makeNewPropertyIdValue(long judgmentId, String name) { + public static ReconPropertyIdValue makeNewPropertyIdValue(long judgmentId, String name) { return new ReconPropertyIdValue(makeNewItemRecon(judgmentId), name); } - public static ReconEntityIdValue makeMatchedPropertyIdValue(String pid, String name) { + public static ReconPropertyIdValue makeMatchedPropertyIdValue(String pid, String name) { return new ReconPropertyIdValue(makeMatchedRecon(pid, name), name); }