diff --git a/extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java b/extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java index d9a6d54b0..2076abcda 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java @@ -54,8 +54,9 @@ import org.openrefine.wikidata.exporters.QuickStatementsExporter; import org.openrefine.wikidata.qa.EditInspector; import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.qa.QAWarningStore; -import org.openrefine.wikidata.schema.ItemUpdate; import org.openrefine.wikidata.schema.WikibaseSchema; +import org.openrefine.wikidata.updates.ItemUpdate; + import com.google.refine.model.Project; import com.google.refine.util.ParsingUtilities; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java index 14291930f..44459da7e 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java @@ -10,8 +10,8 @@ import com.google.refine.browsing.Engine; import com.google.refine.exporters.WriterExporter; import com.google.refine.model.Project; -import org.openrefine.wikidata.schema.ItemUpdate; import org.openrefine.wikidata.schema.WikibaseSchema; +import org.openrefine.wikidata.updates.ItemUpdate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.wikidata.wdtk.datamodel.interfaces.Claim; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java b/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java index 5467ea5c4..3062604ec 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java @@ -6,7 +6,6 @@ import java.io.Writer; import java.util.HashMap; import java.util.List; import java.util.ArrayList; -import java.util.Collections; import java.util.Map; import java.util.Properties; import java.util.stream.Collectors; @@ -17,7 +16,7 @@ import org.json.JSONWriter; import org.openrefine.wikidata.editing.ConnectionManager; import org.openrefine.wikidata.editing.NewItemLibrary; -import org.openrefine.wikidata.schema.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdate; import org.openrefine.wikidata.schema.WikibaseSchema; import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue; import org.slf4j.Logger; @@ -30,7 +29,6 @@ import org.wikidata.wdtk.datamodel.interfaces.ItemDocument; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.util.WebResourceFetcherImpl; import org.wikidata.wdtk.wikibaseapi.ApiConnection; -import org.wikidata.wdtk.wikibaseapi.TermStatementUpdate; import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor; import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java index ebe4a7b7f..c40acf16a 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java @@ -17,7 +17,7 @@ import org.openrefine.wikidata.qa.scrutinizers.SelfReferentialScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.SingleValueScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.UnsourcedScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.WhitespaceScrutinizer; -import org.openrefine.wikidata.schema.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdate; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; /** diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java index da2245190..f3335f04f 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java @@ -1,13 +1,12 @@ package org.openrefine.wikidata.qa.scrutinizers; import java.util.List; -import java.util.Map; import org.openrefine.wikidata.qa.ConstraintFetcher; import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.qa.QAWarning.Severity; import org.openrefine.wikidata.qa.QAWarningStore; -import org.openrefine.wikidata.schema.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdate; /** * Interface for any class that diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemEditScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemEditScrutinizer.java index e5bedb3ab..8053ab6bc 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemEditScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemEditScrutinizer.java @@ -2,7 +2,7 @@ package org.openrefine.wikidata.qa.scrutinizers; import java.util.List; -import org.openrefine.wikidata.schema.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdate; public abstract class ItemEditScrutinizer extends EditScrutinizer { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java index d5c43597d..ee49fa74e 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java @@ -1,7 +1,7 @@ package org.openrefine.wikidata.qa.scrutinizers; import org.openrefine.wikidata.qa.QAWarning; -import org.openrefine.wikidata.schema.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdate; import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; /** diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java index 35ffc3666..ad801c208 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java @@ -2,7 +2,7 @@ package org.openrefine.wikidata.qa.scrutinizers; import java.util.List; -import org.openrefine.wikidata.schema.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdate; public class NoEditsMadeScrutinizer extends EditScrutinizer { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java index 7fb788dad..e09fa5c6e 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java @@ -4,7 +4,7 @@ import java.util.HashSet; import java.util.Set; import org.openrefine.wikidata.qa.QAWarning; -import org.openrefine.wikidata.schema.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdate; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; import org.wikidata.wdtk.datamodel.interfaces.Statement; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementGroupScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementGroupScrutinizer.java index 8e55c60ac..88df82086 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementGroupScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementGroupScrutinizer.java @@ -1,6 +1,6 @@ package org.openrefine.wikidata.qa.scrutinizers; -import org.openrefine.wikidata.schema.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdate; import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; public abstract class StatementGroupScrutinizer extends ItemEditScrutinizer { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java index cefba7803..071bd36d8 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java @@ -1,6 +1,6 @@ package org.openrefine.wikidata.qa.scrutinizers; -import org.openrefine.wikidata.schema.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdate; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.Statement; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java index cf046ae32..95937a5dc 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java @@ -1,6 +1,6 @@ package org.openrefine.wikidata.qa.scrutinizers; -import org.openrefine.wikidata.schema.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdate; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.datamodel.interfaces.Snak; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java index b38afe56d..6ce9c21a4 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java @@ -3,6 +3,8 @@ package org.openrefine.wikidata.schema; import java.util.List; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; +import org.openrefine.wikidata.updates.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.openrefine.wikidata.utils.JacksonJsonizable; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.Statement; @@ -40,7 +42,7 @@ public class WbItemDocumentExpr extends JacksonJsonizable implements WbExpressio @Override public ItemUpdate evaluate(ExpressionContext ctxt) throws SkipSchemaExpressionException { ItemIdValue subjectId = getSubject().evaluate(ctxt); - ItemUpdate update = new ItemUpdate(subjectId); + ItemUpdateBuilder update = new ItemUpdateBuilder(subjectId); for(WbStatementGroupExpr expr : getStatementGroups()) { try { for(Statement s : expr.evaluate(ctxt, subjectId).getStatements()) { @@ -53,7 +55,7 @@ public class WbItemDocumentExpr extends JacksonJsonizable implements WbExpressio for(WbNameDescExpr expr : getNameDescs()) { expr.contributeTo(update, ctxt); } - return update; + return update.build(); } @JsonProperty("subject") diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java index d31c040a6..d09c03bf5 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java @@ -2,6 +2,7 @@ package org.openrefine.wikidata.schema; import org.jsoup.helper.Validate; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; +import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import com.fasterxml.jackson.annotation.JsonCreator; @@ -11,7 +12,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; /** * An expression that represent a term (label, description or alias). * The structure is slightly different from other expressions because - * we need to call different methods on {@link ItemUpdate}. + * we need to call different methods on {@link ItemUpdateBuilder}. * * @author Antonin Delpeuch * @@ -46,7 +47,7 @@ public class WbNameDescExpr { * @param ctxt * the evaluation context for the expression */ - public void contributeTo(ItemUpdate item, ExpressionContext ctxt) { + public void contributeTo(ItemUpdateBuilder item, ExpressionContext ctxt) { try { MonolingualTextValue val = getValue().evaluate(ctxt); switch (getType()) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java index 83bead6be..efd5c8125 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java @@ -19,6 +19,8 @@ import com.google.refine.model.Project; import com.google.refine.model.Row; import org.openrefine.wikidata.schema.WbItemDocumentExpr; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; +import org.openrefine.wikidata.updates.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.openrefine.wikidata.qa.QAWarningStore; import org.openrefine.wikidata.schema.ExpressionContext; import org.openrefine.wikidata.utils.JacksonJsonizable; @@ -73,7 +75,7 @@ public class WikibaseSchema implements OverlayModel { * @return */ public List evaluateItemDocuments(ExpressionContext ctxt) { - List result = new ArrayList(); + List result = new ArrayList<>(); for (WbItemDocumentExpr expr : itemDocumentExprs) { try { @@ -104,7 +106,7 @@ public class WikibaseSchema implements OverlayModel { * generating order (not merged yet). */ public List evaluate(Project project, Engine engine, QAWarningStore warningStore) { - List result = new ArrayList(); + List result = new ArrayList<>(); FilteredRows filteredRows = engine.getAllFilteredRows(); filteredRows.accept(project, new EvaluatingRowVisitor(result, warningStore)); return result; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/ItemUpdate.java b/extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdate.java similarity index 67% rename from extensions/wikidata/src/org/openrefine/wikidata/schema/ItemUpdate.java rename to extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdate.java index 797344eb5..d0722237c 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/ItemUpdate.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdate.java @@ -1,6 +1,7 @@ -package org.openrefine.wikidata.schema; +package org.openrefine.wikidata.updates; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -17,6 +18,9 @@ import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; /** * A class to plan an update of an item, after evaluating the statements @@ -26,12 +30,12 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; * @author Antonin Delpeuch */ public class ItemUpdate { - private ItemIdValue qid; - private Set addedStatements; - private Set deletedStatements; - private Set labels; - private Set descriptions; - private Set aliases; + private final ItemIdValue qid; + private final Set addedStatements; + private final Set deletedStatements; + private final Set labels; + private final Set descriptions; + private final Set aliases; /** * Constructor. @@ -39,61 +43,42 @@ public class ItemUpdate { * @param qid * the subject of the document. It can be a reconciled item value for new items. */ - public ItemUpdate(ItemIdValue qid) { + @JsonCreator + public ItemUpdate( + @JsonProperty("subject") ItemIdValue qid, + @JsonProperty("addedStatements") Set addedStatements, + @JsonProperty("deletedStatements") Set deletedStatements, + @JsonProperty("labels") Set labels, + @JsonProperty("descriptions") Set descriptions, + @JsonProperty("addedAliases") Set aliases) { Validate.notNull(qid); this.qid = qid; - this.addedStatements = new HashSet<>(); - this.deletedStatements = new HashSet(); - this.labels = new HashSet(); - this.descriptions = new HashSet(); - this.aliases = new HashSet(); - } - - /** - * Mark a statement for insertion. If it matches an existing - * statement, it will update the statement instead. - * - * @param statement - * the statement to add or update - */ - public void addStatement(Statement statement) { - addedStatements.add(statement); - } - - /** - * Mark a statement for deletion. If no such statement exists, - * nothing will be deleted. - * - * @param statement - * the statement to delete - */ - public void deleteStatement(Statement statement) { - deletedStatements.add(statement); - } - - /** - * Add a list of statement, as in {@link addStatement}. - * - * @param statements - * the statements to add - */ - public void addStatements(Set statements) { - addedStatements.addAll(statements); - } - - /** - * Delete a list of statements, as in {@link deleteStatement}. - * - * @param statements - * the statements to delete - */ - public void deleteStatements(Set statements) { - deletedStatements.addAll(statements); + if(addedStatements == null) { + addedStatements = Collections.emptySet(); + } + this.addedStatements = addedStatements; + if(deletedStatements == null) { + deletedStatements = Collections.emptySet(); + } + this.deletedStatements = deletedStatements; + if(labels == null) { + labels = Collections.emptySet(); + } + this.labels = labels; + if(descriptions == null) { + descriptions = Collections.emptySet(); + } + this.descriptions = descriptions; + if(aliases == null) { + aliases = Collections.emptySet(); + } + this.aliases = aliases; } /** * @return the subject of the item */ + @JsonProperty("subject") public ItemIdValue getItemId() { return qid; } @@ -101,6 +86,7 @@ public class ItemUpdate { /** * @return the set of all added statements */ + @JsonProperty("addedStatements") public Set getAddedStatements() { return addedStatements; } @@ -108,74 +94,15 @@ public class ItemUpdate { /** * @return the list of all deleted statements */ + @JsonProperty("deletedStatements") public Set getDeletedStatements() { return deletedStatements; } - - /** - * Merges all the changes in other into this instance. - * Both updates should have the same subject. - * - * @param other - * the other change that should be merged - */ - public void merge(ItemUpdate other) { - Validate.isTrue(qid.equals(other.getItemId())); - addStatements(other.getAddedStatements()); - deleteStatements(other.getDeletedStatements()); - labels.addAll(other.getLabels()); - descriptions.addAll(other.getDescriptions()); - aliases.addAll(other.getAliases()); - } - - /** - * @return true when this change is empty - * (no statements or terms changed) - */ - public boolean isNull() { - return (addedStatements.isEmpty() - && deletedStatements.isEmpty() - && labels.isEmpty() - && descriptions.isEmpty() - && aliases.isEmpty()); - } - - /** - * Adds a label to the item. It will override any - * existing label in this language. - * - * @param label - * the label to add - */ - public void addLabel(MonolingualTextValue label) { - labels.add(label); - } - - /** - * Adds a description to the item. It will override any existing - * description in this language. - * - * @param description - * the description to add - */ - public void addDescription(MonolingualTextValue description) { - descriptions.add(description); - } - - /** - * Adds an alias to the item. It will be added to any existing - * aliases in that language. - * - * @param alias - * the alias to add - */ - public void addAlias(MonolingualTextValue alias) { - aliases.add(alias); - } /** * @return the list of updated labels */ + @JsonProperty("labels") public Set getLabels() { return labels; } @@ -183,6 +110,7 @@ public class ItemUpdate { /** * @return the list of updated descriptions */ + @JsonProperty("descriptions") public Set getDescriptions() { return descriptions; } @@ -190,10 +118,48 @@ public class ItemUpdate { /** * @return the list of updated aliases */ + @JsonProperty("addedAliases") public Set getAliases() { return aliases; } + /** + * @return true when this change is empty + * (no statements or terms changed) + */ + @JsonIgnore + public boolean isNull() { + return (addedStatements.isEmpty() + && deletedStatements.isEmpty() + && labels.isEmpty() + && descriptions.isEmpty() + && aliases.isEmpty()); + } + + /** + * Merges all the changes in other into this instance. + * Both updates should have the same subject. + * + * @param other + * the other change that should be merged + */ + public ItemUpdate merge(ItemUpdate other) { + Validate.isTrue(qid.equals(other.getItemId())); + Set newAddedStatements = new HashSet<>(addedStatements); + newAddedStatements.addAll(other.getAddedStatements()); + Set newDeletedStatements = new HashSet<>(deletedStatements); + newDeletedStatements.addAll(other.getDeletedStatements()); + Set newLabels = new HashSet<>(labels); + newLabels.addAll(other.getLabels()); + Set newDescriptions = new HashSet<>(descriptions); + newDescriptions.addAll(other.getDescriptions()); + Set newAliases = new HashSet<>(aliases); + newAliases.addAll(other.getDescriptions()); + return new ItemUpdate( + qid, newAddedStatements, newDeletedStatements, + newLabels, newDescriptions, newAliases); + } + /** * Group added statements in StatementGroups: useful if the * item is new. @@ -215,7 +181,7 @@ public class ItemUpdate { } return result; } - + /** * Group a list of ItemUpdates by subject: this is useful to make one single edit * per item. @@ -224,7 +190,7 @@ public class ItemUpdate { * @return a map from item ids to merged ItemUpdate for that id */ public static Map groupBySubject(List itemDocuments) { - Map map = new HashMap(); + Map map = new HashMap<>(); for(ItemUpdate update : itemDocuments) { if (update.isNull()) { continue; @@ -233,38 +199,14 @@ public class ItemUpdate { ItemIdValue qid = update.getItemId(); if (map.containsKey(qid)) { ItemUpdate oldUpdate = map.get(qid); - oldUpdate.merge(update); + map.put(qid, oldUpdate.merge(update)); } else { map.put(qid, update); } } return map; } - - /** - * This should only be used when creating a new item. - * This ensures that we never add an alias without adding - * a label in the same language. - */ - public void normalizeLabelsAndAliases() { - // Ensure that we are only adding aliases with labels - Set labelLanguages = labels.stream() - .map(l -> l.getLanguageCode()) - .collect(Collectors.toSet()); - System.out.println(labelLanguages); - - Set filteredAliases = new HashSet<>(); - for(MonolingualTextValue alias : aliases) { - if(!labelLanguages.contains(alias.getLanguageCode())) { - labelLanguages.add(alias.getLanguageCode()); - labels.add(alias); - } else { - filteredAliases.add(alias); - } - } - aliases = filteredAliases; - } - + /** * Is this update about a new item? */ @@ -272,6 +214,31 @@ public class ItemUpdate { return "Q0".equals(getItemId().getId()); } + /** + * This should only be used when creating a new item. + * This ensures that we never add an alias without adding + * a label in the same language. + */ + public ItemUpdate normalizeLabelsAndAliases() { + // Ensure that we are only adding aliases with labels + Set labelLanguages = labels.stream() + .map(l -> l.getLanguageCode()) + .collect(Collectors.toSet()); + + Set filteredAliases = new HashSet<>(); + Set newLabels = new HashSet<>(labels); + for(MonolingualTextValue alias : aliases) { + if(!labelLanguages.contains(alias.getLanguageCode())) { + labelLanguages.add(alias.getLanguageCode()); + newLabels.add(alias); + } else { + filteredAliases.add(alias); + } + } + return new ItemUpdate(qid, addedStatements, deletedStatements, + newLabels, descriptions, filteredAliases); + } + @Override public boolean equals(Object other) { if(other == null || !ItemUpdate.class.isInstance(other)) { @@ -310,4 +277,5 @@ public class ItemUpdate { builder.append("\n>"); return builder.toString(); } + } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdateBuilder.java b/extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdateBuilder.java new file mode 100644 index 000000000..8c3bc27fd --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdateBuilder.java @@ -0,0 +1,143 @@ +package org.openrefine.wikidata.updates; + +import java.util.Set; +import java.util.HashSet; + +import org.jsoup.helper.Validate; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; +import org.wikidata.wdtk.datamodel.interfaces.Statement; + + +/** + * Constructs a {@link ItemUpdate} incrementally. + * + * @author Antonin Delpeuch + * + */ +public class ItemUpdateBuilder { + private ItemIdValue qid; + private Set addedStatements; + private Set deletedStatements; + private Set labels; + private Set descriptions; + private Set aliases; + private boolean built; + + /** + * Constructor. + * + * @param qid + * the subject of the document. It can be a reconciled item value for new items. + */ + public ItemUpdateBuilder(ItemIdValue qid) { + Validate.notNull(qid); + this.qid = qid; + this.addedStatements = new HashSet<>(); + this.deletedStatements = new HashSet(); + this.labels = new HashSet(); + this.descriptions = new HashSet(); + this.aliases = new HashSet(); + this.built = false; + } + + /** + * Mark a statement for insertion. If it matches an existing + * statement, it will update the statement instead. + * + * @param statement + * the statement to add or update + */ + public ItemUpdateBuilder addStatement(Statement statement) { + Validate.isTrue(!built, "ItemUpdate has already been built"); + addedStatements.add(statement); + return this; + } + + /** + * Mark a statement for deletion. If no such statement exists, + * nothing will be deleted. + * + * @param statement + * the statement to delete + */ + public ItemUpdateBuilder deleteStatement(Statement statement) { + Validate.isTrue(!built, "ItemUpdate has already been built"); + deletedStatements.add(statement); + return this; + } + + /** + * Add a list of statement, as in {@link addStatement}. + * + * @param statements + * the statements to add + */ + public ItemUpdateBuilder addStatements(Set statements) { + Validate.isTrue(!built, "ItemUpdate has already been built"); + addedStatements.addAll(statements); + return this; + } + + /** + * Delete a list of statements, as in {@link deleteStatement}. + * + * @param statements + * the statements to delete + */ + public ItemUpdateBuilder deleteStatements(Set statements) { + Validate.isTrue(!built, "ItemUpdate has already been built"); + deletedStatements.addAll(statements); + return this; + } + + /** + * Adds a label to the item. It will override any + * existing label in this language. + * + * @param label + * the label to add + */ + public ItemUpdateBuilder addLabel(MonolingualTextValue label) { + Validate.isTrue(!built, "ItemUpdate has already been built"); + labels.add(label); + return this; + } + + /** + * Adds a description to the item. It will override any existing + * description in this language. + * + * @param description + * the description to add + */ + public ItemUpdateBuilder addDescription(MonolingualTextValue description) { + Validate.isTrue(!built, "ItemUpdate has already been built"); + descriptions.add(description); + return this; + } + + /** + * Adds an alias to the item. It will be added to any existing + * aliases in that language. + * + * @param alias + * the alias to add + */ + public ItemUpdateBuilder addAlias(MonolingualTextValue alias) { + Validate.isTrue(!built, "ItemUpdate has already been built"); + aliases.add(alias); + return this; + } + + /** + * Constructs the {@link ItemUpdate}. + * @return + */ + public ItemUpdate build() { + built = true; + return new ItemUpdate(qid, addedStatements, deletedStatements, + labels, descriptions, aliases); + } + +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemDocumentExprTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemDocumentExprTest.java index 93d9d3f8f..98ded10d2 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemDocumentExprTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemDocumentExprTest.java @@ -3,6 +3,8 @@ package org.openrefine.wikidata.schema; import java.util.Collections; import org.openrefine.wikidata.testing.JacksonSerializationTest; +import org.openrefine.wikidata.updates.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.testng.annotations.Test; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; @@ -39,9 +41,7 @@ public class WbItemDocumentExprTest extends WbExpressionTest { @Test public void testEvaluate() { setRow(recon("Q3434"), "2010-07-23", "3.898,4.389", "my alias", recon("Q23")); - ItemUpdate result = new ItemUpdate(subject); - result.addAlias(alias); - result.addStatement(fullStatement); + ItemUpdate result = new ItemUpdateBuilder(subject).addAlias(alias).addStatement(fullStatement).build(); evaluatesTo(result, expr); } @@ -54,16 +54,14 @@ public class WbItemDocumentExprTest extends WbExpressionTest { @Test public void testStatementSkipped() { setRow(recon("Q3434"), "2010-07-23", "3.898,invalid4.389", "my alias", recon("Q23")); - ItemUpdate result = new ItemUpdate(subject); - result.addAlias(alias); + ItemUpdate result = new ItemUpdateBuilder(subject).addAlias(alias).build(); evaluatesTo(result, expr); } @Test public void testAliasSkipped() { setRow(recon("Q3434"), "2010-07-23", "3.898,4.389", "", recon("Q23")); - ItemUpdate result = new ItemUpdate(subject); - result.addStatement(fullStatement); + ItemUpdate result = new ItemUpdateBuilder(subject).addStatement(fullStatement).build(); evaluatesTo(result, expr); } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescExprTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescExprTest.java index f263fc84c..8f4151fcf 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescExprTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescExprTest.java @@ -6,6 +6,7 @@ import java.util.Collections; import org.openrefine.wikidata.testing.JacksonSerializationTest; import org.openrefine.wikidata.testing.TestingDataGenerator; +import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.testng.annotations.Test; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; @@ -25,38 +26,38 @@ public class WbNameDescExprTest extends WbExpressionTest { public void testContributeToLabel() { WbNameDescExpr labelExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.LABEL, TestingDataGenerator.getTestMonolingualExpr("fr", "français", "le croissant magnifique")); - ItemUpdate update = new ItemUpdate(subject); + ItemUpdateBuilder update = new ItemUpdateBuilder(subject); labelExpr.contributeTo(update, ctxt); assertEquals(Collections.singleton(Datamodel.makeMonolingualTextValue("le croissant magnifique", "fr")), - update.getLabels()); + update.build().getLabels()); } @Test public void testContributeToDescription() { WbNameDescExpr descriptionExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.DESCRIPTION, TestingDataGenerator.getTestMonolingualExpr("de", "Deutsch", "wunderschön")); - ItemUpdate update = new ItemUpdate(subject); + ItemUpdateBuilder update = new ItemUpdateBuilder(subject); descriptionExpr.contributeTo(update, ctxt); assertEquals(Collections.singleton(Datamodel.makeMonolingualTextValue("wunderschön", "de")), - update.getDescriptions()); + update.build().getDescriptions()); } @Test public void testContributeToAlias() { WbNameDescExpr aliasExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS, TestingDataGenerator.getTestMonolingualExpr("en", "English", "snack")); - ItemUpdate update = new ItemUpdate(subject); + ItemUpdateBuilder update = new ItemUpdateBuilder(subject); aliasExpr.contributeTo(update, ctxt); assertEquals(Collections.singleton(Datamodel.makeMonolingualTextValue("snack", "en")), - update.getAliases()); + update.build().getAliases()); } @Test public void testSkipped() { - ItemUpdate update = new ItemUpdate(subject); + ItemUpdateBuilder update = new ItemUpdateBuilder(subject); setRow(""); expr.contributeTo(update, ctxt); - assertEquals(new ItemUpdate(subject), update); + assertEquals(new ItemUpdateBuilder(subject).build(), update.build()); } @Test diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java index 71423e2e4..7628f955f 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java @@ -16,6 +16,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; import org.openrefine.wikidata.testing.TestingDataGenerator; +import org.openrefine.wikidata.updates.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.testng.annotations.Test; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.Claim; @@ -101,11 +103,9 @@ public class WikibaseSchemaTest extends RefineTest { Engine engine = new Engine(project); List updates = schema.evaluate(project, engine); List expected = new ArrayList<>(); - ItemUpdate update1 = new ItemUpdate(qid1); - update1.addStatement(statement1); + ItemUpdate update1 = new ItemUpdateBuilder(qid1).addStatement(statement1).build(); expected.add(update1); - ItemUpdate update2 = new ItemUpdate(qid2); - update2.addStatement(statement2); + ItemUpdate update2 = new ItemUpdateBuilder(qid2).addStatement(statement2).build(); expected.add(update2); assertEquals(expected, updates); } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ItemUpdateTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/ItemUpdateTest.java similarity index 66% rename from extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ItemUpdateTest.java rename to extensions/wikidata/tests/src/org/openrefine/wikidata/updates/ItemUpdateTest.java index 265074905..99b82cda0 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ItemUpdateTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/ItemUpdateTest.java @@ -1,4 +1,4 @@ -package org.openrefine.wikidata.schema; +package org.openrefine.wikidata.updates; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -14,6 +14,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.openrefine.wikidata.testing.TestingDataGenerator; +import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.testng.annotations.Test; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.Claim; @@ -31,8 +32,7 @@ public class ItemUpdateTest { private ItemIdValue newSubject = TestingDataGenerator.makeNewItemIdValue(1234L, "new item"); private ItemIdValue sameNewSubject = TestingDataGenerator.makeNewItemIdValue(1234L, "other new item"); private ItemIdValue matchedSubject = TestingDataGenerator.makeMatchedItemIdValue("Q78", "well known item"); - private ItemUpdate update = new ItemUpdate(existingSubject); - + private PropertyIdValue pid1 = Datamodel.makeWikidataPropertyIdValue("P348"); private PropertyIdValue pid2 = Datamodel.makeWikidataPropertyIdValue("P52"); private Claim claim1 = Datamodel.makeClaim(existingSubject, @@ -55,26 +55,29 @@ public class ItemUpdateTest { @Test(expectedExceptions=IllegalArgumentException.class) public void testCreateWithoutSubject() { - new ItemUpdate(null); + new ItemUpdateBuilder(null); } @Test public void testIsNull() { + ItemUpdate update = new ItemUpdateBuilder(existingSubject).build(); assertTrue(update.isNull()); } @Test public void testIsNew() { - ItemUpdate newUpdate = new ItemUpdate(newSubject); + ItemUpdate newUpdate = new ItemUpdateBuilder(newSubject).build(); assertTrue(newUpdate.isNew()); + ItemUpdate update = new ItemUpdateBuilder(existingSubject).build(); assertFalse(update.isNew()); } @Test public void testAddStatements() { - ItemUpdate update = new ItemUpdate(existingSubject); - update.addStatement(statement1); - update.addStatement(statement2); + ItemUpdate update = new ItemUpdateBuilder(existingSubject) + .addStatement(statement1) + .addStatement(statement2) + .build(); assertEquals(Arrays.asList(statement1, statement2).stream().collect(Collectors.toSet()), update.getAddedStatements()); assertEquals(statementGroups, update.getAddedStatementGroups().stream().collect(Collectors.toSet())); @@ -82,39 +85,36 @@ public class ItemUpdateTest { @Test public void testDeleteStatements() { - ItemUpdate update = new ItemUpdate(existingSubject); - update.deleteStatement(statement1); - update.deleteStatement(statement2); + ItemUpdate update = new ItemUpdateBuilder(existingSubject) + .deleteStatement(statement1) + .deleteStatement(statement2) + .build(); assertEquals(Arrays.asList(statement1, statement2).stream().collect(Collectors.toSet()), update.getDeletedStatements()); } @Test public void testMerge() { - ItemUpdate updateA = new ItemUpdate(existingSubject); - updateA.addStatement(statement1); - ItemUpdate updateB = new ItemUpdate(existingSubject); - updateB.addStatement(statement2); + ItemUpdate updateA = new ItemUpdateBuilder(existingSubject).addStatement(statement1).build(); + ItemUpdate updateB = new ItemUpdateBuilder(existingSubject).addStatement(statement2).build(); assertNotEquals(updateA, updateB); - updateA.merge(updateB); + ItemUpdate merged = updateA.merge(updateB); assertEquals(statementGroups, - updateA.getAddedStatementGroups().stream().collect(Collectors.toSet())); + merged.getAddedStatementGroups().stream().collect(Collectors.toSet())); } @Test public void testGroupBySubject() { - ItemUpdate updateA = new ItemUpdate(newSubject); - updateA.addStatement(statement1); - ItemUpdate updateB = new ItemUpdate(sameNewSubject); - updateB.addStatement(statement2); - ItemUpdate updateC = new ItemUpdate(existingSubject); - updateC.addLabel(label); - ItemUpdate updateD = new ItemUpdate(matchedSubject); + ItemUpdate updateA = new ItemUpdateBuilder(newSubject).addStatement(statement1).build(); + ItemUpdate updateB = new ItemUpdateBuilder(sameNewSubject).addStatement(statement2).build(); + ItemUpdate updateC = new ItemUpdateBuilder(existingSubject).addLabel(label).build(); + ItemUpdate updateD = new ItemUpdateBuilder(matchedSubject).build(); Map grouped = ItemUpdate.groupBySubject( Arrays.asList(updateA, updateB, updateC, updateD)); - ItemUpdate mergedUpdate = new ItemUpdate(newSubject); - mergedUpdate.addStatement(statement1); - mergedUpdate.addStatement(statement2); + ItemUpdate mergedUpdate = new ItemUpdateBuilder(newSubject) + .addStatement(statement1) + .addStatement(statement2) + .build(); Map expected = new HashMap<>(); expected.put(newSubject, mergedUpdate); expected.put(existingSubject, updateC); @@ -125,15 +125,17 @@ public class ItemUpdateTest { public void testNormalizeTerms() { MonolingualTextValue aliasEn = Datamodel.makeMonolingualTextValue("alias", "en"); MonolingualTextValue aliasFr = Datamodel.makeMonolingualTextValue("coucou", "fr"); - ItemUpdate updateA = new ItemUpdate(newSubject); - updateA.addLabel(label); - updateA.addAlias(aliasEn); - updateA.addAlias(aliasFr); - updateA.normalizeLabelsAndAliases(); - ItemUpdate expectedUpdate = new ItemUpdate(newSubject); - expectedUpdate.addLabel(label); - expectedUpdate.addAlias(aliasEn); - expectedUpdate.addLabel(aliasFr); - assertEquals(expectedUpdate, updateA); + ItemUpdate updateA = new ItemUpdateBuilder(newSubject) + .addLabel(label) + .addAlias(aliasEn) + .addAlias(aliasFr) + .build(); + ItemUpdate normalized = updateA.normalizeLabelsAndAliases(); + ItemUpdate expectedUpdate = new ItemUpdateBuilder(newSubject) + .addLabel(label) + .addAlias(aliasEn) + .addLabel(aliasFr) + .build(); + assertEquals(expectedUpdate, normalized); } }