s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #2065 from OpenRefine/issue-2063-overriding-terms

Browse Source 
Wikidata: Add labels or descriptions without overriding existing ones
This commit is contained in:
Antonin Delpeuch 2019-07-02 10:18:56 +02:00 committed by GitHub
commit 09a42fd2f5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 357 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
extensions/wikidata/module/langs/translation-en.json
View File

@ -32,7 +32,12 @@
"wikidata-schema/nb-references": " references",
"wikidata-schema/remove-column": "remove column",
"wikidata-schema/label": "Label",
"wikidata-schema/label-if-new": "Label (do not override)",
"wikidata-schema/label-override": "Label (override if present)",
"wikidata-schema/description": "Description",
"wikidata-schema/description-if-new": "Description (do not override)",
"wikidata-schema/description-override": "Description (override if present)",
"wikidata-schema/override-term": "override if present",
"wikidata-schema/alias": "Alias",
"wikidata-schema/item-or-reconciled-column": "type item or drag reconciled column here",
"wikidata-schema/amount": "amount",

7
extensions/wikidata/module/langs/translation-fr.json
View File

@ -28,8 +28,13 @@
"wikidata-schema/property-placeholder": "propriété",
"wikidata-schema/nb-references": " références",
"wikidata-schema/remove-column": "supprimer la colonne",
"wikidata-schema/label": "Libellé",
"wikidata-schema/label-if-new": "Libellé",
"wikidata-schema/label-if-new": "Libellé (sans écraser)",
"wikidata-schema/label-override": "Libellé (écrase si déjà présent)",
"wikidata-schema/description": "Description",
"wikidata-schema/description-if-new": "Description (sans écraser)",
"wikidata-schema/description-override": "Description (écrase si déjà présent)",
"wikidata-schema/override-term": "Écrase si déjà présent",
"wikidata-schema/alias": "Alias",
"wikidata-schema/item-or-reconciled-column": "entrer un élément ou déposer une colonne réconciliée ici",
"wikidata-schema/amount": "quantité",

40
extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js
View File

@ -428,11 +428,13 @@ SchemaAlignmentDialog._itemToJSON = function (item) {
**************************/
SchemaAlignmentDialog._addNameDesc = function(item, json) {
var type = 'ALIAS';
var term_type = 'ALIAS';
var value = null;
var override = false;
if (json) {
type = json.name_type;
term_type = json.name_type.replace('_IF_NEW', '');
value = json.value;
override = json.name_type.indexOf('_IF_NEW') == -1;
}
var container = item.find('.wbs-namedesc-container').first();
@ -451,10 +453,7 @@ SchemaAlignmentDialog._addNameDesc = function(item, json) {
.attr('value', 'ALIAS')
.text($.i18n('wikidata-schema/alias'))
.appendTo(type_input);
type_input.val(type);
type_input.on('change', function(e) {
SchemaAlignmentDialog._hasChanged();
});
type_input.val(term_type);
var toolbar = $('<div></div>').addClass('wbs-toolbar').appendTo(namedesc);
SchemaAlignmentDialog._makeDeleteButton().click(function(e) {
@ -467,10 +466,37 @@ SchemaAlignmentDialog._addNameDesc = function(item, json) {
var value_container = $('<div></div>').addClass('wbs-namedesc-value').appendTo(namedesc);
SchemaAlignmentDialog._initField(value_container, "monolingualtext", value);
var override_container = $('<div></div>').addClass('wbs-namedesc-override').appendTo(namedesc);
var label = $('<label></label>').appendTo(override_container);
var checkbox = $('<input></input>')
.attr('type', 'checkbox')
.prop('checked', override)
.appendTo(label);
var span = $('<span></span>').text($.i18n('wikidata-schema/override-term')).appendTo(label);
checkbox.on('change', function(e) {
SchemaAlignmentDialog._hasChanged();
});
type_input.on('change', function(e) {
var checkbox_visible = type_input.val() !== 'ALIAS';
if (checkbox_visible) {
override_container.show();
} else {
override_container.hide();
}
SchemaAlignmentDialog._hasChanged();
});
}
SchemaAlignmentDialog._nameDescToJSON = function (namedesc) {
var type = namedesc.find('select').first().val();
var term_type = namedesc.find('select').first().val();
var type = term_type;
if (term_type !== 'ALIAS') {
var override = namedesc.find('input[type=checkbox]').first().is(':checked');
if (!override) {
type = term_type + '_IF_NEW';
}
}
var value = namedesc.find('.wbs-namedesc-value').first().data("jsonValue");
return {
type: "wbnamedescexpr",

8
extensions/wikidata/module/scripts/previewrenderer.js
View File

@ -37,13 +37,17 @@ EditRenderer._renderItem = function(json, container) {
// Terms
if ((json.labels && json.labels.length) ||
(json.labelsIfNew && json.labelsIfNew.length) ||
(json.descriptions && json.descriptions.length) ||
(json.descriptionsIfNew && json.descriptionsIfNew.length) ||
(json.addedAliases && json.addedAliases.length)) {
var termsContainer = $('<div></div>').addClass('wbs-namedesc-container')
.appendTo(right);
this._renderTermsList(json.labels, "label", termsContainer);
this._renderTermsList(json.descriptions, "description", termsContainer);
this._renderTermsList(json.labels, "label-override", termsContainer);
this._renderTermsList(json.labelsIfNew, "label-if-new", termsContainer);
this._renderTermsList(json.descriptions, "description-override", termsContainer);
this._renderTermsList(json.descriptionsIfNew, "description-if-new", termsContainer);
this._renderTermsList(json.addedAliases, "alias", termsContainer);
// Clear the float

9
extensions/wikidata/module/styles/dialogs/schema-alignment-dialog.css
View File

@ -451,6 +451,15 @@
width: 300px;
}
.wbs-namedesc-override {
display: inline-block;
margin-left: 10px;
}
.wbs-namedesc-override input {
vertical-align: middle;
}
.wbs-language-input {
width: 100%;
}

11
extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java
View File

@ -147,8 +147,15 @@ public class EditBatchProcessor {
} else {
// Existing item
ItemDocument currentDocument = (ItemDocument) currentDocs.get(update.getItemId().getId());
editor.updateTermsStatements(currentDocument, update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
List<MonolingualTextValue> labels = update.getLabels().stream().collect(Collectors.toList());
labels.addAll(update.getLabelsIfNew().stream()
.filter(label -> !currentDocument.getLabels().containsKey(label.getLanguageCode())).collect(Collectors.toList()));
List<MonolingualTextValue> descriptions = update.getDescriptions().stream().collect(Collectors.toList());
descriptions.addAll(update.getDescriptionsIfNew().stream()
.filter(desc -> !currentDocument.getDescriptions().containsKey(desc.getLanguageCode())).collect(Collectors.toList()));
editor.updateTermsStatements(currentDocument,
labels,
descriptions,
update.getAliases().stream().collect(Collectors.toList()),
new ArrayList<MonolingualTextValue>(),
update.getAddedStatements().stream().collect(Collectors.toList()),

5
extensions/wikidata/src/org/openrefine/wikidata/editing/ReconEntityRewriter.java
View File

@ -90,13 +90,16 @@ public class ReconEntityRewriter extends DatamodelConverter {
public ItemUpdate rewrite(ItemUpdate update) {
Set<MonolingualTextValue> labels = update.getLabels().stream().map(l -> copy(l)).collect(Collectors.toSet());
Set<MonolingualTextValue> labelsIfNew = update.getLabelsIfNew().stream().map(l -> copy(l)).collect(Collectors.toSet());
Set<MonolingualTextValue> descriptions = update.getDescriptions().stream().map(l -> copy(l))
.collect(Collectors.toSet());
Set<MonolingualTextValue> descriptionsIfNew = update.getDescriptionsIfNew().stream().map(l -> copy(l))
.collect(Collectors.toSet());
Set<MonolingualTextValue> aliases = update.getAliases().stream().map(l -> copy(l)).collect(Collectors.toSet());
List<Statement> addedStatements = update.getAddedStatements().stream().map(l -> copy(l))
.collect(Collectors.toList());
Set<Statement> deletedStatements = update.getDeletedStatements().stream().map(l -> copy(l))
.collect(Collectors.toSet());
return new ItemUpdate(update.getItemId(), addedStatements, deletedStatements, labels, descriptions, aliases);
return new ItemUpdate(update.getItemId(), addedStatements, deletedStatements, labels, labelsIfNew, descriptions, descriptionsIfNew, aliases);
}
}

2
extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java
View File

@ -131,7 +131,9 @@ public class QuickStatementsExporter implements WriterExporter {
}
translateNameDescr(qid, item.getLabels(), "L", item.getItemId(), writer);
translateNameDescr(qid, item.getLabelsIfNew(), "L", item.getItemId(), writer);
translateNameDescr(qid, item.getDescriptions(), "D", item.getItemId(), writer);
translateNameDescr(qid, item.getDescriptionsIfNew(), "D", item.getItemId(), writer);
translateNameDescr(qid, item.getAliases(), "A", item.getItemId(), writer);
for (Statement s : item.getAddedStatements()) {

4
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java
View File

@ -45,13 +45,13 @@ public class NewItemScrutinizer extends EditScrutinizer {
if (update.isNew()) {
info(newItemType);
if (update.getLabels().isEmpty() && update.getAliases().isEmpty()) {
if (update.getLabels().isEmpty() && update.getLabelsIfNew().isEmpty() && update.getAliases().isEmpty()) {
QAWarning issue = new QAWarning(noLabelType, null, QAWarning.Severity.CRITICAL, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
if (update.getDescriptions().isEmpty()) {
if (update.getDescriptions().isEmpty() && update.getDescriptionsIfNew().isEmpty()) {
QAWarning issue = new QAWarning(noDescType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);

20
extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java
View File

@ -43,15 +43,15 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@JsonIgnoreProperties(ignoreUnknown = true)
public class WbNameDescExpr {
enum NameDescrType {
LABEL, DESCRIPTION, ALIAS,
enum NameDescType {
LABEL, LABEL_IF_NEW, DESCRIPTION, DESCRIPTION_IF_NEW, ALIAS,
}
private NameDescrType type;
private NameDescType type;
private WbMonolingualExpr value;
@JsonCreator
public WbNameDescExpr(@JsonProperty("name_type") NameDescrType type,
public WbNameDescExpr(@JsonProperty("name_type") NameDescType type,
@JsonProperty("value") WbMonolingualExpr value) {
Validate.notNull(type);
this.type = type;
@ -72,11 +72,17 @@ public class WbNameDescExpr {
MonolingualTextValue val = getValue().evaluate(ctxt);
switch (getType()) {
case LABEL:
item.addLabel(val);
item.addLabel(val, true);
break;
case LABEL_IF_NEW:
item.addLabel(val, false);
break;
case DESCRIPTION:
item.addDescription(val);
item.addDescription(val, true);
break;
case DESCRIPTION_IF_NEW:
item.addDescription(val, false);
break;
case ALIAS:
item.addAlias(val);
break;
@ -87,7 +93,7 @@ public class WbNameDescExpr {
}
@JsonProperty("name_type")
public NameDescrType getType() {
public NameDescType getType() {
return type;
}

117
extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdate.java
View File

@ -61,7 +61,9 @@ public class ItemUpdate {
private final List<Statement> addedStatements;
private final Set<Statement> deletedStatements;
private final Map<String, MonolingualTextValue> labels;
private final Map<String, MonolingualTextValue> labelsIfNew;
private final Map<String, MonolingualTextValue> descriptions;
private final Map<String, MonolingualTextValue> descriptionsIfNew;
private final Map<String, List<MonolingualTextValue>> aliases;
/**
@ -76,9 +78,13 @@ public class ItemUpdate {
* @param deletedStatements
* the statements to remove from the item
* @param labels
* the labels to add on the item
* the labels to add on the item, overriding any existing one in that language
* @param labelsIfNew
* the labels to add on the item, only if no label for that language exists
* @param descriptions
* the descriptions to add on the item
* the descriptions to add on the item, overriding any existing one in that language
* @param descriptionsIfNew
* the descriptions to add on the item, only if no description for that language exists
* @param aliases
* the aliases to add on the item. In theory their order should
* matter but in practice people rarely rely on the order of aliases
@ -89,7 +95,9 @@ public class ItemUpdate {
@JsonProperty("addedStatements") List<Statement> addedStatements,
@JsonProperty("deletedStatements") Set<Statement> deletedStatements,
@JsonProperty("labels") Set<MonolingualTextValue> labels,
@JsonProperty("labelsIfNew") Set<MonolingualTextValue> labelsIfNew,
@JsonProperty("descriptions") Set<MonolingualTextValue> descriptions,
@JsonProperty("descriptionsIfNew") Set<MonolingualTextValue> descriptionsIfNew,
@JsonProperty("addedAliases") Set<MonolingualTextValue> aliases) {
Validate.notNull(qid);
this.qid = qid;
@ -101,8 +109,12 @@ public class ItemUpdate {
deletedStatements = Collections.emptySet();
}
this.deletedStatements = deletedStatements;
this.labels = constructTermMap(labels != null ? labels : Collections.emptyList());
this.descriptions = constructTermMap(descriptions != null ? descriptions : Collections.emptyList());
this.labels = new HashMap<>();
this.labelsIfNew = new HashMap<>();
mergeSingleTermMaps(this.labels, this.labelsIfNew, labels, labelsIfNew);
this.descriptions = new HashMap<>();
this.descriptionsIfNew = new HashMap<>();
mergeSingleTermMaps(this.descriptions, this.descriptionsIfNew, descriptions, descriptionsIfNew);
this.aliases = constructTermListMap(aliases != null ? aliases : Collections.emptyList());
}
@ -119,7 +131,13 @@ public class ItemUpdate {
* @param deletedStatements
* the statements to delete
* @param labels
* the labels to add
* the labels to add on the item, overriding any existing one in that language
* @param labelsIfNew
* the labels to add on the item, only if no label for that language exists
* @param descriptions
* the descriptions to add on the item, overriding any existing one in that language
* @param descriptionsIfNew
* the descriptions to add on the item, only if no description for that language exists
* @param descriptions
* the descriptions to add
* @param aliases
@ -130,13 +148,17 @@ public class ItemUpdate {
List<Statement> addedStatements,
Set<Statement> deletedStatements,
Map<String, MonolingualTextValue> labels,
Map<String, MonolingualTextValue> labelsIfNew,
Map<String, MonolingualTextValue> descriptions,
Map<String, MonolingualTextValue> descriptionsIfNew,
Map<String, List<MonolingualTextValue>> aliases) {
this.qid = qid;
this.addedStatements = addedStatements;
this.deletedStatements = deletedStatements;
this.labels = labels;
this.labelsIfNew = labelsIfNew;
this.descriptions = descriptions;
this.descriptionsIfNew = descriptionsIfNew;
this.aliases = aliases;
}
@ -168,20 +190,36 @@ public class ItemUpdate {
}
/**
* @return the list of updated labels
* @return the list of updated labels, overriding existing ones
*/
@JsonProperty("labels")
public Set<MonolingualTextValue> getLabels() {
return labels.values().stream().collect(Collectors.toSet());
}
/**
* @return the list of updated labels, only added if new
*/
@JsonProperty("labelsIfNew")
public Set<MonolingualTextValue> getLabelsIfNew() {
return labelsIfNew.values().stream().collect(Collectors.toSet());
}
/**
* @return the list of updated descriptions
* @return the list of updated descriptions, overriding existing ones
*/
@JsonProperty("descriptions")
public Set<MonolingualTextValue> getDescriptions() {
return descriptions.values().stream().collect(Collectors.toSet());
}
/**
* @return the list of updated descriptions, only added if new
*/
@JsonProperty("descriptionsIfNew")
public Set<MonolingualTextValue> getDescriptionsIfNew() {
return descriptionsIfNew.values().stream().collect(Collectors.toSet());
}
/**
* @return the list of updated aliases
@ -204,8 +242,13 @@ public class ItemUpdate {
*/
@JsonIgnore
public boolean isEmpty() {
return (addedStatements.isEmpty() && deletedStatements.isEmpty() && labels.isEmpty() && descriptions.isEmpty()
&& aliases.isEmpty());
return (addedStatements.isEmpty() &&
deletedStatements.isEmpty() &&
labels.isEmpty() &&
descriptions.isEmpty() &&
aliases.isEmpty() &&
labelsIfNew.isEmpty() &&
descriptionsIfNew.isEmpty());
}
/**
@ -228,13 +271,11 @@ public class ItemUpdate {
Set<Statement> newDeletedStatements = new HashSet<>(deletedStatements);
newDeletedStatements.addAll(other.getDeletedStatements());
Map<String,MonolingualTextValue> newLabels = new HashMap<>(labels);
for(MonolingualTextValue otherLabel : other.getLabels()) {
newLabels.put(otherLabel.getLanguageCode(), otherLabel);
}
Map<String,MonolingualTextValue> newLabelsIfNew = new HashMap<>(labelsIfNew);
mergeSingleTermMaps(newLabels, newLabelsIfNew, other.getLabels(), other.getLabelsIfNew());
Map<String,MonolingualTextValue> newDescriptions = new HashMap<>(descriptions);
for(MonolingualTextValue otherDescription : other.getDescriptions()) {
newDescriptions.put(otherDescription.getLanguageCode(), otherDescription);
}
Map<String,MonolingualTextValue> newDescriptionsIfNew = new HashMap<>(descriptionsIfNew);
mergeSingleTermMaps(newDescriptions, newDescriptionsIfNew, other.getDescriptions(), other.getDescriptionsIfNew());
Map<String,List<MonolingualTextValue>> newAliases = new HashMap<>(aliases);
for(MonolingualTextValue alias : other.getAliases()) {
List<MonolingualTextValue> aliases = newAliases.get(alias.getLanguageCode());
@ -246,8 +287,8 @@ public class ItemUpdate {
aliases.add(alias);
}
}
return new ItemUpdate(qid, newAddedStatements, newDeletedStatements, newLabels, newDescriptions, newAliases);
}
return new ItemUpdate(qid, newAddedStatements, newDeletedStatements, newLabels, newLabelsIfNew, newDescriptions, newDescriptionsIfNew, newAliases);
}
/**
* Group added statements in StatementGroups: useful if the item is new.
@ -309,16 +350,20 @@ public class ItemUpdate {
public ItemUpdate normalizeLabelsAndAliases() {
// Ensure that we are only adding aliases with labels
Set<MonolingualTextValue> filteredAliases = new HashSet<>();
Map<String, MonolingualTextValue> newLabels = new HashMap<>(labels);
Map<String, MonolingualTextValue> newLabels = new HashMap<>(labelsIfNew);
newLabels.putAll(labels);
for (MonolingualTextValue alias : getAliases()) {
if (!labels.containsKey(alias.getLanguageCode())) {
if (!newLabels.containsKey(alias.getLanguageCode())) {
newLabels.put(alias.getLanguageCode(), alias);
} else {
filteredAliases.add(alias);
}
}
Map<String, MonolingualTextValue> newDescriptions = new HashMap<>(descriptionsIfNew);
newDescriptions.putAll(descriptions);
return new ItemUpdate(qid, addedStatements, deletedStatements,
newLabels, descriptions, constructTermListMap(filteredAliases));
newLabels, Collections.emptyMap(), newDescriptions, Collections.emptyMap(),
constructTermListMap(filteredAliases));
}
@Override
@ -372,9 +417,35 @@ public class ItemUpdate {
return builder.toString();
}
protected Map<String,MonolingualTextValue> constructTermMap(Collection<MonolingualTextValue> mltvs) {
return mltvs.stream()
.collect(Collectors.toMap(MonolingualTextValue::getLanguageCode, Function.identity()));
/**
* Helper function to merge dictionaries of terms to override or provide.
* @param currentTerms
* current map of terms to override
* @param currentTermsIfNew
* current map of terms to provide if not already there
* @param newTerms
* new terms to override
* @param newTermsIfNew
* new terms to provide if not already there
*/
private static void mergeSingleTermMaps(
Map<String,MonolingualTextValue> currentTerms,
Map<String,MonolingualTextValue> currentTermsIfNew,
Set<MonolingualTextValue> newTerms,
Set<MonolingualTextValue> newTermsIfNew) {
for(MonolingualTextValue otherLabel : newTerms) {
String languageCode = otherLabel.getLanguageCode();
currentTerms.put(languageCode, otherLabel);
if (currentTermsIfNew.containsKey(languageCode)) {
currentTermsIfNew.remove(languageCode);
}
}
for(MonolingualTextValue otherLabel : newTermsIfNew) {
String languageCode = otherLabel.getLanguageCode();
if (!currentTermsIfNew.containsKey(languageCode) && !currentTerms.containsKey(languageCode)) {
currentTermsIfNew.put(languageCode, otherLabel);
}
}
}
protected Map<String, List<MonolingualTextValue>> constructTermListMap(Collection<MonolingualTextValue> mltvs) {

58
extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdateBuilder.java
View File

@ -45,7 +45,9 @@ public class ItemUpdateBuilder {
private List<Statement> addedStatements;
private Set<Statement> deletedStatements;
private Set<MonolingualTextValue> labels;
private Set<MonolingualTextValue> labelsIfNew;
private Set<MonolingualTextValue> descriptions;
private Set<MonolingualTextValue> descriptionsIfNew;
private Set<MonolingualTextValue> aliases;
private boolean built;
@ -62,7 +64,9 @@ public class ItemUpdateBuilder {
this.addedStatements = new ArrayList<>();
this.deletedStatements = new HashSet<Statement>();
this.labels = new HashSet<MonolingualTextValue>();
this.labelsIfNew = new HashSet<MonolingualTextValue>();
this.descriptions = new HashSet<MonolingualTextValue>();
this.descriptionsIfNew = new HashSet<MonolingualTextValue>();
this.aliases = new HashSet<MonolingualTextValue>();
this.built = false;
}
@ -118,54 +122,74 @@ public class ItemUpdateBuilder {
}
/**
* Adds a label to the item. It will override any existing label in this
* language.
* Adds a label to the item.
*
* @param label
* the label to add
* @param override
* whether the label should be added even if there is already a label in that language
*/
public ItemUpdateBuilder addLabel(MonolingualTextValue label) {
public ItemUpdateBuilder addLabel(MonolingualTextValue label, boolean override) {
Validate.isTrue(!built, "ItemUpdate has already been built");
labels.add(label);
if (override) {
labels.add(label);
} else {
labelsIfNew.add(label);
}
return this;
}
/**
* Adds a list of labels to the item. It will override any existing label in
* each language.
* Adds a list of labels to the item.
*
* @param labels
* the labels to add
* @param override
* whether the label should be added even if there is already a label in that language
*/
public ItemUpdateBuilder addLabels(Set<MonolingualTextValue> labels) {
public ItemUpdateBuilder addLabels(Set<MonolingualTextValue> labels, boolean override) {
Validate.isTrue(!built, "ItemUpdate has already been built");
this.labels.addAll(labels);
if (override) {
this.labels.addAll(labels);
} else {
labelsIfNew.addAll(labels);
}
return this;
}
/**
* Adds a description to the item. It will override any existing description in
* this language.
* Adds a description to the item.
*
* @param description
* the description to add
* @param override
* whether the description should be added even if there is already a description in that language
*/
public ItemUpdateBuilder addDescription(MonolingualTextValue description) {
public ItemUpdateBuilder addDescription(MonolingualTextValue description, boolean override) {
Validate.isTrue(!built, "ItemUpdate has already been built");
descriptions.add(description);
if (override) {
descriptions.add(description);
} else {
descriptionsIfNew.add(description);
}
return this;
}
/**
* Adds a list of descriptions to the item. It will override any existing
* description in each language.
* Adds a list of descriptions to the item.
*
* @param descriptions
* the descriptions to add
* @param override
* whether the description should be added even if there is already a description in that language
*/
public ItemUpdateBuilder addDescriptions(Set<MonolingualTextValue> descriptions) {
public ItemUpdateBuilder addDescriptions(Set<MonolingualTextValue> descriptions, boolean override) {
Validate.isTrue(!built, "ItemUpdate has already been built");
this.descriptions.addAll(descriptions);
if (override) {
this.descriptions.addAll(descriptions);
} else {
descriptionsIfNew.addAll(descriptions);
}
return this;
}
@ -202,7 +226,7 @@ public class ItemUpdateBuilder {
*/
public ItemUpdate build() {
built = true;
return new ItemUpdate(qid, addedStatements, deletedStatements, labels, descriptions, aliases);
return new ItemUpdate(qid, addedStatements, deletedStatements, labels, labelsIfNew, descriptions, descriptionsIfNew, aliases);
}
}

6
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/QuickStatementsUpdateScheduler.java
View File

@ -65,7 +65,11 @@ public class QuickStatementsUpdateScheduler implements UpdateScheduler {
protected void splitUpdate(ItemUpdate update)
throws ImpossibleSchedulingException {
ItemUpdateBuilder remainingUpdateBuilder = new ItemUpdateBuilder(update.getItemId())
.addLabels(update.getLabels()).addDescriptions(update.getDescriptions()).addAliases(update.getAliases())
.addLabels(update.getLabels(), true)
.addLabels(update.getLabelsIfNew(), false)
.addDescriptions(update.getDescriptions(), true)
.addDescriptions(update.getDescriptionsIfNew(), false)
.addAliases(update.getAliases())
.deleteStatements(update.getDeletedStatements());
Map<ItemIdValue, ItemUpdateBuilder> referencingUpdates = new HashMap<>();

8
extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/WikibaseAPIUpdateScheduler.java
View File

@ -98,8 +98,12 @@ public class WikibaseAPIUpdateScheduler implements UpdateScheduler {
* @param update
*/
protected void splitUpdate(ItemUpdate update) {
ItemUpdateBuilder pointerFreeBuilder = new ItemUpdateBuilder(update.getItemId()).addLabels(update.getLabels())
.addDescriptions(update.getDescriptions()).addAliases(update.getAliases())
ItemUpdateBuilder pointerFreeBuilder = new ItemUpdateBuilder(update.getItemId())
.addLabels(update.getLabels(), true)
.addLabels(update.getLabelsIfNew(), false)
.addDescriptions(update.getDescriptions(), true)
.addDescriptions(update.getDescriptionsIfNew(), false)
.addAliases(update.getAliases())
.deleteStatements(update.getDeletedStatements());
ItemUpdateBuilder pointerFullBuilder = new ItemUpdateBuilder(update.getItemId());

2
extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java
View File

@ -120,7 +120,7 @@ public class EditBatchProcessorTest extends RefineTest {
List<ItemIdValue> qids = ids.stream().map(e -> Datamodel.makeWikidataItemIdValue(e))
.collect(Collectors.toList());
List<ItemUpdate> batch = qids.stream()
.map(qid -> new ItemUpdateBuilder(qid).addDescription(description).build())
.map(qid -> new ItemUpdateBuilder(qid).addDescription(description, true).build())
.collect(Collectors.toList());
int batchSize = 50;

8
extensions/wikidata/tests/src/org/openrefine/wikidata/editing/ReconEntityRewriterTest.java
View File

@ -73,15 +73,15 @@ public class ReconEntityRewriterTest {
ItemUpdate update = new ItemUpdateBuilder(subject)
.addStatement(TestingData.generateStatement(subject, TestingData.newIdB))
.deleteStatement(TestingData.generateStatement(subject, TestingData.existingId))
.addLabel(Datamodel.makeMonolingualTextValue("label", "de"))
.addDescription(Datamodel.makeMonolingualTextValue("beschreibung", "de"))
.addLabel(Datamodel.makeMonolingualTextValue("label", "de"), true)
.addDescription(Datamodel.makeMonolingualTextValue("beschreibung", "de"), false)
.addAlias(Datamodel.makeMonolingualTextValue("darstellung", "de")).build();
ItemUpdate rewritten = rewriter.rewrite(update);
ItemUpdate expected = new ItemUpdateBuilder(subject)
.addStatement(TestingData.generateStatement(subject, newlyCreated))
.deleteStatement(TestingData.generateStatement(subject, TestingData.existingId))
.addLabel(Datamodel.makeMonolingualTextValue("label", "de"))
.addDescription(Datamodel.makeMonolingualTextValue("beschreibung", "de"))
.addLabel(Datamodel.makeMonolingualTextValue("label", "de"), true)
.addDescription(Datamodel.makeMonolingualTextValue("beschreibung", "de"), false)
.addAlias(Datamodel.makeMonolingualTextValue("darstellung", "de")).build();
assertEquals(expected, rewritten);
}

19
extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java
View File

@ -91,9 +91,24 @@ public class QuickStatementsExporterTest extends RefineTest {
@Test
public void testNameDesc()
throws IOException {
/**
* Adding labels and description without overriding is not supported by QS, so
* we fall back on adding them with overriding.
*/
ItemUpdate update = new ItemUpdateBuilder(qid1)
.addLabel(Datamodel.makeMonolingualTextValue("some label", "en"), true)
.addDescription(Datamodel.makeMonolingualTextValue("some description", "en"), true)
.build();
assertEquals("Q1377\tLen\t\"some label\"\n" + "Q1377\tDen\t\"some description\"\n", export(update));
}
@Test
public void testOptionalNameDesc()
throws IOException {
ItemUpdate update = new ItemUpdateBuilder(newIdA)
.addLabel(Datamodel.makeMonolingualTextValue("my new item", "en"))
.addDescription(Datamodel.makeMonolingualTextValue("isn't it awesome?", "en"))
.addLabel(Datamodel.makeMonolingualTextValue("my new item", "en"), false)
.addDescription(Datamodel.makeMonolingualTextValue("isn't it awesome?", "en"), false)
.addAlias(Datamodel.makeMonolingualTextValue("fabitem", "en")).build();
assertEquals("CREATE\n" + "LAST\tLen\t\"my new item\"\n" + "LAST\tDen\t\"isn't it awesome?\"\n"

8
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizerTest.java
View File

@ -65,8 +65,8 @@ public class NewItemScrutinizerTest extends ScrutinizerTest {
public void testGoodNewItem() {
ItemUpdate update = new ItemUpdateBuilder(TestingData.newIdA)
.addLabel(Datamodel.makeMonolingualTextValue("bonjour", "fr"))
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en")).addStatement(p31Statement)
.addLabel(Datamodel.makeMonolingualTextValue("bonjour", "fr"), false)
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en"), true).addStatement(p31Statement)
.build();
scrutinize(update);
assertWarningsRaised(NewItemScrutinizer.newItemType);
@ -75,8 +75,8 @@ public class NewItemScrutinizerTest extends ScrutinizerTest {
@Test
public void testDeletedStatements() {
ItemUpdate update = new ItemUpdateBuilder(TestingData.newIdA)
.addLabel(Datamodel.makeMonolingualTextValue("bonjour", "fr"))
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en")).addStatement(p31Statement)
.addLabel(Datamodel.makeMonolingualTextValue("bonjour", "fr"), false)
.addDescription(Datamodel.makeMonolingualTextValue("interesting item", "en"), true).addStatement(p31Statement)
.deleteStatement(TestingData.generateStatement(TestingData.newIdA, TestingData.matchedId)).build();
scrutinize(update);
assertWarningsRaised(NewItemScrutinizer.newItemType, NewItemScrutinizer.deletedStatementsType);

2
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizerTest.java
View File

@ -43,6 +43,6 @@ public abstract class ValueScrutinizerTest extends SnakScrutinizerTest {
}
public void scrutinizeLabel(MonolingualTextValue text) {
scrutinize(new ItemUpdateBuilder(TestingData.existingId).addLabel(text).build());
scrutinize(new ItemUpdateBuilder(TestingData.existingId).addLabel(text, true).build());
}
}

2
extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemDocumentExprTest.java
View File

@ -45,7 +45,7 @@ public class WbItemDocumentExprTest extends WbExpressionTest<ItemUpdate> {
public WbItemDocumentExprTest() {
WbStatementGroupExprTest sgt = new WbStatementGroupExprTest();
WbNameDescExpr nde = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS,
WbNameDescExpr nde = new WbNameDescExpr(WbNameDescExpr.NameDescType.ALIAS,
new WbMonolingualExpr(new WbLanguageConstant("en", "English"), new WbStringVariable("column D")));
WbItemVariable subjectExpr = new WbItemVariable("column E");
expr = new WbItemDocumentExpr(subjectExpr, Collections.singletonList(nde), Collections.singletonList(sgt.expr));

12
extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbNameDescExprTest.java
View File

@ -38,7 +38,7 @@ import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
public class WbNameDescExprTest extends WbExpressionTest<MonolingualTextValue> {
private ItemIdValue subject = Datamodel.makeWikidataItemIdValue("Q56");
public WbNameDescExpr expr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS,
public WbNameDescExpr expr = new WbNameDescExpr(WbNameDescExpr.NameDescType.ALIAS,
new WbMonolingualExpr(new WbLanguageConstant("en", "English"), new WbStringVariable("column A")));
public String jsonRepresentation = "{\"name_type\":\"ALIAS\",\"value\":{\"type\":\"wbmonolingualexpr\",\"language\":"
@ -47,7 +47,7 @@ public class WbNameDescExprTest extends WbExpressionTest<MonolingualTextValue> {
@Test
public void testContributeToLabel() {
WbNameDescExpr labelExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.LABEL,
WbNameDescExpr labelExpr = new WbNameDescExpr(WbNameDescExpr.NameDescType.LABEL,
TestingData.getTestMonolingualExpr("fr", "français", "le croissant magnifique"));
ItemUpdateBuilder update = new ItemUpdateBuilder(subject);
labelExpr.contributeTo(update, ctxt);
@ -57,7 +57,7 @@ public class WbNameDescExprTest extends WbExpressionTest<MonolingualTextValue> {
@Test
public void testContributeToDescription() {
WbNameDescExpr descriptionExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.DESCRIPTION,
WbNameDescExpr descriptionExpr = new WbNameDescExpr(WbNameDescExpr.NameDescType.DESCRIPTION,
TestingData.getTestMonolingualExpr("de", "Deutsch", "wunderschön"));
ItemUpdateBuilder update = new ItemUpdateBuilder(subject);
descriptionExpr.contributeTo(update, ctxt);
@ -67,7 +67,7 @@ public class WbNameDescExprTest extends WbExpressionTest<MonolingualTextValue> {
@Test
public void testContributeToAlias() {
WbNameDescExpr aliasExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS,
WbNameDescExpr aliasExpr = new WbNameDescExpr(WbNameDescExpr.NameDescType.ALIAS,
TestingData.getTestMonolingualExpr("en", "English", "snack"));
ItemUpdateBuilder update = new ItemUpdateBuilder(subject);
aliasExpr.contributeTo(update, ctxt);
@ -86,8 +86,8 @@ public class WbNameDescExprTest extends WbExpressionTest<MonolingualTextValue> {
@Test
public void testGetters() {
WbMonolingualExpr monolingualExpr = TestingData.getTestMonolingualExpr("en", "English", "not sure what");
WbNameDescExpr aliasExpr = new WbNameDescExpr(WbNameDescExpr.NameDescrType.ALIAS, monolingualExpr);
assertEquals(WbNameDescExpr.NameDescrType.ALIAS, aliasExpr.getType());
WbNameDescExpr aliasExpr = new WbNameDescExpr(WbNameDescExpr.NameDescType.ALIAS, monolingualExpr);
assertEquals(WbNameDescExpr.NameDescType.ALIAS, aliasExpr.getType());
assertEquals(monolingualExpr, aliasExpr.getValue());
}

91
extensions/wikidata/tests/src/org/openrefine/wikidata/updates/ItemUpdateTest.java
View File

@ -132,7 +132,7 @@ public class ItemUpdateTest {
public void testGroupBySubject() {
ItemUpdate updateA = new ItemUpdateBuilder(newSubject).addStatement(statement1).build();
ItemUpdate updateB = new ItemUpdateBuilder(sameNewSubject).addStatement(statement2).build();
ItemUpdate updateC = new ItemUpdateBuilder(existingSubject).addLabel(label).build();
ItemUpdate updateC = new ItemUpdateBuilder(existingSubject).addLabel(label, true).build();
ItemUpdate updateD = new ItemUpdateBuilder(matchedSubject).build();
Map<EntityIdValue, ItemUpdate> grouped = ItemUpdate
.groupBySubject(Arrays.asList(updateA, updateB, updateC, updateD));
@ -148,12 +148,12 @@ public class ItemUpdateTest {
public void testNormalizeTerms() {
MonolingualTextValue aliasEn = Datamodel.makeMonolingualTextValue("alias", "en");
MonolingualTextValue aliasFr = Datamodel.makeMonolingualTextValue("coucou", "fr");
ItemUpdate updateA = new ItemUpdateBuilder(newSubject).addLabel(label).addAlias(aliasEn).addAlias(aliasFr)
ItemUpdate updateA = new ItemUpdateBuilder(newSubject).addLabel(label, true).addAlias(aliasEn).addAlias(aliasFr)
.build();
assertFalse(updateA.isNull());
ItemUpdate normalized = updateA.normalizeLabelsAndAliases();
ItemUpdate expectedUpdate = new ItemUpdateBuilder(newSubject).addLabel(label).addAlias(aliasEn)
.addLabel(aliasFr).build();
ItemUpdate expectedUpdate = new ItemUpdateBuilder(newSubject).addLabel(label, true).addAlias(aliasEn)
.addLabel(aliasFr, true).build();
assertEquals(expectedUpdate, normalized);
}
@ -161,9 +161,88 @@ public class ItemUpdateTest {
public void testMergeLabels() {
MonolingualTextValue label1 = Datamodel.makeMonolingualTextValue("first label", "en");
MonolingualTextValue label2 = Datamodel.makeMonolingualTextValue("second label", "en");
ItemUpdate update1 = new ItemUpdateBuilder(existingSubject).addLabel(label1).build();
ItemUpdate update2 = new ItemUpdateBuilder(existingSubject).addLabel(label2).build();
ItemUpdate update1 = new ItemUpdateBuilder(existingSubject).addLabel(label1, true).build();
ItemUpdate update2 = new ItemUpdateBuilder(existingSubject).addLabel(label2, true).build();
ItemUpdate merged = update1.merge(update2);
assertEquals(Collections.singleton(label2), merged.getLabels());
}
@Test
public void testMergeLabelsIfNew() {
MonolingualTextValue label1 = Datamodel.makeMonolingualTextValue("first label", "en");
MonolingualTextValue label2 = Datamodel.makeMonolingualTextValue("second label", "en");
ItemUpdate update1 = new ItemUpdateBuilder(existingSubject).addLabel(label1, false).build();
ItemUpdate update2 = new ItemUpdateBuilder(existingSubject).addLabel(label2, false).build();
ItemUpdate merged = update1.merge(update2);
assertEquals(Collections.singleton(label1), merged.getLabelsIfNew());
assertEquals(Collections.emptySet(), merged.getLabels());
}
@Test
public void testMergeLabelsIfNewOverriding() {
MonolingualTextValue label1 = Datamodel.makeMonolingualTextValue("first label", "en");
MonolingualTextValue label2 = Datamodel.makeMonolingualTextValue("second label", "en");
ItemUpdate update1 = new ItemUpdateBuilder(existingSubject).addLabel(label1, true).build();
ItemUpdate update2 = new ItemUpdateBuilder(existingSubject).addLabel(label2, false).build();
ItemUpdate merged = update1.merge(update2);
assertEquals(Collections.singleton(label1), merged.getLabels());
assertEquals(Collections.emptySet(), merged.getLabelsIfNew());
}
@Test
public void testMergeLabelsIfNewOverriding2() {
MonolingualTextValue label1 = Datamodel.makeMonolingualTextValue("first label", "en");
MonolingualTextValue label2 = Datamodel.makeMonolingualTextValue("second label", "en");
ItemUpdate update1 = new ItemUpdateBuilder(existingSubject).addLabel(label1, false).build();
ItemUpdate update2 = new ItemUpdateBuilder(existingSubject).addLabel(label2, true).build();
ItemUpdate merged = update1.merge(update2);
assertEquals(Collections.singleton(label2), merged.getLabels());
assertEquals(Collections.emptySet(), merged.getLabelsIfNew());
}
@Test
public void testMergeDescriptionsIfNew() {
MonolingualTextValue description1 = Datamodel.makeMonolingualTextValue("first description", "en");
MonolingualTextValue description2 = Datamodel.makeMonolingualTextValue("second description", "en");
ItemUpdate update1 = new ItemUpdateBuilder(existingSubject).addDescription(description1, false).build();
ItemUpdate update2 = new ItemUpdateBuilder(existingSubject).addDescription(description2, false).build();
ItemUpdate merged = update1.merge(update2);
assertEquals(Collections.singleton(description1), merged.getDescriptionsIfNew());
assertEquals(Collections.emptySet(), merged.getDescriptions());
assertFalse(merged.isEmpty());
}
@Test
public void testMergeDescriptionsIfNewOverriding() {
MonolingualTextValue description1 = Datamodel.makeMonolingualTextValue("first description", "en");
MonolingualTextValue description2 = Datamodel.makeMonolingualTextValue("second description", "en");
ItemUpdate update1 = new ItemUpdateBuilder(existingSubject).addDescription(description1, true).build();
ItemUpdate update2 = new ItemUpdateBuilder(existingSubject).addDescription(description2, false).build();
ItemUpdate merged = update1.merge(update2);
assertEquals(Collections.singleton(description1), merged.getDescriptions());
assertEquals(Collections.emptySet(), merged.getDescriptionsIfNew());
}
@Test
public void testMergeDescriptionsIfNewOverriding2() {
MonolingualTextValue description1 = Datamodel.makeMonolingualTextValue("first description", "en");
MonolingualTextValue description2 = Datamodel.makeMonolingualTextValue("second description", "en");
ItemUpdate update1 = new ItemUpdateBuilder(existingSubject).addDescription(description1, false).build();
ItemUpdate update2 = new ItemUpdateBuilder(existingSubject).addDescription(description2, true).build();
ItemUpdate merged = update1.merge(update2);
assertEquals(Collections.singleton(description2), merged.getDescriptions());
assertEquals(Collections.emptySet(), merged.getDescriptionsIfNew());
}
@Test
public void testConstructOverridingLabels() {
MonolingualTextValue label1 = Datamodel.makeMonolingualTextValue("first label", "en");
MonolingualTextValue label2 = Datamodel.makeMonolingualTextValue("second label", "en");
ItemUpdate update = new ItemUpdateBuilder(existingSubject)
.addLabel(label1, false)
.addLabel(label2, true)
.build();
assertEquals(Collections.singleton(label2), update.getLabels());
assertEquals(Collections.emptySet(), update.getLabelsIfNew());
}
}

6
extensions/wikidata/tests/src/org/openrefine/wikidata/updates/scheduler/UpdateSchedulerTest.java
View File

@ -87,7 +87,7 @@ public abstract class UpdateSchedulerTest {
throws ImpossibleSchedulingException {
ItemUpdate update1 = new ItemUpdateBuilder(existingIdA).addStatement(sAtoB).build();
ItemUpdate update2 = new ItemUpdateBuilder(existingIdA)
.addLabel(Datamodel.makeMonolingualTextValue("hello", "fr")).addStatement(sAtoB).build();
.addLabel(Datamodel.makeMonolingualTextValue("hello", "fr"), true).addStatement(sAtoB).build();
ItemUpdate merged = update1.merge(update2);
assertEquals(Collections.singletonList(merged), schedule(update1, update2));
}
@ -95,9 +95,9 @@ public abstract class UpdateSchedulerTest {
@Test
public void testMergeNew()
throws ImpossibleSchedulingException {
ItemUpdate update1 = new ItemUpdateBuilder(newIdA).addLabel(Datamodel.makeMonolingualTextValue("hello", "fr"))
ItemUpdate update1 = new ItemUpdateBuilder(newIdA).addLabel(Datamodel.makeMonolingualTextValue("hello", "fr"), true)
.addStatement(sNewAtoB).build();
ItemUpdate update2 = new ItemUpdateBuilder(newIdA).addLabel(Datamodel.makeMonolingualTextValue("hello", "fr"))
ItemUpdate update2 = new ItemUpdateBuilder(newIdA).addLabel(Datamodel.makeMonolingualTextValue("hello", "fr"), true)
.build();
ItemUpdate merged = update1.merge(update2);
assertEquals(Collections.singletonList(merged), schedule(update1, update2));