s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update EntityCache class (#2991)

Browse Source 
* updated EntityCache class

* added overridden methods in CacheLoader and updated test file
This commit is contained in:
Ekta Mishra 2020-07-29 00:33:22 +05:30 committed by GitHub
parent 1dcc83209c
commit fb6c309da8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 86 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
extensions/wikidata/src/org/openrefine/wikidata/utils/EntityCache.java
View File

@ -23,8 +23,9 @@
******************************************************************************/
package org.openrefine.wikidata.utils;
import java.util.concurrent.TimeUnit;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
@ -33,9 +34,12 @@ import org.wikidata.wdtk.wikibaseapi.BasicApiConnection;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
public class EntityCache {
@ -54,6 +58,7 @@ public class EntityCache {
_cache = CacheBuilder.newBuilder().maximumSize(4096).expireAfterWrite(1, TimeUnit.HOURS)
.build(new CacheLoader<String, EntityDocument>() {
@Override
public EntityDocument load(String entityId)
throws Exception {
EntityDocument doc = _fetcher.getEntityDocument(entityId);
@ -63,6 +68,19 @@ public class EntityCache {
throw new MediaWikiApiErrorException("400", "Unknown entity id \"" + entityId + "\"");
}
}
@Override
public Map<String, EntityDocument> loadAll(Iterable<? extends String> entityIds)
throws Exception {
Map<String, EntityDocument> entityDocumentMap = _fetcher.getEntityDocuments(StreamSupport.stream(entityIds.spliterator(), false)
.collect(Collectors.toList()));
if (!entityDocumentMap.isEmpty()) {
return entityDocumentMap;
} else {
throw new MediaWikiApiErrorException("400", "Unknown entity ids in \"" + entityIds.toString() + "\"");
}
}
});
}
@ -77,6 +95,11 @@ public class EntityCache {
return _entityCache;
}
public List<EntityDocument> getMultipleDocuments(List<EntityIdValue> entityIds) throws ExecutionException {
List<String> ids = entityIds.stream().map(entityId -> entityId.getId()).collect(Collectors.toList());
return _cache.getAll(ids).values().stream().collect(Collectors.toList());
}
public static EntityDocument getEntityDocument(EntityIdValue id) {
return getEntityCache().get(id);
}

65
extensions/wikidata/tests/src/org/openrefine/wikidata/utils/EntityCacheTests.java
View File

@ -1,21 +1,27 @@
package org.openrefine.wikidata.utils;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.io.IOException;
import org.testng.Assert;
import org.testng.annotations.Test;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class EntityCacheTests {
@Test
@ -33,4 +39,49 @@ public class EntityCacheTests {
// the fetcher was only called once thanks to caching
verify(fetcher, times(1)).getEntityDocument(id.getId());
}
@Test
public void testGetAll() throws MediaWikiApiErrorException, IOException, ExecutionException {
WikibaseDataFetcher fetcher = mock(WikibaseDataFetcher.class);
PropertyIdValue idA = Datamodel.makeWikidataPropertyIdValue("P42");
PropertyIdValue idB = Datamodel.makeWikidataPropertyIdValue("P43");
PropertyIdValue idC = Datamodel.makeWikidataPropertyIdValue("P44");
PropertyIdValue idD = Datamodel.makeWikidataPropertyIdValue("P45");
PropertyDocument docA = Datamodel.makePropertyDocument(idA, Datamodel.makeDatatypeIdValue(DatatypeIdValue.DT_GEO_SHAPE));
PropertyDocument docB = Datamodel.makePropertyDocument(idB, Datamodel.makeDatatypeIdValue(DatatypeIdValue.DT_GEO_SHAPE));
PropertyDocument docC = Datamodel.makePropertyDocument(idC, Datamodel.makeDatatypeIdValue(DatatypeIdValue.DT_GEO_SHAPE));
PropertyDocument docD = Datamodel.makePropertyDocument(idD, Datamodel.makeDatatypeIdValue(DatatypeIdValue.DT_GEO_SHAPE));
EntityCache SUT = new EntityCache(fetcher);
List<String> entityIdListA = Arrays.asList(idA.getId(), idB.getId());
List<String> entityIdListB = Arrays.asList(idC.getId(), idD.getId());
List<String> entityIdListC = Arrays.asList(idB.getId(), idC.getId());
List<EntityDocument> docListA = Arrays.asList(docA, docB);
List<EntityDocument> docListB = Arrays.asList(docC, docD);
List<EntityDocument> docListC = Arrays.asList(docB, docC);
Map<String, EntityDocument> docMapA = new HashMap<>();
docMapA.put(idA.getId(), docA);
docMapA.put(idB.getId(), docB);
Map<String, EntityDocument> docMapB = new HashMap<>();
docMapB.put(idC.getId(), docC);
docMapB.put(idD.getId(), docD);
Map<String, EntityDocument> docMapC = new HashMap<>();
docMapC.put(idB.getId(), docB);
docMapC.put(idC.getId(), docC);
when(fetcher.getEntityDocuments(entityIdListA)).thenReturn(docMapA);
when(fetcher.getEntityDocuments(entityIdListB)).thenReturn(docMapB);
when(fetcher.getEntityDocuments(entityIdListC)).thenReturn(docMapC);
Assert.assertEquals(SUT.getMultipleDocuments(Arrays.asList(idA, idB)), docListA);
Assert.assertEquals(SUT.getMultipleDocuments(Arrays.asList(idC, idD)), docListB);
Assert.assertEquals(SUT.getMultipleDocuments(Arrays.asList(idB, idC)), docListC);
verify(fetcher, times(0)).getEntityDocuments(entityIdListC);
}
}