s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1530 from OpenRefine/wikidata-extension

Browse Source 
Wikidata extension
This commit is contained in:
Antonin Delpeuch 2018-05-06 08:46:53 +02:00 committed by GitHub
commit c0e39f507a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
191 changed files with 17554 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.classpath
View File

@ -4,13 +4,15 @@
<classpathentry kind="src" path="main/resources"/>
<classpathentry kind="src" path="extensions/jython/tests/src"/>
<classpathentry kind="src" path="server/src"/>
<classpathentry kind="src" path="main/tests/server/src"/>
<classpathentry kind="src" path="extensions/gdata/src"/>
<classpathentry kind="src" path="extensions/database/src"/>
<classpathentry kind="src" path="extensions/database/test"/>
<classpathentry kind="src" path="extensions/jython/src"/>
<classpathentry kind="src" path="extensions/pc-axis/src"/>
<classpathentry kind="src" path="extensions/sample/src"/>
<classpathentry kind="src" path="main/tests/server/src"/>
<classpathentry kind="src" path="extensions/wikidata/src"/>
<classpathentry kind="src" path="extensions/wikidata/tests/src"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/ant-tools-1.8.0.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/arithcode-1.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/butterfly-1.0.1.jar" sourcepath="main/webapp/WEB-INF/lib-src/butterfly-1.0.1-sources.jar"/>
@ -127,5 +129,7 @@
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jena-tdb2-3.6.0.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jsonld-java-0.11.1.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/libthrift-0.10.0.jar"/>
<classpathentry kind="lib" path="extensions/wikidata/module/MOD-INF/lib/wdtk-datamodel-0.9.0-SNAPSHOT-jar-with-dependencies.jar" sourcepath="extensions/wikidata/module/MOD-INF/lib-src/wdtk-datamodel-0.9.0-SNAPSHOT-sources.jar"/>
<classpathentry kind="lib" path="extensions/wikidata/module/MOD-INF/lib/wdtk-wikibaseapi-0.9.0-SNAPSHOT.jar" sourcepath="extensions/wikidata/module/MOD-INF/lib-src/wdtk-wikibaseapi-0.9.0-SNAPSHOT-sources.jar"/>
<classpathentry kind="output" path="main/webapp/WEB-INF/classes"/>
</classpath>

9
extensions/build.xml
View File

@ -13,7 +13,8 @@
<ant dir="jython/" target="build" />
<ant dir="gdata/" target="build" />
<ant dir="pc-axis/" target="build" />
<ant dir="database/" target="build" />
<ant dir="database/" target="build" />
<ant dir="wikidata/" target="build" />
</target>
<target name="clean">
@ -22,12 +23,14 @@
<ant dir="jython/" target="clean" />
<ant dir="gdata/" target="clean" />
<ant dir="pc-axis/" target="clean" />
<ant dir="database/" target="clean" />
<ant dir="database/" target="clean" />
<ant dir="wikidata/" target="clean" />
</target>
<target name="test">
<echo message="Testing extensions" />
<ant dir="jython/" target="test" />
<ant dir="database/" target="test" />
<ant dir="database/" target="test" />
<ant dir="wikidata/" target="test" />
</target>
</project>

24
extensions/wikidata/.classpath Normal file
View File

@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
<classpathentry kind="src" path="resources"/>
<classpathentry kind="lib" path="/OpenRefine/server/lib/servlet-api-2.5.jar" sourcepath="/OpenRefine/server/lib-src/servlet-api-2.5-sources.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/butterfly-1.0.1.jar" sourcepath="/OpenRefine/main/webapp/WEB-INF/lib-src/butterfly-trunk.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/tests/server/lib/testng-6.8.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/slf4j-api-1.5.6.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/log4j-1.2.15.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/slf4j-log4j12-1.5.6.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/velocity-1.5.jar"/>
<classpathentry kind="lib" path="/OpenRefine/server/lib/jetty-6.1.22.jar"/>
<classpathentry kind="lib" path="/OpenRefine/server/lib/jetty-util-6.1.22.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/tests/server/lib/mockito-all-1.9.5.jar"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6">
<attributes>
<attribute name="owner.project.facets" value="java"/>
</attributes>
</classpathentry>
<classpathentry combineaccessrules="false" kind="src" path="/OpenRefine"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/wdtk-datamodel-0.8.0-SNAPSHOT-jar-with-dependencies.jar" sourcepath="module/MOD-INF/lib-src/wdtk-datamodel-0.8.0-SNAPSHOT-sources.jar"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/wdtk-wikibaseapi-0.8.0-SNAPSHOT.jar" sourcepath="module/MOD-INF/lib-src/wdtk-wikibaseapi-0.8.0-SNAPSHOT-sources.jar"/>
<classpathentry kind="output" path="module/MOD-INF/classes"/>
</classpath>

3
extensions/wikidata/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
tests/classes/
tests/report/
tests/test-output/

133
extensions/wikidata/build.xml Normal file
View File

@ -0,0 +1,133 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Google Refine Wikidata Extension Build File +-->
<project name="refine-wikidata-extension" default="build" basedir="." xmlns:jacoco="antlib:org.jacoco.ant">
<property environment="env"/>
<condition property="version" value="trunk">
<not><isset property="version"/></not>
</condition>
<condition property="revision" value="rXXXX">
<not><isset property="revision"/></not>
</condition>
<condition property="full_version" value="0.0.0.0">
<not><isset property="full_version"/></not>
</condition>
<condition property="dist.dir" value="dist">
<not><isset property="dist.dir"/></not>
</condition>
<property name="fullname" value="${name}-${version}-${revision}" />
<property name="java_version" value="1.8"/>
<property name="refine.dir" value="${basedir}/../../main" />
<property name="refine.webinf.dir" value="${refine.dir}/webapp/WEB-INF" />
<property name="refine.modinf.dir" value="${refine.dir}/webapp/modules/core/MOD-INF" />
<property name="refine.classes.dir" value="${refine.webinf.dir}/classes" />
<property name="refine.lib.dir" value="${refine.webinf.dir}/lib" />
<property name="server.dir" value="${basedir}/../../server" />
<property name="server.lib.dir" value="${server.dir}/lib" />
<property name="refine.tests.lib.dir" value="${refine.dir}/tests/server/lib" />
<property name="refine.tests.classes.dir" value="${refine.dir}/tests/server/classes" />
<property name="src.dir" value="${basedir}/src" />
<property name="module.dir" value="${basedir}/module" />
<property name="modinf.dir" value="${module.dir}/MOD-INF" />
<property name="lib.dir" value="${modinf.dir}/lib" />
<property name="classes.dir" value="${modinf.dir}/classes" />
<property name="extension.tests.dir" value="${basedir}/tests" />
<property name="tests.build.dir" value="${extension.tests.dir}/build" />
<property name="tests.src.dir" value="${extension.tests.dir}/src" />
<property name="tests.classes.dir" value="${extension.tests.dir}/classes" />
<property name="tests.report.dir" value="${extension.tests.dir}/report"/>
<property name="tests.report.html.dir" value="${tests.report.dir}/html"/>
<property name="tests.report.xml.path" value="${tests.report.dir}/jacoco.xml"/>
<path id="class.path">
<fileset dir="${lib.dir}">
<include name="**/*.jar" />
</fileset>
<fileset dir="${refine.lib.dir}">
<include name="**/*.jar" />
</fileset>
<fileset dir="${server.lib.dir}">
<include name="**/*.jar" />
</fileset>
<pathelement path="${refine.classes.dir}"/>
<pathelement path="${classes.dir}" />
</path>
<path id="extension.tests.class.path">
<path refid="class.path"/>
<pathelement location="${tests.classes.dir}"/>
<pathelement location="${refine.tests.classes.dir}"/>
<pathelement location="${refine.tests.lib.dir}/testng-6.8.jar"/>
<fileset dir="${refine.tests.lib.dir}">
<include name="**/*.jar" />
</fileset>
</path>
<taskdef resource="testngtasks" classpath="${refine.tests.lib.dir}/testng-6.8.jar"/>
<taskdef uri="antlib:org.jacoco.ant" resource="org/jacoco/ant/antlib.xml">
<classpath path="${refine.tests.lib.dir}/jacocoant.jar"/>
</taskdef>
<target name="build_java">
<mkdir dir="${classes.dir}" />
<javac source="${java_version}" target="${java_version}" encoding="utf-8" destdir="${classes.dir}" debug="true" includeAntRuntime="no">
<src path="${src.dir}"/>
<classpath refid="class.path" />
</javac>
</target>
<target name="build" depends="build_java"/>
<target name="build_tests" depends="build">
<mkdir dir="${tests.classes.dir}" />
<javac source="${java_version}" target="${java_version}" encoding="utf-8" srcdir="${tests.src.dir}" destdir="${tests.classes.dir}" debug="true" includeAntRuntime="no">
<classpath refid="extension.tests.class.path" />
</javac>
<copy file="${tests.src.dir}/tests.log4j.properties" tofile="${tests.classes.dir}/tests.log4j.properties"/>
</target>
<target name="test" depends="build_tests">
<jacoco:coverage destfile="${tests.report.dir}/jacoco.exec">
<testng verbose="2" haltOnFailure="true" workingdir="${extension.tests.dir}"
listener="org.testng.reporters.DotTestListener" excludedgroups="broken"
classpathref="extension.tests.class.path">
<xmlfileset file="${extension.tests.dir}/conf/tests.xml"/>
</testng>
</jacoco:coverage>
<jacoco:report>
<executiondata>
<file file="${tests.report.dir}/jacoco.exec"/>
</executiondata>
<structure name="Wikidata extension">
<classfiles>
<fileset dir="${classes.dir}" />
</classfiles>
<sourcefiles encoding="UTF-8">
<fileset dir="${src.dir}"/>
</sourcefiles>
</structure>
<html destdir="${tests.report.html.dir}"/>
<xml destfile="${tests.report.xml.path}"/>
</jacoco:report>
</target>
<target name="clean">
<delete dir="${classes.dir}" />
<delete dir="${tests.classes.dir}" />
<delete dir="${tests.report.dir}" />
<delete dir="${tests.build.dir}" />
</target>
</project>

32
extensions/wikidata/credits.txt Normal file
View File

@ -0,0 +1,32 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2018 Antonin Delpeuch
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#-------------------------------------------------------------------------------
The following images were obtained from Wikimedia Commons:
- Information.svg by User:Bobarino: (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Information.svg
- Warning.svg by User:Ezekiel63745 (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Information_orange.svg
- Important.svg originally from David Vignoni (GNU LGPL)
https://commons.wikimedia.org/wiki/File:Nuvola_apps_important.svg
- Critical.svg by User:Stannered (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Stop_x_nuvola.svg

78
extensions/wikidata/module/MOD-INF/controller.js Normal file
View File

@ -0,0 +1,78 @@
importPackage(org.openrefine.wikidata.commands);
/*
* Function invoked to initialize the extension.
*/
function init() {
var RefineServlet = Packages.com.google.refine.RefineServlet;
RefineServlet.registerClassMapping(
"org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation$WikibaseSchemaChange",
"org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation$WikibaseSchemaChange");
RefineServlet.registerClassMapping(
"org.openrefine.wikidata.operations.PerformWikibaseEditsOperation$PerformWikibaseEditsChange",
"org.openrefine.wikidata.operations.PerformWikibaseEditsOperation$PerformWikibaseEditsChange");
RefineServlet.cacheClass(Packages.org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation$WikibaseSchemaChange);
RefineServlet.cacheClass(Packages.org.openrefine.wikidata.operations.PerformWikibaseEditsOperation$PerformWikibaseEditsChange);
/*
* Attach a Wikibase schema to each project.
*/
Packages.com.google.refine.model.Project.registerOverlayModel(
"wikibaseSchema",
Packages.org.openrefine.wikidata.schema.WikibaseSchema);
/*
* Operations
*/
Packages.com.google.refine.operations.OperationRegistry.registerOperation(
module, "save-wikibase-schema", Packages.org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation);
Packages.com.google.refine.operations.OperationRegistry.registerOperation(
module, "perform-wikibase-edits", Packages.org.openrefine.wikidata.operations.PerformWikibaseEditsOperation);
/*
* Exporters
*/
var ExporterRegistry = Packages.com.google.refine.exporters.ExporterRegistry;
var QSExporter = Packages.org.openrefine.wikidata.exporters.QuickStatementsExporter;
ExporterRegistry.registerExporter("quickstatements", new QSExporter());
/*
* Commands
*/
RefineServlet.registerCommand(module, "save-wikibase-schema", new SaveWikibaseSchemaCommand());
RefineServlet.registerCommand(module, "preview-wikibase-schema", new PreviewWikibaseSchemaCommand());
RefineServlet.registerCommand(module, "perform-wikibase-edits", new PerformWikibaseEditsCommand());
RefineServlet.registerCommand(module, "login", new LoginCommand());
/*
* Resources
*/
ClientSideResourceManager.addPaths(
"project/scripts",
module,
[
"scripts/menu-bar-extension.js",
"scripts/warningsrenderer.js",
"scripts/langsuggest.js",
"scripts/bettersuggest.js",
"scripts/previewrenderer.js",
"scripts/dialogs/schema-alignment-dialog.js",
"scripts/dialogs/manage-account-dialog.js",
"scripts/dialogs/perform-edits-dialog.js",
"scripts/jquery.uls.data.js",
]);
ClientSideResourceManager.addPaths(
"project/styles",
module,
[
"styles/dialogs/schema-alignment-dialog.css",
"styles/dialogs/manage-account-dialog.less",
"styles/dialogs/perform-edits.less",
]);
}

BIN
extensions/wikidata/module/MOD-INF/lib-src/wdtk-datamodel-0.9.0-SNAPSHOT-sources.jar Normal file
View File

Binary file not shown.

BIN
extensions/wikidata/module/MOD-INF/lib-src/wdtk-wikibaseapi-0.9.0-SNAPSHOT-sources.jar Normal file
View File

Binary file not shown.

BIN
extensions/wikidata/module/MOD-INF/lib/wdtk-datamodel-0.9.0-SNAPSHOT-jar-with-dependencies.jar Normal file
View File

Binary file not shown.

BIN
extensions/wikidata/module/MOD-INF/lib/wdtk-wikibaseapi-0.9.0-SNAPSHOT.jar Normal file
View File

Binary file not shown.

3
extensions/wikidata/module/MOD-INF/module.properties Normal file
View File

@ -0,0 +1,3 @@
name = wikidata
description = OpenRefine Wikidata export
requires = core

BIN
extensions/wikidata/module/images/Critical.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.2 KiB

269
extensions/wikidata/module/images/Critical.svg Normal file
View File

@ -0,0 +1,269 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="240"
height="240"
id="svg4846">
<defs
id="defs4848">
<linearGradient
id="linearGradient5032">
<stop
style="stop-color:#ce1212;stop-opacity:1"
offset="0"
id="stop5034" />
<stop
style="stop-color:#ff1212;stop-opacity:1"
offset="1"
id="stop5036" />
</linearGradient>
<linearGradient
id="linearGradient5024">
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0"
id="stop5026" />
<stop
style="stop-color:#ffffff;stop-opacity:0"
offset="1"
id="stop5028" />
</linearGradient>
<radialGradient
cx="295.47125"
cy="186.09634"
r="179.55"
id="g5"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(1.14292e-8,1.531256,-0.876504,-2.165967e-5,462.7486,-245.0023)">
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0"
id="stop4889" />
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0.5"
id="stop4891" />
<stop
style="stop-color:#d5d5d5;stop-opacity:1"
offset="1"
id="stop4893" />
</radialGradient>
<linearGradient
x1="250.39845"
y1="101.53633"
x2="412.0943"
y2="264.54187"
id="g4"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787565)">
<stop
style="stop-color:#ffd9d9;stop-opacity:1"
offset="0"
id="stop4884" />
<stop
style="stop-color:#ff2727;stop-opacity:1"
offset="1"
id="stop4886" />
</linearGradient>
<linearGradient
x1="187.87357"
y1="224.59892"
x2="581.83746"
y2="483.10001"
id="g3"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787558)">
<stop
style="stop-color:#ec6c60;stop-opacity:1"
offset="0"
id="stop4879" />
<stop
style="stop-color:#d11412;stop-opacity:1"
offset="1"
id="stop4881" />
</linearGradient>
<linearGradient
x1="530.80951"
y1="486.63101"
x2="174.80548"
y2="211.22995"
id="g2"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787565)">
<stop
style="stop-color:#9a0000;stop-opacity:1"
offset="0"
id="stop4874" />
<stop
style="stop-color:#f22803;stop-opacity:1"
offset="1"
id="stop4876" />
</linearGradient>
<linearGradient
x1="41.194874"
y1="616.47717"
x2="118.93135"
y2="527.55511"
id="g1"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(4.556763,0,0,-4.315033,37.49756,2758.519)">
<stop
style="stop-color:#bb0000;stop-opacity:1"
offset="0"
id="stop4869" />
<stop
style="stop-color:#5f0000;stop-opacity:1"
offset="1"
id="stop4871" />
</linearGradient>
<radialGradient
cx="295.47125"
cy="186.09634"
r="179.55"
id="radialGradient4978"
xlink:href="#g5"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0,0.2725026,-0.1559828,-3.8545592e-6,437.26559,613.56411)" />
<linearGradient
x1="250.39845"
y1="101.53633"
x2="412.0943"
y2="264.54187"
id="linearGradient4981"
xlink:href="#g4"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="187.87357"
y1="224.59892"
x2="581.83746"
y2="483.10001"
id="linearGradient4984"
xlink:href="#g3"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="530.80951"
y1="486.63101"
x2="174.80548"
y2="211.22995"
id="linearGradient4987"
xlink:href="#g2"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="41.194874"
y1="616.47717"
x2="118.93135"
y2="527.55511"
id="linearGradient4990"
xlink:href="#g1"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.8109225,0,0,-0.7679041,361.58783,1148.0714)" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5030"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5038"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<filter
id="filter5068">
<feGaussianBlur
id="feGaussianBlur5070"
stdDeviation="0.16424528"
inkscape:collect="always" />
</filter>
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5099"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5101"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5125"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5127"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5153"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(3.7565752,0,0,3.7565752,-1281.1474,-1839.9864)" />
</defs>
<g
transform="translate(-463.84454,-666.74359)"
id="layer1">
<g
id="g5072">
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.5876353,-1.4860452,1.4860452,3.5876353,417.73023,715.73613)"
style="fill:#7e0000;fill-opacity:1"
id="path3717" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.5876353,-1.4860452,1.4860452,3.5876353,425.24338,723.24928)"
style="fill:#000000;fill-opacity:0.25657893;filter:url(#filter5068)"
id="path5042" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.3778534,-1.3991511,1.3991511,3.3778534,427.22392,719.66855)"
style="fill:#e78a8a;fill-opacity:1"
id="path3741" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.0836017,-1.2772686,1.2772686,3.0836017,440.54023,725.18436)"
style="fill:url(#linearGradient5125);fill-opacity:1"
id="path3743" />
<path
d="M 74.625,56.9375 L 56.9375,74.625 L 99.15625,116.84375 L 56.9375,159.0625 L 74.625,176.75 L 116.84375,134.53125 L 159.0625,176.75 L 176.75,159.0625 L 134.53125,116.84375 L 176.75,74.625 L 159.0625,56.9375 L 116.84375,99.15625 L 74.625,56.9375 z "
transform="translate(463.84454,666.74359)"
style="fill:#fdeeee;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:25;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path5139" />
<path
d="M 535.2741,674.9233 L 471.99929,738.19812 L 471.99929,765.37946 C 558.38932,785.44115 593.76927,739.43264 688.23715,764.37675 L 688.23715,738.19812 L 624.84494,674.9233 L 535.2741,674.9233 z "
style="fill:url(#linearGradient5153);fill-opacity:1"
id="path5021" />
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 9.0 KiB

BIN
extensions/wikidata/module/images/Important.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.5 KiB

269
extensions/wikidata/module/images/Important.svg Normal file
View File

@ -0,0 +1,269 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="240"
height="240"
id="svg4846">
<defs
id="defs4848">
<linearGradient
id="linearGradient5032">
<stop
style="stop-color:#ce1212;stop-opacity:1"
offset="0"
id="stop5034" />
<stop
style="stop-color:#ff1212;stop-opacity:1"
offset="1"
id="stop5036" />
</linearGradient>
<linearGradient
id="linearGradient5024">
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0"
id="stop5026" />
<stop
style="stop-color:#ffffff;stop-opacity:0"
offset="1"
id="stop5028" />
</linearGradient>
<radialGradient
cx="295.47125"
cy="186.09634"
r="179.55"
id="g5"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(1.14292e-8,1.531256,-0.876504,-2.165967e-5,462.7486,-245.0023)">
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0"
id="stop4889" />
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0.5"
id="stop4891" />
<stop
style="stop-color:#d5d5d5;stop-opacity:1"
offset="1"
id="stop4893" />
</radialGradient>
<linearGradient
x1="250.39845"
y1="101.53633"
x2="412.0943"
y2="264.54187"
id="g4"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787565)">
<stop
style="stop-color:#ffd9d9;stop-opacity:1"
offset="0"
id="stop4884" />
<stop
style="stop-color:#ff2727;stop-opacity:1"
offset="1"
id="stop4886" />
</linearGradient>
<linearGradient
x1="187.87357"
y1="224.59892"
x2="581.83746"
y2="483.10001"
id="g3"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787558)">
<stop
style="stop-color:#ec6c60;stop-opacity:1"
offset="0"
id="stop4879" />
<stop
style="stop-color:#d11412;stop-opacity:1"
offset="1"
id="stop4881" />
</linearGradient>
<linearGradient
x1="530.80951"
y1="486.63101"
x2="174.80548"
y2="211.22995"
id="g2"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787565)">
<stop
style="stop-color:#9a0000;stop-opacity:1"
offset="0"
id="stop4874" />
<stop
style="stop-color:#f22803;stop-opacity:1"
offset="1"
id="stop4876" />
</linearGradient>
<linearGradient
x1="41.194874"
y1="616.47717"
x2="118.93135"
y2="527.55511"
id="g1"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(4.556763,0,0,-4.315033,37.49756,2758.519)">
<stop
style="stop-color:#bb0000;stop-opacity:1"
offset="0"
id="stop4869" />
<stop
style="stop-color:#5f0000;stop-opacity:1"
offset="1"
id="stop4871" />
</linearGradient>
<radialGradient
cx="295.47125"
cy="186.09634"
r="179.55"
id="radialGradient4978"
xlink:href="#g5"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0,0.2725026,-0.1559828,-3.8545592e-6,437.26559,613.56411)" />
<linearGradient
x1="250.39845"
y1="101.53633"
x2="412.0943"
y2="264.54187"
id="linearGradient4981"
xlink:href="#g4"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="187.87357"
y1="224.59892"
x2="581.83746"
y2="483.10001"
id="linearGradient4984"
xlink:href="#g3"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="530.80951"
y1="486.63101"
x2="174.80548"
y2="211.22995"
id="linearGradient4987"
xlink:href="#g2"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="41.194874"
y1="616.47717"
x2="118.93135"
y2="527.55511"
id="linearGradient4990"
xlink:href="#g1"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.8109225,0,0,-0.7679041,361.58783,1148.0714)" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5030"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5038"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<filter
id="filter5068">
<feGaussianBlur
id="feGaussianBlur5070"
stdDeviation="0.16424528"
inkscape:collect="always" />
</filter>
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5099"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5101"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5125"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5127"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5153"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(3.7565752,0,0,3.7565752,-1281.1474,-1839.9864)" />
</defs>
<g
transform="translate(-463.84454,-666.74359)"
id="layer1">
<g
id="g5072">
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.5876353,-1.4860452,1.4860452,3.5876353,417.73023,715.73613)"
style="fill:#7e0000;fill-opacity:1"
id="path3717" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.5876353,-1.4860452,1.4860452,3.5876353,425.24338,723.24928)"
style="fill:#000000;fill-opacity:0.25657893;filter:url(#filter5068)"
id="path5042" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.3778534,-1.3991511,1.3991511,3.3778534,427.22392,719.66855)"
style="fill:#e78a8a;fill-opacity:1"
id="path3741" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.0836017,-1.2772686,1.2772686,3.0836017,440.54023,725.18436)"
style="fill:url(#linearGradient5125);fill-opacity:1"
id="path3743" />
<path
d="M 74.625,56.9375 L 56.9375,74.625 L 99.15625,116.84375 L 56.9375,159.0625 L 74.625,176.75 L 116.84375,134.53125 L 159.0625,176.75 L 176.75,159.0625 L 134.53125,116.84375 L 176.75,74.625 L 159.0625,56.9375 L 116.84375,99.15625 L 74.625,56.9375 z "
transform="translate(463.84454,666.74359)"
style="fill:#fdeeee;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:25;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path5139" />
<path
d="M 535.2741,674.9233 L 471.99929,738.19812 L 471.99929,765.37946 C 558.38932,785.44115 593.76927,739.43264 688.23715,764.37675 L 688.23715,738.19812 L 624.84494,674.9233 L 535.2741,674.9233 z "
style="fill:url(#linearGradient5153);fill-opacity:1"
id="path5021" />
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 9.0 KiB

BIN
extensions/wikidata/module/images/Information.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.2 KiB

252
extensions/wikidata/module/images/Information.svg Normal file
View File

@ -0,0 +1,252 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="256"
height="256"
id="svg2">
<defs
id="defs4">
<linearGradient
id="linearGradient2820">
<stop
style="stop-color:black;stop-opacity:0.3137255"
offset="0"
id="stop2822" />
<stop
style="stop-color:black;stop-opacity:0.19607843"
offset="0.92982459"
id="stop2826" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop2824" />
</linearGradient>
<linearGradient
id="linearGradient10053">
<stop
style="stop-color:white;stop-opacity:1"
offset="0"
id="stop10055" />
<stop
style="stop-color:white;stop-opacity:0.47959185"
offset="1"
id="stop10057" />
</linearGradient>
<linearGradient
id="linearGradient7070">
<stop
style="stop-color:white;stop-opacity:0.94897962"
offset="0"
id="stop7072" />
<stop
style="stop-color:white;stop-opacity:0.74489796"
offset="1"
id="stop7074" />
</linearGradient>
<linearGradient
id="linearGradient4153">
<stop
style="stop-color:#2a4fe4;stop-opacity:1"
offset="0"
id="stop4155" />
<stop
style="stop-color:#447bfa;stop-opacity:1"
offset="0.2"
id="stop7080" />
<stop
style="stop-color:#69a3ff;stop-opacity:1"
offset="0.625"
id="stop7084" />
<stop
style="stop-color:#89beff;stop-opacity:1"
offset="0.75"
id="stop7082" />
<stop
style="stop-color:#80beff;stop-opacity:1"
offset="1"
id="stop4157" />
</linearGradient>
<linearGradient
id="linearGradient4145">
<stop
style="stop-color:black;stop-opacity:1"
offset="0"
id="stop4147" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop4149" />
</linearGradient>
<linearGradient
id="linearGradient25332">
<stop
style="stop-color:#385ae5;stop-opacity:1"
offset="0"
id="stop25334" />
<stop
style="stop-color:#66a1ff;stop-opacity:1"
offset="0"
id="stop26309" />
<stop
style="stop-color:#66a1f0;stop-opacity:0.94117647"
offset="1"
id="stop25336" />
</linearGradient>
<linearGradient
id="linearGradient23386">
<stop
style="stop-color:#2b51e4;stop-opacity:0"
offset="0"
id="stop23388" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop23390" />
</linearGradient>
<linearGradient
id="linearGradient15560">
<stop
style="stop-color:#1864ed;stop-opacity:1"
offset="0"
id="stop15562" />
<stop
style="stop-color:#3a7bef;stop-opacity:0.25510204"
offset="1"
id="stop15564" />
</linearGradient>
<linearGradient
id="linearGradient15536">
<stop
style="stop-color:white;stop-opacity:1"
offset="0"
id="stop15538" />
<stop
style="stop-color:white;stop-opacity:0.59090906"
offset="1"
id="stop15540" />
</linearGradient>
<filter
id="filter9681">
<feGaussianBlur
inkscape:collect="always"
stdDeviation="1.7392633"
id="feGaussianBlur9683" />
</filter>
<filter
id="filter22413">
<feGaussianBlur
inkscape:collect="always"
stdDeviation="0.33355481"
id="feGaussianBlur22415" />
</filter>
<linearGradient
x1="168.97348"
y1="226.79765"
x2="94.360741"
y2="42.124634"
id="linearGradient4159"
xlink:href="#linearGradient4153"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="162.12244"
y1="228.19299"
x2="101.21178"
y2="40.729256"
id="linearGradient7076"
xlink:href="#linearGradient7070"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="127.8125"
y1="31.59375"
x2="127.8125"
y2="118.62509"
id="linearGradient10059"
xlink:href="#linearGradient10053"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="150.76942"
y1="72.579872"
x2="162.2932"
y2="131.31783"
id="linearGradient2818"
xlink:href="#linearGradient2820"
gradientUnits="userSpaceOnUse" />
<radialGradient
cx="130.61937"
cy="132.01637"
r="115.95089"
fx="130.61937"
fy="132.01637"
id="radialGradient3721"
xlink:href="#linearGradient2820"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="127.8125"
y1="31.59375"
x2="127.8125"
y2="118.62509"
id="linearGradient4614"
xlink:href="#linearGradient15536"
gradientUnits="userSpaceOnUse" />
</defs>
<g
style="opacity:1;fill:url(#linearGradient2818);fill-opacity:1;display:inline"
id="layer2">
<path
d="M 246.57026,132.01637 C 246.57026,196.05428 194.65728,247.96726 130.61937,247.96726 C 66.581461,247.96726 14.668479,196.05428 14.66848,132.01637 C 14.668479,67.978461 66.581461,16.065479 130.61937,16.065479 C 194.65728,16.065479 246.57026,67.978461 246.57026,132.01637 L 246.57026,132.01637 z "
transform="matrix(0.995962,0,0,0.995962,-0.171109,-0.165469)"
style="opacity:1;fill:url(#radialGradient3721);fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path12053" />
</g>
<g
style="opacity:1;display:inline"
id="layer4">
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
transform="matrix(1.082583,0,0,1.082583,-14.54063,-17.56537)"
style="opacity:1;fill:#3a73ef;fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path2175" />
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
transform="matrix(1.042113,0,0,1.042113,-9.212006,-12.12368)"
style="opacity:1;fill:url(#linearGradient7076);fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path13592" />
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
transform="matrix(0.951055,0,0,0.951055,2.777393,0.12014)"
style="opacity:1;fill:url(#linearGradient4159);fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path14565" />
</g>
<g
style="opacity:1;display:inline"
id="layer5" />
<g
style="opacity:1;display:inline"
id="layer3">
<g
id="g15568">
<path
d="M 123.30585,191.07575 C 129.02124,187.76374 138.24211,176.49437 141.6732,168.65732 C 141.6732,168.65732 138.1399,167.1567 138.1399,167.1567 C 131.0411,179.5796 120.02693,188.70016 123.71066,173.32458 C 123.71066,173.32458 145.45687,105.04964 145.45687,105.04964 C 139.77784,104.57639 111.00423,109.17626 106.51327,110.10356 C 106.51327,110.10356 105.79723,113.88169 105.79723,113.88169 C 115.61397,114.21756 116.94651,118.2341 116.23663,121.5238 C 116.23663,121.5238 99.927949,173.57586 99.927949,173.57586 C 99.927949,173.57586 97.881571,179.46474 97.947071,183.84902 C 98.105167,193.86006 111.11196,197.7125 123.30585,191.07575 z "
style="opacity:1;fill:white;fill-opacity:1;stroke:none;stroke-width:0.1;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;display:inline"
id="path3339" />
<path
d="M 151.9268,75.963402 C 151.93091,83.589999 145.74948,89.774755 138.12288,89.774755 C 130.49628,89.774755 124.31485,83.589999 124.31896,75.963402 C 124.31485,68.336805 130.49628,62.152049 138.12288,62.152049 C 145.74948,62.152049 151.93091,68.336805 151.9268,75.963402 L 151.9268,75.963402 z "
transform="matrix(1.006031,0,0,1.006031,-1.167682,-0.583638)"
style="opacity:1;fill:white;fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;display:inline"
id="path7182" />
</g>
</g>
<g
style="opacity:1;fill:url(#linearGradient4614);fill-opacity:1;display:inline"
id="layer7">
<path
d="M 128,31.59375 C 86.871341,31.59375 51.74836,57.401715 37.90625,93.6875 C 56.732365,110.35912 96.245675,118.37197 128.03125,118.625 C 128.07285,118.62467 128.02111,118.62533 128.0625,118.625 C 160.09769,118.36974 199.12268,109.67594 217.71875,92.6875 C 203.61988,56.922098 168.75455,31.593751 128,31.59375 z "
style="opacity:0.45;fill:url(#linearGradient4614);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1"
id="path18502" />
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

BIN
extensions/wikidata/module/images/Warning.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.7 KiB

348
extensions/wikidata/module/images/Warning.svg Normal file
View File

@ -0,0 +1,348 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="256"
height="256"
id="svg2"
sodipodi:version="0.32"
inkscape:version="0.48.0 r9654"
sodipodi:docname="Information red.svg"
inkscape:output_extension="org.inkscape.output.svg.inkscape">
<metadata
id="metadata60">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<sodipodi:namedview
inkscape:window-height="851"
inkscape:window-width="1440"
inkscape:pageshadow="2"
inkscape:pageopacity="0.0"
guidetolerance="10.0"
gridtolerance="10.0"
objecttolerance="10.0"
borderopacity="1.0"
bordercolor="#666666"
pagecolor="#ffffff"
id="base"
showgrid="false"
inkscape:zoom="2.8047763"
inkscape:cx="65.457674"
inkscape:cy="135.60528"
inkscape:window-x="-8"
inkscape:window-y="-8"
inkscape:current-layer="layer4"
inkscape:window-maximized="1"
showguides="true"
inkscape:guide-bbox="true">
<sodipodi:guide
orientation="1,0"
position="-224.9697,26.597403"
id="guide3113" />
</sodipodi:namedview>
<defs
id="defs4">
<linearGradient
inkscape:collect="always"
id="linearGradient3139">
<stop
style="stop-color:#ff6600;stop-opacity:1;"
offset="0"
id="stop3141" />
<stop
style="stop-color:#ffffff;stop-opacity:1;"
offset="1"
id="stop3143" />
</linearGradient>
<linearGradient
id="linearGradient3105">
<stop
style="stop-color:#ff2a2a;stop-opacity:1;"
offset="0"
id="stop3107" />
<stop
style="stop-color:#ffffff;stop-opacity:1;"
offset="1"
id="stop3109" />
</linearGradient>
<inkscape:perspective
sodipodi:type="inkscape:persp3d"
inkscape:vp_x="0 : 128 : 1"
inkscape:vp_y="0 : 1000 : 0"
inkscape:vp_z="256 : 128 : 1"
inkscape:persp3d-origin="128 : 85.333333 : 1"
id="perspective62" />
<linearGradient
id="linearGradient2820">
<stop
style="stop-color:black;stop-opacity:0.3137255"
offset="0"
id="stop2822" />
<stop
style="stop-color:black;stop-opacity:0.19607843"
offset="0.92982459"
id="stop2826" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop2824" />
</linearGradient>
<linearGradient
id="linearGradient10053">
<stop
style="stop-color:white;stop-opacity:1"
offset="0"
id="stop10055" />
<stop
style="stop-color:white;stop-opacity:0.47959185"
offset="1"
id="stop10057" />
</linearGradient>
<linearGradient
id="linearGradient7070">
<stop
style="stop-color:white;stop-opacity:0.94897962"
offset="0"
id="stop7072" />
<stop
style="stop-color:white;stop-opacity:0.74489796"
offset="1"
id="stop7074" />
</linearGradient>
<linearGradient
id="linearGradient4153">
<stop
style="stop-color:#2a4fe4;stop-opacity:1"
offset="0"
id="stop4155" />
<stop
style="stop-color:#447bfa;stop-opacity:1"
offset="0.2"
id="stop7080" />
<stop
style="stop-color:#69a3ff;stop-opacity:1"
offset="0.625"
id="stop7084" />
<stop
style="stop-color:#89beff;stop-opacity:1"
offset="0.75"
id="stop7082" />
<stop
style="stop-color:#80beff;stop-opacity:1"
offset="1"
id="stop4157" />
</linearGradient>
<linearGradient
id="linearGradient4145">
<stop
style="stop-color:black;stop-opacity:1"
offset="0"
id="stop4147" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop4149" />
</linearGradient>
<linearGradient
id="linearGradient25332">
<stop
style="stop-color:#385ae5;stop-opacity:1"
offset="0"
id="stop25334" />
<stop
style="stop-color:#66a1ff;stop-opacity:1"
offset="0"
id="stop26309" />
<stop
style="stop-color:#66a1f0;stop-opacity:0.94117647"
offset="1"
id="stop25336" />
</linearGradient>
<linearGradient
id="linearGradient23386">
<stop
style="stop-color:#2b51e4;stop-opacity:0"
offset="0"
id="stop23388" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop23390" />
</linearGradient>
<linearGradient
id="linearGradient15560">
<stop
style="stop-color:#1864ed;stop-opacity:1"
offset="0"
id="stop15562" />
<stop
style="stop-color:#3a7bef;stop-opacity:0.25510204"
offset="1"
id="stop15564" />
</linearGradient>
<linearGradient
id="linearGradient15536">
<stop
style="stop-color:white;stop-opacity:1"
offset="0"
id="stop15538" />
<stop
style="stop-color:white;stop-opacity:0.59090906"
offset="1"
id="stop15540" />
</linearGradient>
<filter
id="filter9681">
<feGaussianBlur
inkscape:collect="always"
stdDeviation="1.7392633"
id="feGaussianBlur9683" />
</filter>
<filter
id="filter22413">
<feGaussianBlur
inkscape:collect="always"
stdDeviation="0.33355481"
id="feGaussianBlur22415" />
</filter>
<linearGradient
x1="168.97348"
y1="226.79765"
x2="94.360741"
y2="42.124634"
id="linearGradient4159"
xlink:href="#linearGradient4153"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="162.12244"
y1="228.19299"
x2="101.21178"
y2="40.729256"
id="linearGradient7076"
xlink:href="#linearGradient7070"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="127.8125"
y1="31.59375"
x2="127.8125"
y2="118.62509"
id="linearGradient10059"
xlink:href="#linearGradient10053"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="150.76942"
y1="72.579872"
x2="162.2932"
y2="131.31783"
id="linearGradient2818"
xlink:href="#linearGradient2820"
gradientUnits="userSpaceOnUse" />
<radialGradient
cx="130.61937"
cy="132.01637"
r="115.95089"
fx="130.61937"
fy="132.01637"
id="radialGradient3721"
xlink:href="#linearGradient2820"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="127.8125"
y1="31.59375"
x2="127.8125"
y2="118.62509"
id="linearGradient4614"
xlink:href="#linearGradient15536"
gradientUnits="userSpaceOnUse" />
<linearGradient
inkscape:collect="always"
xlink:href="#linearGradient3139"
id="linearGradient3145"
x1="186.39575"
y1="170.82901"
x2="-165.99458"
y2="78.232819"
gradientUnits="userSpaceOnUse" />
<linearGradient
inkscape:collect="always"
xlink:href="#linearGradient3139"
id="linearGradient3147"
gradientUnits="userSpaceOnUse"
x1="184.89622"
y1="178.70155"
x2="-117.25975"
y2="71.859802" />
</defs>
<g
style="opacity:1;fill:url(#linearGradient2818);fill-opacity:1;display:inline"
id="layer2">
<path
d="M 246.57026,132.01637 C 246.57026,196.05428 194.65728,247.96726 130.61937,247.96726 C 66.581461,247.96726 14.668479,196.05428 14.66848,132.01637 C 14.668479,67.978461 66.581461,16.065479 130.61937,16.065479 C 194.65728,16.065479 246.57026,67.978461 246.57026,132.01637 L 246.57026,132.01637 z "
transform="matrix(0.995962,0,0,0.995962,-0.171109,-0.165469)"
style="opacity:1;fill:url(#radialGradient3721);fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path12053" />
</g>
<g
style="opacity:1;display:inline"
id="layer4">
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
style="opacity:1;fill:#d45500;fill-opacity:1;stroke:none;stroke-width:0.10000000000000001;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path2175"
transform="matrix(1.082583,0,0,1.082583,-14.54063,-17.56537)" />
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
style="opacity:1;fill:#ffb380;fill-opacity:1;stroke:none;stroke-width:0.10000000000000001;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path13592"
transform="matrix(1.042113,0,0,1.042113,-9.212006,-12.12368)" />
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
style="opacity:1;fill:url(#linearGradient3147);fill-opacity:1;stroke:none;stroke-width:0.10000000000000001;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path14565"
transform="matrix(0.951055,0,0,0.951055,2.777393,0.12014)" />
</g>
<g
style="opacity:1;display:inline"
id="layer5" />
<g
style="opacity:1;display:inline"
id="layer3">
<g
id="g15568">
<path
d="M 123.30585,191.07575 C 129.02124,187.76374 138.24211,176.49437 141.6732,168.65732 C 141.6732,168.65732 138.1399,167.1567 138.1399,167.1567 C 131.0411,179.5796 120.02693,188.70016 123.71066,173.32458 C 123.71066,173.32458 145.45687,105.04964 145.45687,105.04964 C 139.77784,104.57639 111.00423,109.17626 106.51327,110.10356 C 106.51327,110.10356 105.79723,113.88169 105.79723,113.88169 C 115.61397,114.21756 116.94651,118.2341 116.23663,121.5238 C 116.23663,121.5238 99.927949,173.57586 99.927949,173.57586 C 99.927949,173.57586 97.881571,179.46474 97.947071,183.84902 C 98.105167,193.86006 111.11196,197.7125 123.30585,191.07575 z "
style="opacity:1;fill:white;fill-opacity:1;stroke:none;stroke-width:0.1;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;display:inline"
id="path3339" />
<path
d="M 151.9268,75.963402 C 151.93091,83.589999 145.74948,89.774755 138.12288,89.774755 C 130.49628,89.774755 124.31485,83.589999 124.31896,75.963402 C 124.31485,68.336805 130.49628,62.152049 138.12288,62.152049 C 145.74948,62.152049 151.93091,68.336805 151.9268,75.963402 L 151.9268,75.963402 z "
transform="matrix(1.006031,0,0,1.006031,-1.167682,-0.583638)"
style="opacity:1;fill:white;fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;display:inline"
id="path7182" />
</g>
</g>
<g
style="opacity:1;fill:url(#linearGradient4614);fill-opacity:1;display:inline"
id="layer7">
<path
d="M 128,31.59375 C 86.871341,31.59375 51.74836,57.401715 37.90625,93.6875 C 56.732365,110.35912 96.245675,118.37197 128.03125,118.625 C 128.07285,118.62467 128.02111,118.62533 128.0625,118.625 C 160.09769,118.36974 199.12268,109.67594 217.71875,92.6875 C 203.61988,56.922098 168.75455,31.593751 128,31.59375 z "
style="opacity:0.45;fill:url(#linearGradient4614);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1"
id="path18502" />
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

7
extensions/wikidata/module/images/Wikidata-logo-en.svg Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" version="1.2" width="1052.36" height="744.09003">
<path d="m 119.4207,543.01679 h 29.16277 V 43.052497 H 119.4207 V 543.01679 z m 60.27256,0 h 89.42926 V 43.052497 H 179.69326 V 543.01679 z M 298.29136,43.052497 V 542.99895 h 89.42926 V 43.052497 h -89.42926 z" style="fill:#990000" />
<path d="m 838.97933,543.0524 h 29.16883 V 43.052437 H 838.97933 V 543.0524 z M 899.25795,43.052437 V 543.0524 h 29.16277 V 43.052437 H 899.25795 z M 417.85387,543.05247 h 29.16277 V 43.052497 H 417.85387 V 543.05247 z M 478.13249,43.052497 V 543.03463 h 29.16884 V 43.052497 h -29.16884 z" style="fill:#339966" />
<path d="m 537.42244,543.05247 h 89.4414 V 43.052497 h -89.4414 V 543.05247 z m 118.5981,0 h 31.10372 V 43.052497 H 656.02054 V 543.05247 z M 716.2931,43.052497 V 543.03463 h 89.42926 V 43.052497 H 716.2931 z" style="fill:#006699" />
<path d="m 883.98859,598.08811 46.01562,103.68359 -27.75781,0 -8.53516,-20.48438 -46.38672,0 -7.71875,20.48438 -27.38672,0 41.78516,-103.68359 29.98438,0 m 2.07812,65.08984 -16.69922,-40.30078 -15.21484,40.30078 31.91406,0 m -60.26562,-65.08984 0,19 -37.55469,0 0,84.68359 -25.82813,0 0,-84.68359 -37.55468,0 0,-19 100.9375,0 m -133.59375,0 46.01562,103.68359 -27.75781,0 -8.53516,-20.48438 -46.38672,0 -7.71875,20.48438 -27.38672,0 41.78516,-103.68359 29.98438,0 m 2.07812,65.08984 -16.69922,-40.30078 -15.21484,40.30078 31.91406,0 m -176.56641,-65.08984 45.71875,0 c 17.66399,10e-5 31.6171,4.84905 41.85938,14.54687 10.29156,9.64852 15.43739,22.142 15.4375,37.48047 -1.1e-4,16.08076 -5.22016,28.72268 -15.66016,37.92578 -10.39071,9.15365 -25.23444,13.73047 -44.53125,13.73047 l -42.82422,0 0,-103.68359 m 25.82813,19 0,65.68359 16.84766,0 c 10.93483,2e-5 19.2968,-2.99347 25.08593,-8.98047 5.78898,-6.03642 8.68351,-13.97782 8.6836,-23.82422 -9e-5,-10.19264 -2.9441,-18.20826 -8.83203,-24.04687 -5.83862,-5.88794 -14.25007,-8.83195 -25.23438,-8.83203 l -16.55078,0 m -49.72656,-19 0,103.68359 -25.82813,0 0,-103.68359 25.82813,0 m -48.91016,0 -33.76953,48.6875 45.79297,54.99609 -32.35938,0 -41.04297,-48.61328 0,48.61328 -25.82812,0 0,-103.68359 25.82812,0 0,46.68359 32.95313,-46.68359 28.42578,0 m -111.10547,0 0,103.68359 -25.82812,0 0,-103.68359 25.82812,0 m -127.21094,37.40625 -32.0625,67.61328 -11.13281,0 -45.57031,-105.01953 27.38672,0 24.41797,57.29687 26.64453,-57.29687 20.78125,0 26.64453,57.29687 24.26953,-57.29687 27.38672,0 -45.42188,105.01953 -11.13281,0 -32.21094,-67.61328" style="fill:#484848;fill-opacity:1;stroke:none;font-family:Gill Sans MT" />
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.6 KiB

BIN
extensions/wikidata/module/images/wikidata.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.5 KiB

205
extensions/wikidata/module/langs/translation-en.json Normal file
View File

@ -0,0 +1,205 @@
{
"wikidata-extension": {
"menu-label": "Wikidata",
"edit-wikidata-schema": "Edit Wikidata schema",
"manage-wikidata-account": "Manage Wikidata account",
"perform-edits-on-wikidata": "Upload edits to Wikidata",
"export-to-qs": "Export to QuickStatements",
"quickstatements-export-name": "QuickStatements"
},
"wikidata-schema": {
"dialog-header": "Align to Wikidata",
"dialog-explanation": "The Wikidata schema below specifies how your tabular data will be transformed into Wikidata edits. You can drag and drop the column names below in most input boxes: for each row, edits will be generated with the values in these columns.",
"preview-explanation": "This tab shows the first edits (out of {nb_edits}) that will be made once you upload the changes to Wikidata. You can use facets to inspect the edits on particular items.",
"schema-tab-header": "Schema",
"warnings-tab-header": "Issues",
"edits-preview-tab-header": "Preview",
"statements-header": "Statements",
"terms-header": "Terms",
"empty-statements": "no statements added",
"empty-terms": "no labels, descriptions or aliases added",
"add-item-button": "add item",
"add-term": "add term",
"remove": "remove",
"add-statement": "add statement",
"add-value": "add value",
"add-qualifier": "add qualifier",
"add-reference": "add reference",
"add-reference-snak": "add",
"property-placeholder": "property",
"nb-references": "&nbsp;references",
"remove-column": "remove column",
"label": "Label",
"description": "Description",
"alias": "Alias",
"item-or-reconciled-column": "type item or drag reconciled column here",
"amount": "amount",
"unit": "unit",
"full-url": "full URL including the protocol",
"tabular-data-with-prefix": "filename starting with \"Data:\"",
"commons-media": "filename",
"math-expression": "mathematical expression",
"geoshape-with-prefix": "filename starting with \"Data:\"",
"datatype-not-supported-yet": "This datatype is not supported yet, sorry.",
"invalid-schema-warning-issues": "Your schema is incomplete, fix it to see the issues.",
"invalid-schema-warning-preview": "Your schema is incomplete, fix it to see the preview.",
"discard-button": "Discard changes",
"save-button": "Save schema",
"close-button": "Close",
"unsaved-changes-alt": "You have made unsaved changes to your Wikidata schema.",
"save-schema-alt": "Save the schema to OpenRefine. The changes will not be uploaded to Wikidata yet.",
"discard-schema-changes-alt": "Discard the changes made to the schema.",
"incomplete-schema-could-not-be-saved": "Your schema is incomplete so it cannot be saved yet.",
"unsaved-warning": "You have made unsaved changes to your Wikidata schema. Close anyway?"
},
"wikidata-preview": {
"new-id": "new item"
},
"wikidata-account": {
"dialog-header": "Wikidata account",
"explain-log-in": "Logging in to <a href=\"https://www.wikidata.org/\" target=\"_blank\">Wikidata</a> lets you to upload edits directly from OpenRefine.",
"username-label": "Username:",
"username-placeholder": "Enter your username",
"password-label": "Password:",
"password-placeholder": "Enter your password",
"remember-credentials-label": "Remember credentials (stored unencrypted in OpenRefine's preferences)",
"close": "Close",
"log-in": "Log in",
"logged-in-as": "You are logged in as:",
"log-out": "Log out",
"connecting-to-wikidata": "Connecting to Wikidata..."
},
"perform-wikidata-edits": {
"dialog-header": "Upload edits to Wikidata",
"review-your-edits": "You are about to upload {nb_edits} edits to Wikidata. Please check them carefully. Large edit batches should be submitted for <a href=\"https://www.wikidata.org/wiki/Wikidata:Requests_for_permissions/Bot\" target=\"_blank\">bot review</a> first.",
"logged-in-as": "You are logged in as",
"edit-summary-label": "Edit summary:",
"edit-summary-placeholder": "a few words to describe your edits",
"perform-edits": "Upload edits",
"cancel": "Cancel",
"analyzing-edits": "Analyzing your edits..."
},
"warnings-messages": {
"new-item-created": {
"title": "This edit batch will create new Wikidata items.",
"body": "Please make sure that these items do not exist yet and are <a href=\"https://www.wikidata.org/wiki/Wikidata:Notability\" target=\"_blank\">suitable for inclusion in Wikidata</a>."
},
"new-item-without-labels-or-aliases": {
"title": "New items created without any label or alias.",
"body": "You should at least provide one label for new items such as {example_entity}, so that others can understand what the item is about."
},
"new-item-without-descriptions": {
"title": "New items created without any description.",
"body": "Adding descriptions on new items such as {example_entity} will make it easier to disambiguate the items from namesakes."
},
"new-item-with-deleted-statements": {
"title": "Deleting statements on new items.",
"body": "There is probably something wrong in your schema or project."
},
"new-item-without-P31-or-P279": {
"title": "New items created without any type.",
"body": "You should provide an \"instance of\" (P31) or \"subclass of\" (P279) statement for each item that you create, such as {example_entity}."
},
"add-statements-with-invalid-format": {
"title": "{property_entity} statements with invalid format.",
"body": "Values for this property are expected to match the regular expression <span class=\"wb-issue-preformat\">{regex}</span>, which is not the case for <span class=\"wb-issue-preformat\">{example_value}</span> added on {example_item_entity}."
},
"remove-statements-with-invalid-format": {
"title": "Removed statements with invalid format.",
"body": "If these statements currently exist on Wikidata, this will solve constraint violations."
},
"missing-inverse-statements": {
"title": "Inverse statements missing for {added_property_entity}.",
"body": "Any {added_property_entity} statement such as the one from {source_entity} to {target_entity} should also be added in reverse with {inverse_property_entity}: in this case, {target_entity} {inverse_property_entity} {source_entity}."
},
"self-referential-statements": {
"title": "Self-referential statements.",
"body": "While not forbidden, self-referential statements are generally suspicious. You have some on {example_entity}."
},
"unsourced-statements": {
"title": "Statements without references.",
"body": "Most statements should have references. You can add them easily in the schema."
},
"property-restricted-to-reference-found-in-mainsnak": {
"title": "{property_entity} used as statement.",
"body": "You are using {property_entity} in a statement but it is designed to be used in references only."
},
"property-restricted-to-reference-found-in-qualifier": {
"title": "{property_entity} used as qualifier.",
"body": "You are using in {property_entity} in a qualifier but it is designed to be used in references only."
},
"property-restricted-to-qualifier-found-in-mainsnak": {
"title": "{property_entity} used as statement.",
"body": "You are using {property_entity} in a statement but it is designed to be used in qualifiers only."
},
"property-restricted-to-qualifier-found-in-reference": {
"title": "{property_entity} used as reference.",
"body": "You are using {property_entity} in a reference but it is designed to be used in qualifiers only."
},
"property-restricted-to-mainsnak-found-in-qualifier": {
"title": "{property_entity} used as qualifier.",
"body": "You are using {property_entity} in a qualifier but it is designed to be used as a statement only."
},
"property-restricted-to-mainsnak-found-in-reference": {
"title": "{property_entity} used as reference.",
"body": "You are using {property_entity} in a reference but it is designed to be used as a statement only."
},
"missing-mandatory-qualifiers": {
"title": "{statement_property_entity} is missing a {missing_property_entity} qualifier.",
"body": "Statements using {statement_property_entity} such as the one on {example_item_entity} are missing a mandatory {missing_property_entity} qualifier."
},
"disallowed-qualifiers": {
"title": "Qualifier {disallowed_property_entity} is incompatible with {statement_property_entity}.",
"body": "Statements using {statement_property_entity} such as the one on {example_item_entity} should not have a {disallowed_property_entity} qualifier as they are incompatible."
},
"single-valued-property-added-more-than-once": {
"title": "{property_entity} added more than once on the same item.",
"body": "This property is expected to be used at most once on each item but has been added multiple times on the same item, for instance on {example_entity}."
},
"identical-values-for-distinct-valued-property": {
"title": "Identical values for {property_entity}",
"body": "This property should have distinct values, but the same value was found on {item1_entity} and {item2_entity} for instance."
},
"no-edit-generated": {
"title": "No edit was generated.",
"body": "There might be something wrong with your schema."
},
"no-issue-detected": {
"title": "No issue was detected in your edits.",
"body": "Note that OpenRefine cannot detect all the types of problems Wikidata edits can have."
},
"ignored-qualifiers": {
"title": "Some qualifiers were ignored.",
"body": "Qualifier values could not be parsed, so they will not be added to the corresponding statements."
},
"ignored-references": {
"title": "Some references were ignored.",
"body": "None of their statements could be parsed, so no reference was added."
},
"monolingual-text-without-language": {
"title": "No language provided for monolingual text.",
"body": "Some label, description, alias or monolingual text value have been skipped because no language was provided. Example value: <span class=\"wb-issue-preformat\">{example_text}</span>."
},
"leading-whitespace": {
"title": "Leading whitespace in strings.",
"body": "Strings such as <span class=\"wb-issue-preformat\">{example_string}</span> have leading whitespace."
},
"trailing-whitespace": {
"title": "Trailing whitespace in strings.",
"body": "Strings such as <span class=\"wb-issue-preformat\">{example_string}</span> have trailing whitespace."
},
"duplicate-whitespace": {
"title": "Duplicate whitespace in strings.",
"body": "Strings such as <span class=\"wb-issue-preformat\">{example_string}</span> contain duplicate whitespace."
},
"non-printable-characters": {
"title": "Non-printable characters in strings.",
"body": "Strings such as <span class=\"wb-issue-preformat\">{example_string}</span> contain non-printable characters."
},
"invalid-identifier-space": {
"title": "Invalid identifier space for reconciled cells.",
"body": "Some reconciled cells such as <span class=\"wb-issue-preformat\">{example_cell}</span> were ignored because they are not reconciled to Wikidata."
}
}
}

22
extensions/wikidata/module/scripts/bettersuggest.js Normal file
View File

@ -0,0 +1,22 @@
/**
* Adds a few tweaks to a suggest widget, mostly to indicate
* the status of the validation.
*/
var fixSuggestInput = function(input) {
input.bind("fb-select", function(evt, data) {
input.addClass('wbs-validated-input');
input.blur();
}).bind("fb-textchange", function(evt, data) {
input.removeClass('wbs-validated-input');
}).blur(function() {
setTimeout(function() {
if(! input.hasClass('wbs-validated-input')) {
input.addClass('wbs-unvalidated-input');
}
}, 100);
}).focus(function() {
input.removeClass('wbs-unvalidated-input');
});
}

49
extensions/wikidata/module/scripts/dialogs/manage-account-dialog.html Normal file
View File

@ -0,0 +1,49 @@
<div class="dialog-frame" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<a href="https://www.wikidata.org/" target="_blank">
<img src="extension/wikidata/images/wikidata.png" class="wikidata-logo" alt="Wikidata logo" />
</a>
<div class="right-of-logo">
<p class="body-text" bind="explainLogIn">
</p>
<div class="wikibase-user-management-area">
<div class="wikibase-user-login" bind="loginArea">
<div bind="invalidCredentials" class="wikibase-invalid-credentials"></div>
<form bind="loginForm" class="wikibase-login-form" method="post">
<table>
<tr>
<td><label for="wb-username" bind="usernameLabel"></label></td>
<td><input name="wb-username" type="text" bind="usernameInput" /></td>
</tr>
<tr>
<td><label for="wb-password" bind="passwordLabel"></label></td>
<td><input name="wb-password" type="password" bind="passwordInput" /></td>
</tr>
<tr>
<td><input type="checkbox" name="remember-credentials" /></td>
<td><label for="remember-credentials" bind="rememberCredentialsLabel"></label></td>
</tr>
</table>
</form>
<div class="wikibase-login-buttons">
<button class="button cancel-button" bind="cancelButton1"></button>
<button class="button button-primary" bind="loginButton"></button>
</div>
</div>
<div class="wikibase-user-logout" bind="logoutArea">
<p><span bind="loggedInAs"></span>
<a bind="loggedInUsername" target="_blank"></a></p>
<form bind="logoutForm" method="post">
<input type="hidden" name="wb-username" value="null" />
<input name="wb-password" type="hidden" value="null" />
</form>
<div class="wikibase-login-buttons">
<button class="button cancel-button" bind="cancelButton2"></button>
<button class="button button-primary" bind="logoutButton"></button>
</div>
</div>
</div>
</div>
</div>
</div>

114
extensions/wikidata/module/scripts/dialogs/manage-account-dialog.js Normal file
View File

@ -0,0 +1,114 @@
var ManageAccountDialog = {};
ManageAccountDialog.firstLogin = true;
ManageAccountDialog.launch = function(logged_in_username, callback) {
$.post(
"command/core/get-all-preferences",
null,
function (preferences) {
ManageAccountDialog.display(logged_in_username, preferences.wikidata_credentials, callback);
},
"json"
);
};
ManageAccountDialog.display = function(logged_in_username, saved_credentials, callback) {
var self = this;
var frame = $(DOM.loadHTML("wikidata", "scripts/dialogs/manage-account-dialog.html"));
var elmts = this._elmts = DOM.bind(frame);
ManageAccountDialog.firstLaunch = false;
this._elmts.dialogHeader.text($.i18n._('wikidata-account')["dialog-header"]);
this._elmts.explainLogIn.html($.i18n._('wikidata-account')["explain-log-in"]);
this._elmts.usernameLabel.text($.i18n._('wikidata-account')["username-label"]);
this._elmts.usernameInput.attr("placeholder", $.i18n._('wikidata-account')["username-placeholder"]);
this._elmts.passwordLabel.text($.i18n._('wikidata-account')["password-label"]);
this._elmts.passwordInput.attr("placeholder", $.i18n._('wikidata-account')["password-placeholder"]);
this._elmts.rememberCredentialsLabel.text($.i18n._('wikidata-account')["remember-credentials-label"]);
this._elmts.dialogHeader.text($.i18n._('wikidata-account')["dialog-header"]);
this._elmts.cancelButton1.text($.i18n._('wikidata-account')["close"]);
this._elmts.cancelButton2.text($.i18n._('wikidata-account')["close"]);
this._elmts.loggedInAs.text($.i18n._('wikidata-account')["logged-in-as"]);
this._elmts.logoutButton.text($.i18n._('wikidata-account')["log-out"]);
this._elmts.loginButton.text($.i18n._('wikidata-account')["log-in"]);
this._level = DialogSystem.showDialog(frame);
var dismiss = function() {
DialogSystem.dismissUntil(self._level - 1);
};
if (logged_in_username != null) {
elmts.loginArea.hide();
} else {
elmts.logoutArea.hide();
}
elmts.loggedInUsername
.text(logged_in_username)
.attr('href', 'https://www.wikidata.org/wiki/User:'+logged_in_username);
frame.find('.cancel-button').click(function() {
dismiss();
callback(null);
});
elmts.loginButton.click(function() {
frame.hide();
$.post(
"command/wikidata/login",
elmts.loginForm.serialize(),
function(data) {
if (data.logged_in) {
dismiss();
callback(data.username);
} else {
frame.show();
elmts.invalidCredentials.text("Invalid credentials.");
}
});
});
elmts.logoutButton.click(function() {
$.post(
"command/wikidata/login",
"logout=true",
function(data) {
if (!data.logged_in) {
dismiss();
callback(null);
}
});
});
};
ManageAccountDialog.isLoggedIn = function(callback) {
var discardWaiter = function() { };
if(ManageAccountDialog.firstLogin) {
discardWaiter = DialogSystem.showBusy($.i18n._('wikidata-account')["connecting-to-wikidata"]);
}
$.get(
"command/wikidata/login",
function(data) {
discardWaiter();
ManageAccountDialog.firstLogin = false;
callback(data.username);
});
};
ManageAccountDialog.ensureLoggedIn = function(callback) {
ManageAccountDialog.isLoggedIn(function(logged_in_username) {
if (logged_in_username == null) {
ManageAccountDialog.launch(null, callback);
} else {
callback(logged_in_username);
}
});
};
ManageAccountDialog.checkAndLaunch = function () {
ManageAccountDialog.isLoggedIn(function(logged_in_username) {
ManageAccountDialog.launch(logged_in_username, function(success) { });
});
};

17
extensions/wikidata/module/scripts/dialogs/perform-edits-dialog.html Normal file
View File

@ -0,0 +1,17 @@
<div class="dialog-frame" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<p class="body-text" bind="reviewYourEdits">
</p>
<div class="perform-edits-warnings-area" bind="warningsArea">
</div>
<div class="wikibase-perform-edits-area">
<p><span bind="loggedInAs"></span> <a bind="loggedInUsername" target="_blank"></a>.</p>
<p><span bind="editSummaryLabel"></span> <input type="text" name="editSummary" bind="editSummary" class="edit-summary" value="" /></p>
<div class="wikibase-login-buttons">
<button class="button cancel-button" bind="cancelButton"></button>
<button class="button button-primary" bind="performEditsButton"></button>
</div>
</div>
</div>
</div>

115
extensions/wikidata/module/scripts/dialogs/perform-edits-dialog.js Normal file
View File

@ -0,0 +1,115 @@
var PerformEditsDialog = {};
PerformEditsDialog.launch = function(logged_in_username, max_severity) {
var self = this;
var elmts = this._elmts;
var frame = this.frame;
if (this.missingSchema) {
return;
}
this._level = DialogSystem.showDialog(frame);
this._elmts.dialogHeader.text($.i18n._('perform-wikidata-edits')["dialog-header"]);
this._elmts.loggedInAs.text($.i18n._('perform-wikidata-edits')["logged-in-as"]);
this._elmts.editSummaryLabel.text($.i18n._('perform-wikidata-edits')["edit-summary-label"]);
this._elmts.editSummary.attr('placeholder', $.i18n._('perform-wikidata-edits')["edit-summary-placeholder"]);
this._elmts.performEditsButton.text($.i18n._('perform-wikidata-edits')["perform-edits"]);
this._elmts.cancelButton.text($.i18n._('perform-wikidata-edits')["cancel"]);
var dismiss = function() {
DialogSystem.dismissUntil(self._level - 1);
};
elmts.loggedInUsername
.text(logged_in_username)
.attr('href','https://www.wikidata.org/wiki/User:'+logged_in_username);
frame.find('.cancel-button').click(function() {
dismiss();
});
if (max_severity === 'CRITICAL') {
elmts.performEditsButton.prop("disabled",true).addClass("button-disabled");
} else {
elmts.performEditsButton.click(function() {
if(elmts.editSummary.val().length == 0) {
elmts.editSummary.focus();
} else {
Refine.postProcess(
"wikidata",
"perform-wikibase-edits",
{},
{
summary: elmts.editSummary.val(),
},
{ includeEngine: true, cellsChanged: true, columnStatsChanged: true },
{ onDone:
function() {
dismiss();
}
});
}
});
}
};
PerformEditsDialog.updateEditCount = function(edit_count) {
this._elmts.reviewYourEdits.html(
$.i18n._('perform-wikidata-edits')["review-your-edits"]
.replace('{nb_edits}', edit_count));
}
PerformEditsDialog._updateWarnings = function(data) {
var warnings = data.warnings;
var mainDiv = this._elmts.warningsArea;
// clear everything
mainDiv.empty();
PerformEditsDialog.updateEditCount(data.edit_count);
var table = $('<table></table>').appendTo(mainDiv);
for (var i = 0; i != warnings.length; i++) {
var rendered = WarningsRenderer._renderWarning(warnings[i]);
rendered.appendTo(table);
}
}
PerformEditsDialog.checkAndLaunch = function () {
var self = this;
this.frame = $(DOM.loadHTML("wikidata", "scripts/dialogs/perform-edits-dialog.html"));
this._elmts = DOM.bind(this.frame);
this.missingSchema = false;
var onSaved = function() {
ManageAccountDialog.ensureLoggedIn(function(logged_in_username) {
if (logged_in_username) {
var discardWaiter = DialogSystem.showBusy($.i18n._('perform-wikidata-edits')["analyzing-edits"]);
$.post(
"command/wikidata/preview-wikibase-schema?" + $.param({ project: theProject.id }),
{ engine: JSON.stringify(ui.browsingEngine.getJSON()) },
function(data) {
discardWaiter();
if(data['code'] != 'error') {
PerformEditsDialog._updateWarnings(data);
PerformEditsDialog.launch(logged_in_username, data['max_severity']);
} else {
SchemaAlignmentDialog.launch(
PerformEditsDialog.checkAndLaunch);
}
},
"json"
);
}
});
};
if (SchemaAlignmentDialog.isSetUp() && SchemaAlignmentDialog._hasUnsavedChanges) {
SchemaAlignmentDialog._save(onSaved);
} else {
onSaved();
}
};

49
extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.html Normal file
View File

@ -0,0 +1,49 @@
<div class="dialog-frame" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<div id="schema-alignment-tabs" class="refine-tabs">
<ul>
<li><a href="#schema-alignment-tabs-schema" bind="schemaTabHeader"></a></li>
<li><a href="#schema-alignment-tabs-warnings"><span bind="warningsTabHeader"></span> <span class="schema-alignment-total-warning-count" bind="warningsTabCount"></span></a></li>
<li><a href="#schema-alignment-tabs-preview-edits" bind="editsPreviewTabHeader"></a></li>
</ul>
<div id="schema-alignment-tabs-schema">
<div bind="schemaHeader">
<p class="body-text" bind="dialogExplanation"></p>
<div class="schema-alignment-dialog-columns-area" bind="columnsArea">
</div>
</div>
<div class="schema-alignment-dialog-canvas" bind="canvas">
<div class="schema-alignment-dialog-statements-area" bind="statementsArea">
<div id="schema-alignment-statements-container">
</div>
<div class="wbs-toolbar"><a class="wbs-add-item" bind="addItemButton"></a></div>
</div>
</div>
</div>
<div id="schema-alignment-tabs-warnings" style="display: none;">
<div class="invalid-schema-warning" bind="invalidSchemaWarningIssues"></div>
<div class="schema-alignment-dialog-warnings" bind="warningsArea"></div>
</div>
<div id="schema-alignment-tabs-preview-edits" style="display: none;">
<p class="body-text" bind="previewExplanation"></p>
<div class="invalid-schema-warning" bind="invalidSchemaWarningPreview"></div>
<div class="schema-alignment-dialog-preview"></div>
</div>
</div>
</div>
<div class="dialog-footer" bind="dialogFooter"><div class="grid-layout layout-normal layout-full"><table><tr>
<td>
<button class="button" bind="resetButton"></button>
</td>
<td style="text-align:right;">
<button class="button button-primary" bind="saveButton"></button>
</td>
<td style="text-align:center;" width="30%">
<span class="schema-alignment-status-indicator" bind="statusIndicator"></span>
</td>
<td style="text-align:right;" width="1%">
<button class="button" bind="closeButton"></button>
</td>
</tr></table></div></div>
</div>

1192
extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js Normal file
View File

File diff suppressed because it is too large Load Diff

5
extensions/wikidata/module/scripts/issues-tab.html Normal file
View File

@ -0,0 +1,5 @@
<div id="schema-issues-tab">
<div class="invalid-schema-warning" bind="invalidSchemaWarningIssues"></div>
<div class="schema-issues-area" bind="warningsArea"></div>
</div>

11
extensions/wikidata/module/scripts/jquery.uls.data.js Normal file
View File

File diff suppressed because one or more lines are too long

78
extensions/wikidata/module/scripts/langsuggest.js Normal file
View File

@ -0,0 +1,78 @@
/********************
* LANGUAGE SUGGEST *
********************/
// This list was manually copied from https://www.wikidata.org/w/api.php?action=paraminfo&modules=wbsetlabel on 2017-10-06
// I don't think it is worth making every OpenRefine client perform this query at every startup, because it typically does
// not change very often.
// See https://stackoverflow.com/questions/46507037/how-to-get-all-allowed-languages-for-wikidata/46562061#46562061
WIKIDATA_LANGUAGES = [ "aa", "ab", "ace", "ady", "ady-cyrl", "aeb", "aeb-arab", "aeb-latn", "af", "ak", "aln", "als", "am", "an", "ang", "anp", "ar", "arc", "arn", "arq", "ary", "arz",
"as", "ase", "ast", "atj", "av", "avk", "awa", "ay", "az", "azb", "ba", "ban", "bar", "bat-smg", "bbc", "bbc-latn", "bcc", "bcl", "be", "be-tarask", "be-x-old", "bg", "bgn", "bh", "bho",
"bi", "bjn", "bm", "bn", "bo", "bpy", "bqi", "br", "brh", "bs", "bto", "bug", "bxr", "ca", "cbk-zam", "cdo", "ce", "ceb", "ch", "cho", "chr", "chy", "ckb", "co", "cps", "cr", "crh",
"crh-cyrl", "crh-latn", "cs", "csb", "cu", "cv", "cy", "da", "de", "de-at", "de-ch", "de-formal", "din", "diq", "dsb", "dtp", "dty", "dv", "dz", "ee", "egl", "el", "eml", "en", "en-ca",
"en-gb", "eo", "es", "et", "eu", "ext", "fa", "ff", "fi", "fit", "fiu-vro", "fj", "fo", "fr", "frc", "frp", "frr", "fur", "fy", "ga", "gag", "gan", "gan-hans", "gan-hant", "gd", "gl",
"glk", "gn", "gom", "gom-deva", "gom-latn", "gor", "got", "grc", "gsw", "gu", "gv", "ha", "hak", "haw", "he", "hi", "hif", "hif-latn", "hil", "ho", "hr", "hrx", "hsb", "ht", "hu", "hy",
"hz", "ia", "id", "ie", "ig", "ii", "ik", "ike-cans", "ike-latn", "ilo", "inh", "io", "is", "it", "iu", "ja", "jam", "jbo", "jut", "jv", "ka", "kaa", "kab", "kbd", "kbd-cyrl", "kbp",
"kea", "kg", "khw", "ki", "kiu", "kj", "kk", "kk-arab", "kk-cn", "kk-cyrl", "kk-kz", "kk-latn", "kk-tr", "kl", "km", "kn", "ko", "ko-kp", "koi", "kr", "krc", "kri", "krj", "krl", "ks",
"ks-arab", "ks-deva", "ksh", "ku", "ku-arab", "ku-latn", "kv", "kw", "ky", "la", "lad", "lb", "lbe", "lez", "lfn", "lg", "li", "lij", "liv", "lki", "lmo", "ln", "lo", "loz", "lrc", "lt",
"ltg", "lus", "luz", "lv", "lzh", "lzz", "mai", "map-bms", "mdf", "mg", "mh", "mhr", "mi", "min", "mk", "ml", "mn", "mo", "mr", "mrj", "ms", "mt", "mus", "mwl", "my", "myv", "mzn", "na",
"nah", "nan", "nap", "nb", "nds", "nds-nl", "ne", "new", "ng", "niu", "nl", "nl-informal", "nn", "no", "nod", "nov", "nrm", "nso", "nv", "ny", "nys", "oc", "olo", "om", "or", "os", "ota",
"pa", "pag", "pam", "pap", "pcd", "pdc", "pdt", "pfl", "pi", "pih", "pl", "pms", "pnb", "pnt", "prg", "ps", "pt", "pt-br", "qu", "qug", "rgn", "rif", "rm", "rmy", "rn", "ro", "roa-rup",
"roa-tara", "ru", "rue", "rup", "ruq", "ruq-cyrl", "ruq-latn", "rw", "rwr", "sa", "sah", "sat", "sc", "scn", "sco", "sd", "sdc", "sdh", "se", "sei", "ses", "sg", "sgs", "sh", "shi",
"shi-latn", "shi-tfng", "shn", "si", "simple", "sje", "sk", "skr", "skr-arab", "sl", "sli", "sm", "sma", "smj", "sn", "so", "sq", "sr", "sr-ec", "sr-el", "srn", "srq", "ss", "st", "stq",
"su", "sv", "sw", "szl", "ta", "tay", "tcy", "te", "tet", "tg", "tg-cyrl", "tg-latn", "th", "ti", "tk", "tl", "tly", "tn", "to", "tokipona", "tpi", "tr", "tru", "ts", "tt", "tt-cyrl",
"tt-latn", "tum", "tw", "ty", "tyv", "tzm", "udm", "ug", "ug-arab", "ug-latn", "uk", "ur", "uz", "uz-cyrl", "uz-latn", "ve", "vec", "vep", "vi", "vls", "vmf", "vo", "vot", "vro", "wa",
"war", "wo", "wuu", "xal", "xh", "xmf", "yi", "yo", "yue", "za", "zea", "zh", "zh-classical", "zh-cn", "zh-hans", "zh-hant", "zh-hk", "zh-min-nan", "zh-mo", "zh-my", "zh-sg", "zh-tw",
"zh-yue", "zu" ]
$.suggest("langsuggest", {
_init: function() {
this.api_url = "https://www.wikidata.org/w/api.php";
this._status.SELECT = "Select a language from the list:";
},
request: function(val, cursor) {
var self = this;
var ajax_options = {
url: self.api_url,
data: { action: "languagesearch",
search: val,
format: "json", },
success: function(data) {
self.response(self.convertResults(data));
},
dataType: "jsonp",
};
$.ajax(ajax_options);
},
convertResults: function(data) {
var array = [];
for (var key in data.languagesearch) {
if (data.languagesearch.hasOwnProperty(key) && WIKIDATA_LANGUAGES.indexOf(key) != -1) {
array.push({ id: key, name: key, search_name: data.languagesearch[key] });
}
}
return array;
},
create_item: function(data, response_data) {
var css = this.options.css;
var li = $("<li>").addClass(css.item);
var type = $("<div>").addClass("fbs-item-type").text(data.id);
var native_name = this.get_native_name(data.id);
var full_name = native_name ? native_name : data.search_name;
var label = $("<label>").text(full_name);
li.append($("<div>").addClass(css.item_name).append(type).append(label));
return li;
},
get_native_name: function(lang_code) {
var language = $.uls.data.languages[lang_code];
if (language) {
return language[2];
}
},
});

90
extensions/wikidata/module/scripts/menu-bar-extension.js Normal file
View File

@ -0,0 +1,90 @@
// Load the localization file
var dictionary = {};
$.ajax({
url : "command/core/load-language?",
type : "POST",
async : false,
data : {
module : "wikidata",
// lang : lang
},
success : function(data) {
dictionary = data;
}
});
$.i18n.setDictionary(dictionary);
ExporterManager.MenuItems.push({});
ExporterManager.MenuItems.push(
{
"id" : "exportQuickStatements",
"label": $.i18n._('wikidata-extension')["quickstatements-export-name"],
"click": function() { WikibaseExporterMenuBar.exportTo("quickstatements"); }
}
);
WikibaseExporterMenuBar = {};
WikibaseExporterMenuBar.exportTo = function(format) {
var form = document.createElement("form");
$(form).css("display", "none")
.attr("method", "post")
.attr("action", "command/core/export-rows/statements.txt")
.attr("target", "gridworks-export");
$('<input />')
.attr("name", "engine")
.attr("value", JSON.stringify(ui.browsingEngine.getJSON()))
.appendTo(form);
$('<input />')
.attr("name", "project")
.attr("value", theProject.id)
.appendTo(form);
$('<input />')
.attr("name", "format")
.attr("value", format)
.appendTo(form);
document.body.appendChild(form);
window.open("about:blank", "gridworks-export");
form.submit();
document.body.removeChild(form);
};
//extend the column header menu
$(function(){
ExtensionBar.MenuItems.push(
{
"id":"reconcile",
"label": $.i18n._('wikidata-extension')["menu-label"],
"submenu" : [
{
id: "wikidata/edit-schema",
label: $.i18n._('wikidata-extension')["edit-wikidata-schema"],
click: function() { SchemaAlignmentDialog.launch(false); }
},
{
id:"wikidata/manage-account",
label: $.i18n._('wikidata-extension')["manage-wikidata-account"],
click: function() { ManageAccountDialog.checkAndLaunch(); }
},
{
id:"wikidata/perform-edits",
label: $.i18n._('wikidata-extension')["perform-edits-on-wikidata"],
click: function() { PerformEditsDialog.checkAndLaunch(); }
},
{
id:"wikidata/export-qs",
label: $.i18n._('wikidata-extension')["export-to-qs"],
click: function() { WikibaseExporterMenuBar.exportTo("quickstatements"); }
},
]
}
);
});

6
extensions/wikidata/module/scripts/preview-tab.html Normal file
View File

@ -0,0 +1,6 @@
<div id="schema-preview-tab">
<p class="panel-explanation" bind="previewExplanation"></p>
<div class="invalid-schema-warning" bind="invalidSchemaWarningPreview"></div>
<div class="schema-alignment-dialog-preview"></div>
</div>

256
extensions/wikidata/module/scripts/previewrenderer.js Normal file
View File

@ -0,0 +1,256 @@
/**
* renders an item update (an edit on an item) in HTML.
*/
var EditRenderer = {};
// settings
EditRenderer.maxTerms = 15; // max number of terms displayed
EditRenderer.maxStatements = 25; // max number of statements per statement group
// main method: takes a DOM element and a list
// of edits to render there.
EditRenderer.renderEdits = function(edits, container) {
for(var i = 0; i != edits.length; i++) {
EditRenderer._renderItem(edits[i], container);
}
}
/**************/
/*** ITEMS ****/
/**************/
EditRenderer._renderItem = function(json, container) {
var subject = json;
var statementGroups = null;
var nameDescs = null;
if (json) {
subject = json.subject;
statementGroups = json.statementGroups;
nameDescs = json.nameDescs;
}
var item = $('<div></div>').addClass('wbs-item').appendTo(container);
var inputContainer = $('<div></div>').addClass('wbs-item-input').appendTo(item);
EditRenderer._renderEntity(json.subject, inputContainer);
var right = $('<div></div>').addClass('wbs-item-contents').appendTo(item);
// Terms
if ((json.labels && json.labels.length) ||
(json.descriptions && json.descriptions.length) ||
(json.addedAliases && json.addedAliases.length)) {
var termsContainer = $('<div></div>').addClass('wbs-namedesc-container')
.appendTo(right);
this._renderTermsList(json.labels, "label", termsContainer);
this._renderTermsList(json.descriptions, "description", termsContainer);
this._renderTermsList(json.aliases, "alias", termsContainer);
// Clear the float
$('<div></div>').attr('style', 'clear: right').appendTo(right);
}
// Statements
if (json.addedStatementGroups && json.addedStatementGroups.length) {
// $('<div></div>').addClass('wbs-statements-header')
// .text($.i18n._('wikidata-schema')['statements-header']).appendTo(right);
var statementsGroupContainer = $('<div></div>').addClass('wbs-statement-group-container')
.appendTo(right);
for(var i = 0; i != json.addedStatementGroups.length; i++) {
EditRenderer._renderStatementGroup(json.addedStatementGroups[i], statementsGroupContainer);
}
}
}
/**************************
* NAMES AND DESCRIPTIONS *
**************************/
EditRenderer._renderTermsList = function(termList, termType, termsContainer) {
if(!termList) {
return;
}
for(var i = 0; i != Math.min(termList.length, this.maxTerms); i++) {
EditRenderer._renderTerm(termType, termList[i], termsContainer);
}
if(termList.length > this.maxTerms) {
$('<div></div>').addClass('wbs-namedesc').text('...').appendTo(termsContainer);
}
}
EditRenderer._renderTerm = function(termType, json, container) {
var namedesc = $('<div></div>').addClass('wbs-namedesc').appendTo(container);
var type_container = $('<div></div>').addClass('wbs-namedesc-type').appendTo(namedesc);
var type_span = $('<span></span>').appendTo(type_container)
.text($.i18n._('wikidata-schema')[termType]);
var right = $('<div></div>').addClass('wbs-right').appendTo(namedesc);
var value_container = $('<div></div>').addClass('wbs-namedesc-value').appendTo(namedesc);
EditRenderer._renderValue({datavalue:json,datatype:'monolingualtext'}, value_container);
}
/********************
* STATEMENT GROUPS *
********************/
EditRenderer._renderStatementGroup = function(json, container) {
var statementGroup = $('<div></div>').addClass('wbs-statement-group').appendTo(container);
var inputContainer = $('<div></div>').addClass('wbs-prop-input').appendTo(statementGroup);
var right = $('<div></div>').addClass('wbs-right').appendTo(statementGroup);
EditRenderer._renderEntity(json.property, inputContainer);
var statementContainer = $('<div></div>').addClass('wbs-statement-container').appendTo(right);
for (var i = 0; i != json.statements.length; i++) {
EditRenderer._renderStatement(json.statements[i], statementContainer);
}
if(json.statements.length > EditRenderer.maxStatements) {
$('<div></div>')
.text('...')
.addClass('wbs-statement')
.appendTo(statementContainer);
}
}
/**************
* STATEMENTS *
**************/
EditRenderer._renderStatement = function(json, container) {
var statement = $('<div></div>').addClass('wbs-statement').appendTo(container);
var inputContainer = $('<div></div>').addClass('wbs-target-input').appendTo(statement);
EditRenderer._renderValue(json.mainsnak, inputContainer);
// add rank
var rank = $('<div></div>').addClass('wbs-rank-selector-icon').prependTo(inputContainer);
// add qualifiers...
var right = $('<div></div>').addClass('wbs-right').appendTo(statement);
var qualifierContainer = $('<div></div>').addClass('wbs-qualifier-container').appendTo(right);
if (json.qualifiers) {
for (var pid in json.qualifiers) {
if (json.qualifiers.hasOwnProperty(pid)) {
var qualifiers = json.qualifiers[pid];
for (var i = 0; i != qualifiers.length; i++) {
EditRenderer._renderSnak(qualifiers[i], qualifierContainer);
}
}
}
}
// and references
$('<div></div>').attr('style', 'clear: right').appendTo(statement);
var referencesToggleContainer = $('<div></div>').addClass('wbs-references-toggle').appendTo(statement);
var triangle = $('<div></div>').addClass('triangle-icon').addClass('pointing-right').appendTo(referencesToggleContainer);
var referencesToggle = $('<a></a>').appendTo(referencesToggleContainer);
right = $('<div></div>').addClass('wbs-right').appendTo(statement);
var referenceContainer = $('<div></div>').addClass('wbs-reference-container').appendTo(right);
referencesToggle.click(function () {
triangle.toggleClass('pointing-down');
triangle.toggleClass('pointing-right');
referenceContainer.toggle(100);
});
referenceContainer.hide();
if (json.references) {
for (var i = 0; i != json.references.length; i++) {
EditRenderer._renderReference(json.references[i], referenceContainer);
}
}
EditRenderer._updateReferencesNumber(referenceContainer);
}
/*********************************
* QUALIFIER AND REFERENCE SNAKS *
*********************************/
EditRenderer._renderSnak = function(json, container) {
var qualifier = $('<div></div>').addClass('wbs-qualifier').appendTo(container);
var toolbar1 = $('<div></div>').addClass('wbs-toolbar').appendTo(qualifier);
var inputContainer = $('<div></div>').addClass('wbs-prop-input').appendTo(qualifier);
var right = $('<div></div>').addClass('wbs-right').appendTo(qualifier);
var statementContainer = $('<div></div>').addClass('wbs-statement-container').appendTo(right);
EditRenderer._renderEntity(json.full_property, inputContainer);
EditRenderer._renderValue(json, statementContainer);
}
/**************
* REFERENCES *
**************/
EditRenderer._renderReference = function(json, container) {
var reference = $('<div></div>').addClass('wbs-reference').appendTo(container);
var referenceHeader = $('<div></div>').addClass('wbs-reference-header').appendTo(reference);
var right = $('<div></div>').addClass('wbs-right').appendTo(reference);
var qualifierContainer = $('<div></div>').addClass('wbs-qualifier-container').appendTo(right);
for (var pid in json.snaks) {
if (json.snaks.hasOwnProperty(pid)) {
var snaks = json.snaks[pid];
for(var i = 0; i != snaks.length; i++) {
EditRenderer._renderSnak(snaks[i], qualifierContainer);
}
}
}
}
EditRenderer._updateReferencesNumber = function(container) {
var childrenCount = container.children().length;
var statement = container.parents('.wbs-statement');
var a = statement.find('.wbs-references-toggle a').first();
a.html(childrenCount+$.i18n._('wikidata-schema')["nb-references"]);
}
/*******************
* VALUE RENDERING *
*******************/
EditRenderer.renderedValueCache = {};
EditRenderer._renderEntity = function(json, container) {
var html = WarningsRenderer._renderEntity(json);
$(html).appendTo(container);
}
EditRenderer._renderValue = function(json, container) {
var input = $('<span></span>').appendTo(container);
var mode = json.datatype;
if (mode === "wikibase-item" || mode === "wikibase-property") {
EditRenderer._renderEntity(json.datavalue, container);
} else {
var jsonValue = JSON.stringify(json.datavalue);
if (jsonValue in EditRenderer.renderedValueCache) {
$('<span>'+EditRenderer.renderedValueCache[jsonValue]+'</span>').appendTo(container);
} else {
var params = {
action: 'wbformatvalue',
generate: 'text/html',
datavalue: jsonValue,
options: '{"lang":"'+$.i18n._('core-recon')["wd-recon-lang"]+'"}',
format: 'json'
};
if ('property' in json) {
params.property = json.property;
} else {
params.datatype = json.datatype;
}
$.get(
'https://www.wikidata.org/w/api.php',
params,
function (data) {
if('result' in data) {
EditRenderer.renderedValueCache[jsonValue] = data.result;
$('<span>'+data.result+'</span>').appendTo(container);
}
},
'jsonp'
);
}
}
}

17
extensions/wikidata/module/scripts/schema-alignment-tab.html Normal file
View File

@ -0,0 +1,17 @@
<div id="schema-alignment-tab">
<div bind="schemaHeader">
<div class="schema-alignment-save"><button class="button button-primary" bind="saveButton"></button><button class="button button-primary" bind="discardButton"></button></div>
<p class="panel-explanation" bind="dialogExplanation"></p>
<div style="clear: right"></div>
<div class="schema-alignment-dialog-columns-area" bind="columnsArea">
</div>
</div>
<div class="schema-alignment-dialog-canvas" bind="canvas">
<div class="schema-alignment-dialog-statements-area" bind="statementsArea">
<div id="schema-alignment-statements-container">
</div>
<div class="wbs-toolbar"><a class="wbs-add-item" bind="addItemButton"></a></div>
</div>
</div>
</div>

79
extensions/wikidata/module/scripts/warningsrenderer.js Normal file
View File

@ -0,0 +1,79 @@
var WarningsRenderer = {};
// renders a Wikibase entity into a link
WarningsRenderer._renderEntity = function(entity) {
if (!entity.id && entity.value) {
entity.id = entity.value.id;
}
var id = entity.id;
var is_new = entity.siteIri == "http://localhost/entity/";
if (is_new) {
id = $.i18n._('wikidata-preview')['new-id'];
}
var fullLabel = id;
if (entity.label) {
fullLabel = entity.label + ' (' + id + ')';
}
var url = entity.iri;
if (!url && entity.value) {
url = 'http://www.wikidata.org/entity/'+entity.value.id;
}
if (is_new) {
return '<span class="wb-preview-new-entity">'+fullLabel+'</span>';
} else {
return '<a href="'+url+'" class="wb-preview-entity" target="_blank">'+fullLabel+'</a>';
}
}
// replaces the issue properties in localization template
WarningsRenderer._replaceIssueProperties = function(template, properties) {
if (!properties) {
return template;
}
var expanded = template;
for (var key in properties) {
if (properties.hasOwnProperty(key)) {
var rendered = properties[key];
if (key.endsWith('_entity')) {
rendered = WarningsRenderer._renderEntity(properties[key]);
}
expanded = expanded.replace(new RegExp('{'+key+'}', 'g'), rendered);
}
}
return expanded;
}
WarningsRenderer._renderWarning = function(warning) {
var localized = $.i18n._('warnings-messages')[warning.type];
var title = warning.type;
var body = "";
if (localized) {
title = WarningsRenderer._replaceIssueProperties(localized.title, warning.properties);
body = WarningsRenderer._replaceIssueProperties(localized.body, warning.properties);
}
var tr = $('<tr></tr>').addClass('wb-warning');
var severityTd = $('<td></td>')
.addClass('wb-warning-severity')
.addClass('wb-warning-severity-'+warning.severity)
.appendTo(tr);
var bodyTd = $('<td></td>')
.addClass('wb-warning-body')
.appendTo(tr);
var h1 = $('<h1></h1>')
.html(title)
.appendTo(bodyTd);
var p = $('<p></p>')
.html(body)
.appendTo(bodyTd);
var countTd = $('<td></td>')
.addClass('wb-warning-count')
.appendTo(tr);
var countSpan = $('<span></span>')
.text(warning.count)
.appendTo(countTd);
return tr;
}

64
extensions/wikidata/module/styles/dialogs/manage-account-dialog.less Normal file
View File

@ -0,0 +1,64 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@import-less url("../theme.less");
.wikibase-login-form {
text-align: center;
}
.wikibase-login-buttons {
text-align: right;
}
.wikibase-invalid-credentials {
color: red;
}
.wikibase-user-login tr td:first-child {
text-align: right;
}
.wikibase-user-login tr td:last-child {
padding-left: 10px;
text-align: left;
}
.wikidata-logo {
float: left;
margin: -10px;
}
.right-of-logo {
margin-left: 110px;
}

39
extensions/wikidata/module/styles/dialogs/perform-edits.less Normal file
View File

@ -0,0 +1,39 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@import-less url("../theme.less");
.edit-summary {
width: 300px;
}

516
extensions/wikidata/module/styles/dialogs/schema-alignment-dialog.css Normal file
View File

@ -0,0 +1,516 @@
@import-less url("../theme.less");
.main-view-panel-tab-header#summary-bar {
font-size: 1.3em;
font-weight: normal;
position: initial;
}
#wikidata-schema-panel, #wikidata-issues-panel, #wikidata-preview-panel {
display: block;
overflow: hidden;
height: 100%;
background-color: white;
font-size: 1.2em;
}
#schema-preview-tab, #schema-issues-tab, #schema-alignment-tab {
overflow: auto;
height: 100%;
}
#schema-alignment-tab {
overflow: hidden;
}
.main-view-panel-tab-header {
margin-top: 9px;
margin-left: 7px;
font-size: 1.3em;
display: inline-block;
background-color: transparent;
color: #11c;
padding: 3px 7px;
cursor: pointer;
}
.main-view-panel-tab-header.active {
background-color: white;
color: black;
border: 1px solid #818fb7;
border-bottom: none;
}
.main-view-panel-tab-header.active:first-child {
background-color: #f2f2f2;
}
.schema-alignment-dialog-canvas {
background: white;
overflow-y: auto;
}
.schema-alignment-dialog-statements-area {
padding: 10px;
max-width: 900px;
overflow-y: auto;
}
.panel-explanation {
margin: 1em;
}
.schema-alignment-save {
float: right;
padding: 1em;
}
.schema-alignment-save button:disabled {
color: gray;
}
.schema-alignment-save button {
font-weight: normal;
}
.schema-alignment-dialog-columns-area {
border: 1px solid #bcf;
border-left: 0;
border-right: 0;
padding: 5px;
max-height: 100px;
overflow-y: auto;
}
.wbs-draggable-column {
border: 1px solid #aaa;
padding: 2px;
margin: 2px;
background-color: #eee;
display: inline-block;
padding-bottom: 3px;
max-width: 95%;
}
.ui-droppable .wbs-draggable-column {
margin: 0px;
padding: 0px;
padding-left: 2px;
}
.wbs-reconciled-column {
border-bottom: 3px solid #282;
padding-bottom: 1px;
}
.wbs-restricted-column-name {
max-width: -moz-calc(100% - 20px);
max-width: -webkit-calc(100% - 20px);
max-width: -o-calc(100% - 20px);
max-width: calc(100% - 20px);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
display: inline-block;
vertical-align: middle;
white-space: nowrap;
}
.wbs-draggable-column .wbs-remove {
float: right;
}
.wbs-accepting-input {
box-shadow: inset 0 0 10px #a7cdff;
}
.wbs-validated-input {
border: 1px solid green !important;
}
.wbs-unvalidated-input {
border: 1px solid red !important;
background-color: #ffbfbf;
}
.wbs-icon {
background-position: center;
background-size: 20px 20px;
width: 20px;
height: 20px;
display: inline-block;
vertical-align: middle;
opacity: 0.7;
}
.wbs-remove .wbs-icon {
background-image:linear-gradient(transparent,transparent),url("data:image/svg+xml,%3Csvg%20xmlns=%22http://www.w3.org/2000/svg%22%20width=%2224%22%20height=%2224%22%20viewBox=%220%200%2024%2024%22%3E%20%3Cg%20id=%22remove%22%3E%20%3Cpath%20id=%22trash-can%22%20d=%22M12%2010h-1v6h1v-6zm-2%200H9v6h1v-6zm4%200h-1v6h1v-6zm0-4V5H9v1H6v3h1v7.966l1%201.03v-.073V18h6.984l.016-.018v.015l1-1.03V9h1V6h-3zm1%2011H8V9h7v8zm1-9H7V7h9v1z%22/%3E%20%3C/g%3E%20%3C/svg%3E");
}
.wbs-remove span:last-child:hover {
text-decoration: underline;
}
.wbs-rank-selector-icon {
background-position: -36px;
width: 8px;
flex: 0 0 8px;
height: 20px;
margin-right: 2px;
background-image:linear-gradient(transparent,transparent),url("data:image/svg+xml,%3Csvg%20xmlns=%22http://www.w3.org/2000/svg%22%20xmlns:xlink=%22http://www.w3.org/1999/xlink%22%20width=%22107%22%20height=%2220%22%3E%20%3Cdefs%3E%20%3Cpath%20d=%22M3.1%2C0%200%2C3.8%200%2C6%208%2C6%208%2C3.8%204.9%2C0zm8.2%2C7%20-2.3%2C2%200%2C2%202.3%2C2%203.4%2C0%202.3%2C-2%200%2C-2%20-2.3%2C-2zm6.7%2C7%200%2C2.2%203.1%2C3.8%201.8%2C0%203.1%2C-3.8%200%2C-2.2z%22%20id=%22a%22/%3E%20%3Cpath%20d=%22m18.5%2C10.75%200%2C-1.5%202%2C-1.75%203%2C0%202%2C1.75%200%2C1.5%20-2%2C1.75%20-3%2C0zm0%2C-6.75%200%2C1.5%207%2C0%200%2C-1.5%20-2.875%2C-3.5%20-1.25%2C0zm-9%2C12%200%2C-1.5%207%2C0%200%2C1.5%20-2.875%2C3.5%20-1.25%2C0zm0%2C-12%200%2C1.5%207%2C0%200%2C-1.5%20-2.875%2C-3.5%20-1.25%2C0zm-9%2C12%200%2C-1.5%207%2C0%200%2C1.5%20-2.875%2C3.5%20-1.25%2C0zm0%2C-5.25%200%2C-1.5%202%2C-1.75%203%2C0%202%2C1.75%200%2C1.5%20-2%2C1.75%20-3%2C0z%22%20id=%22b%22%20fill=%22none%22/%3E%20%3C/defs%3E%20%3Cuse%20fill=%22%23000%22%20x=%220%22%20y=%220%22%20xlink:href=%22%23a%22/%3E%20%3Cuse%20stroke=%22%23000%22%20x=%220%22%20y=%220%22%20xlink:href=%22%23b%22/%3E%20%3Cuse%20fill=%22%2372777d%22%20x=%2227%22%20y=%220%22%20xlink:href=%22%23a%22/%3E%20%3Cuse%20stroke=%22%2372777d%22%20x=%2227%22%20y=%220%22%20xlink:href=%22%23b%22/%3E%20%3Cuse%20fill=%22%2336c%22%20x=%2254%22%20y=%220%22%20xlink:href=%22%23a%22/%3E%20%3Cuse%20stroke=%22%2336c%22%20x=%2254%22%20y=%220%22%20xlink:href=%22%23b%22/%3E%20%3Cuse%20fill=%22%23447ff5%22%20x=%2281%22%20y=%220%22%20xlink:href=%22%23a%22/%3E%20%3Cuse%20stroke=%22%23447ff5%22%20x=%2281%22%20y=%220%22%20xlink:href=%22%23b%22/%3E%20%3C/svg%3E");
}
.wbs-item-input, .wbs-prop-input, .wbs-target-input {
width: 250px;
min-height: 20px;
display: inline-block;
margin: 5px;
white-space: normal;
word-wrap: break-word;
}
.wbs-target-input {
display: inline-flex;
}
.wbs-prop-input {
width: 120px;
}
.wbs-qualifier .wbs-prop-input {
width: 85px;
}
.wbs-qualifier .wbs-target-input {
width: 120px;
}
/* Fix input rendering for Firefox */
.wbs-item-input input, .wbs-prop-input input, .wbs-target-input input,
.wbs-monolingual-container input {
width: 100%;
border: solid #b5b5b5 1px;
padding: 2px;
}
.wbs-draggable-column img {
float: right;
}
.wbs-toolbar {
float: right;
width: 100px;
padding: 2px;
}
.wbs-toolbar a {
cursor: pointer;
}
.wbs-remove,
.wbs-add-item, .wbs-add-statement-group, .wbs-add-statement,
.wbs-add-qualifier, .wbs-add-reference, .wbs-add-namedesc {
color: #0645ad !important;
font-size: 0.9em;
cursor: pointer;
}
.wbs-add-item:hover, .wbs-add-statement-group:hover, .wbs-add-statement:hover,
.wbs-add-qualifier:hover, .wbs-add-reference:hover, .wbs-add-namedesc:hover {
text-decoration: none;
}
.wbs-add-item span:hover, .wbs-add-statement-group span:hover, .wbs-add-statement span:hover,
.wbs-add-qualifier span:hover, .wbs-add-reference span:hover, .wbs-add-namedesc span:hover {
text-decoration: underline;
}
.wbs-remove {
display: inline-block;
}
.wbs-item-contents {
padding-left: 10px;
margin-left: 20px;
border-left: 3px solid gray;
}
.wbs-add-item b, .wbs-add-statement-group b, .wbs-add-statement b,
.wbs-add-qualifier b, .wbs-add-reference b, .wbs-add-namedesc b {
font-size: 1em;
color: grey;
width: 20px;
height: 20px;
}
.wbs-statement-group-container:empty:before {
content: attr(data-emptyplaceholder);
color: gray;
}
.schema-alignment-dialog-preview .wbs-qualifier .wbs-statement-container {
display: table-cell;
vertical-align: bottom;
padding-left: 10px;
height: 20px;
}
.wbs-statement-group-container, .wbs-statement-container, .wbs-qualifier-container, .wbs-reference-container {
width: 100%;
display: block;
overflow: auto;
}
.wbs-item, .wbs-statement-group, .wbs-statement, .wbs-qualifier, .wbs-reference {
display: block;
overflow: auto;
}
.wbs-item {
margin-bottom: 5px;
padding-right: 5px;
}
.wbs-statement-group {
background: #eaecf0;
margin-bottom: 5px;
border: 1px solid #c8ccd1;
}
.wbs-statement {
background: white;
border-bottom: 1px solid #eaecf0;
}
.wbs-statement:last-child {
border-bottom: 0;
}
.wbs-right {
float: right;
width: 85%;
}
.wbs-statement-group .wbs-right {
width: 75%;
}
.wbs-statement .wbs-right {
width: 90%;
}
.wbs-qualifier, .wbs-reference {
position: relative;
overflow-x: hidden;
}
.wbs-qualifier .wbs-right {
width: auto;
position: absolute;
top: 0px;
margin-left: 100px;
}
.wbs-reference > .wbs-right {
width: 100%;
}
.wbs-references-toggle {
width: 12em;
margin: 5px;
}
.wbs-references-toggle a {
color: #0645ad !important;
}
.wbs-references-toggle .triangle-icon {
background: transparent url(data:image/svg+xml,%3C%3Fxml%20version%3D%221.0%22%20encoding%3D%22UTF-8%22%20standalone%3D%22no%22%3F%3E%0A%3Csvg%0A%20%20%20xmlns%3Adc%3D%22http%3A//purl.org/dc/elements/1.1/%22%0A%20%20%20xmlns%3Acc%3D%22http%3A//creativecommons.org/ns%23%22%0A%20%20%20xmlns%3Ardf%3D%22http%3A//www.w3.org/1999/02/22-rdf-syntax-ns%23%22%0A%20%20%20xmlns%3Asvg%3D%22http%3A//www.w3.org/2000/svg%22%0A%20%20%20xmlns%3D%22http%3A//www.w3.org/2000/svg%22%0A%20%20%20id%3D%22svg8%22%0A%20%20%20version%3D%221.1%22%0A%20%20%20viewBox%3D%220%200%202.6458332%202.6458332%22%0A%20%20%20height%3D%2210%22%0A%20%20%20width%3D%2210%22%3E%0A%20%20%3Cdefs%0A%20%20%20%20%20id%3D%22defs2%22%20/%3E%0A%20%20%3Cmetadata%0A%20%20%20%20%20id%3D%22metadata5%22%3E%0A%20%20%20%20%3Crdf%3ARDF%3E%0A%20%20%20%20%20%20%3Ccc%3AWork%0A%20%20%20%20%20%20%20%20%20rdf%3Aabout%3D%22%22%3E%0A%20%20%20%20%20%20%20%20%3Cdc%3Aformat%3Eimage/svg+xml%3C/dc%3Aformat%3E%0A%20%20%20%20%20%20%20%20%3Cdc%3Atype%0A%20%20%20%20%20%20%20%20%20%20%20rdf%3Aresource%3D%22http%3A//purl.org/dc/dcmitype/StillImage%22%20/%3E%0A%20%20%20%20%20%20%20%20%3Cdc%3Atitle%3E%3C/dc%3Atitle%3E%0A%20%20%20%20%20%20%3C/cc%3AWork%3E%0A%20%20%20%20%3C/rdf%3ARDF%3E%0A%20%20%3C/metadata%3E%0A%20%20%3Cg%0A%20%20%20%20%20transform%3D%22translate%280%2C-294.35416%29%22%0A%20%20%20%20%20id%3D%22layer1%22%3E%0A%20%20%20%20%3Cpath%0A%20%20%20%20%20%20%20id%3D%22path12%22%0A%20%20%20%20%20%20%20d%3D%22m%200.21601809%2C294.5035%202.28763941%2C1.14744%20-2.29850705%2C1.20871%20z%22%0A%20%20%20%20%20%20%20style%3D%22fill%3A%23645eea%3Bfill-opacity%3A1%3Bfill-rule%3Aevenodd%3Bstroke%3A%23645eea%3Bstroke-width%3A0.06161711px%3Bstroke-linecap%3Abutt%3Bstroke-linejoin%3Amiter%3Bstroke-opacity%3A1%22%20/%3E%0A%20%20%3C/g%3E%0A%3C/svg%3E%0A) no-repeat center center;
background-size: 100% auto;
height: 8px;
width: 8px;
margin-right: 4px;
display: inline-block;
}
.pointing-right {
transform: rotate(0deg);
transition: .3s cubic-bezier(.17,.67,.21,1.69);
}
.pointing-down {
transform: rotate(90deg);
transition: .3s cubic-bezier(.17,.67,.21,1.69);
}
.wbs-qualifier .wbs-statement {
overflow: hidden;
background: transparent;
}
.wbs-monolingual-container, .wbs-quantity-container {
display: inline-block;
}
.wbs-reference-header {
height: 22px;
overflow: hidden;
display: block;
background-color: #b6c8ec;
}
.wbs-reference {
background-color: #eaf3ff;
margin-bottom: 5px;
}
.wbs-reference .wbs-statement {
background-color: #eaf3ff;
}
.wbs-namedesc-header, .wbs-statements-header {
font-size: 1.2em;
}
.wbs-namedesc {
border-left: 1px solid #c8ccd1;
border-right: 1px solid #c8ccd1;
}
.wbs-namedesc:first-child {
border-top: 1px solid #c8ccd1;
}
.wbs-namedesc:last-child {
border-bottom: 1px solid #c8ccd1;
}
.wbs-namedesc-container:empty:before {
content: attr(data-emptyplaceholder);
color: gray;
padding: 5px;
border: none;
}
.wbs-namedesc {
background-color: #eaecf0;
padding: 5px;
}
.wbs-namedesc-type, .wbs-namedesc-value {
display: inline-block;
}
.wbs-namedesc-value {
padding-left: 20px;
width: 300px;
}
.wbs-language-input {
width: 100%;
}
.schema-alignment-columns-header {
margin-bottom: 0.3em;
}
/*** Warnings rendering ****/
#wikidata-issues-panel table {
width: 100%;
}
.wb-warning h1 {
font-size: 1.2em;
}
.schema-issues-area table,
.perform-edits-warnings-area table {
width: 100%;
}
tr.wb-warning:nth-of-type(odd) {
background-color: #f2f2f2;
}
.wb-warning-count span,
.schema-alignment-total-warning-count {
color: white;
background-color: #777;
padding: 0px 5px;
border-radius: 0.5em;
}
.wb-warning-severity {
width: 60px;
height: 60px;
background-repeat: no-repeat;
background-position: center;
}
.wb-warning-severity-INFO {
background-image: url('../../images/Information.png');
}
.wb-warning-severity-WARNING {
background-image: url('../../images/Warning.png');
}
.wb-warning-severity-IMPORTANT {
background-image: url('../../images/Important.png');
}
.wb-warning-severity-CRITICAL {
background-image: url('../../images/Critical.png');
}
.wb-warning-body {
padding: 5px;
vertical-align: top;
}
.wb-warning-count {
padding: 5px;
vertical-align: middle;
text-align: center;
}
.wb-warning-end {
float: clear;
}
div.perform-edits-warnings-area {
min-height: 340px;
max-height: 400px;
overflow-x: hidden;
overflow-y: auto;
border: 1px solid #bbb;
}
.wb-issue-preformat {
border: 1px solid #eaecf0;
background-color: #f8f9fa;
padding: 1px 3px;
border-radius: 2px;
font-family: monospace;
}
.wb-preview-new-entity {
color: #11c;
}
/*** QuickStatements Preview ***/
div.schema-alignment-dialog-preview {
min-height: 340px;
max-width: 900px;
overflow: auto;
background: white;
padding: 10px;
margin-top: 3px;
white-space: pre;
font-size: 9pt;
}

32
extensions/wikidata/src/org/openrefine/wikidata/commands/CommandUtilities.java Normal file
View File

@ -0,0 +1,32 @@
package org.openrefine.wikidata.commands;
import java.io.IOException;
import java.io.Writer;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONWriter;
public class CommandUtilities {
/**
* Helper introduced to ease returning error messages from a response. Curiously
* this is not part of {@link Command}: the respond method uses the "status" JSON
* key instead of the "code" one required by the JS code.
*
* @param response
* @param errorMessage
* @throws IOException
*/
public static void respondError(HttpServletResponse response, String errorMessage)
throws IOException {
Writer w = response.getWriter();
JSONWriter writer = new JSONWriter(w);
writer.object();
writer.key("code"); writer.value("error");
writer.key("message"); writer.value(errorMessage);
writer.endObject();
w.flush();
w.close();
}
}

77
extensions/wikidata/src/org/openrefine/wikidata/commands/LoginCommand.java Normal file
View File

@ -0,0 +1,77 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.IOException;
import java.io.StringWriter;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import org.openrefine.wikidata.editing.ConnectionManager;
import com.google.refine.commands.Command;
public class LoginCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String username = request.getParameter("wb-username");
String password = request.getParameter("wb-password");
String remember = request.getParameter("remember-credentials");
ConnectionManager manager = ConnectionManager.getInstance();
if (username != null && password != null) {
manager.login(username, password, "on".equals(remember));
} else if ("true".equals(request.getParameter("logout"))) {
manager.logout();
}
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
StringWriter sb = new StringWriter(2048);
JSONWriter writer = new JSONWriter(sb);
try {
writer.object();
writer.key("logged_in");
writer.value(manager.isLoggedIn());
writer.key("username");
writer.value(manager.getUsername());
writer.endObject();
} catch (JSONException e) {
logger.error(e.getMessage());
}
respond(response, sb.toString());
}
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
doPost(request, response);
}
}

44
extensions/wikidata/src/org/openrefine/wikidata/commands/PerformWikibaseEditsCommand.java Normal file
View File

@ -0,0 +1,44 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONObject;
import org.openrefine.wikidata.operations.PerformWikibaseEditsOperation;
import com.google.refine.commands.EngineDependentCommand;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
public class PerformWikibaseEditsCommand extends EngineDependentCommand {
@Override
protected AbstractOperation createOperation(Project project, HttpServletRequest request, JSONObject engineConfig)
throws Exception {
String summary = request.getParameter("summary");
return new PerformWikibaseEditsOperation(engineConfig, summary);
}
}

140
extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java Normal file
View File

@ -0,0 +1,140 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.IOException;
import java.io.StringWriter;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.qa.EditInspector;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import static org.openrefine.wikidata.commands.CommandUtilities.respondError;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.browsing.Engine;
import com.google.refine.commands.Command;
import com.google.refine.model.Project;
public class PreviewWikibaseSchemaCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String jsonString = request.getParameter("schema");
WikibaseSchema schema = null;
if (jsonString != null) {
try {
schema = WikibaseSchema.reconstruct(jsonString);
} catch (JSONException e) {
respondError(response, "Wikibase schema could not be parsed.");
return;
}
} else {
schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema");
}
if (schema == null) {
respondError(response, "No Wikibase schema provided.");
return;
}
QAWarningStore warningStore = new QAWarningStore();
// Evaluate project
Engine engine = getEngine(request, project);
List<ItemUpdate> editBatch = schema.evaluate(project, engine, warningStore);
StringWriter sb = new StringWriter(2048);
JSONWriter writer = new JSONWriter(sb);
writer.object();
{
// Inspect the edits and generate warnings
EditInspector inspector = new EditInspector(warningStore);
inspector.inspect(editBatch);
writer.key("warnings");
writer.array();
for (QAWarning warning : warningStore.getWarnings()) {
warning.write(writer, new Properties());
}
writer.endArray();
// Add max warning level
writer.key("max_severity");
writer.value(warningStore.getMaxSeverity().toString());
// this is not the length of the warnings array written before,
// but the total number of issues raised (before deduplication)
writer.key("nb_warnings");
writer.value(warningStore.getNbWarnings());
// Dump the first 10 edits, scheduled with the default scheduler
WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler();
List<ItemUpdate> nonNullEdits = scheduler.schedule(editBatch).stream()
.filter(e -> !e.isNull())
.collect(Collectors.toList());
writer.key("edit_count");
writer.value(nonNullEdits.size());
List<ItemUpdate> firstEdits = nonNullEdits.stream()
.limit(10)
.collect(Collectors.toList());
ObjectMapper mapper = new ObjectMapper();
String firstEditsJson = mapper.writeValueAsString(firstEdits);
writer.key("edits_preview");
writer.value(new JSONArray(firstEditsJson));
}
writer.endObject();
respond(response, sb.toString());
} catch (Exception e) {
respondException(response, e);
}
}
}

78
extensions/wikidata/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommand.java Normal file
View File

@ -0,0 +1,78 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONObject;
import org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation;
import org.openrefine.wikidata.schema.WikibaseSchema;
import static org.openrefine.wikidata.commands.CommandUtilities.respondError;
import com.google.refine.commands.Command;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.process.Process;
import com.google.refine.util.ParsingUtilities;
public class SaveWikibaseSchemaCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
String jsonString = request.getParameter("schema");
if (jsonString == null) {
respondError(response, "No Wikibase schema provided.");
return;
}
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
WikibaseSchema schema = WikibaseSchema.reconstruct(json);
AbstractOperation op = new SaveWikibaseSchemaOperation(schema);
Process process = op.createProcess(project, new Properties());
performProcessAndRespond(request, response, project, process);
} catch (JSONException e) {
// We do not use respondException here because this is an expected
// exception which happens every time a user tries to save an incomplete
// schema - the exception should not be logged.
respondError(response, "Wikibase schema could not be parsed.");
} catch (Exception e) {
// This is an unexpected exception, so we log it.
respondException(response, e);
}
}
}

148
extensions/wikidata/src/org/openrefine/wikidata/editing/ConnectionManager.java Normal file
View File

@ -0,0 +1,148 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.io.IOException;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
import org.wikidata.wdtk.wikibaseapi.LoginFailedException;
import com.google.refine.ProjectManager;
import com.google.refine.preference.PreferenceStore;
/**
* Manages a connection to Wikidata, with login credentials stored in the
* preferences.
*
* Ideally, we should store only the cookies and not the password. But
* Wikidata-Toolkit does not allow for that as cookies are kept private.
*
* This class is also hard-coded for Wikidata: generalization to other Wikibase
* instances should be feasible though.
*
* @author Antonin Delpeuch
*/
public class ConnectionManager {
final static Logger logger = LoggerFactory.getLogger("connection_mananger");
public static final String PREFERENCE_STORE_KEY = "wikidata_credentials";
private PreferenceStore prefStore;
private ApiConnection connection;
private static final ConnectionManager instance = new ConnectionManager();
public static ConnectionManager getInstance() {
return instance;
}
private ConnectionManager() {
prefStore = ProjectManager.singleton.getPreferenceStore();
connection = null;
restoreSavedConnection();
}
public void login(String username, String password, boolean rememberCredentials) {
if (rememberCredentials) {
try {
JSONArray array = new JSONArray();
JSONObject obj = new JSONObject();
obj.put("username", username);
obj.put("password", password);
array.put(obj);
prefStore.put(PREFERENCE_STORE_KEY, array);
} catch (JSONException e) {
logger.error(e.getMessage());
}
}
connection = ApiConnection.getWikidataApiConnection();
try {
connection.login(username, password);
} catch (LoginFailedException e) {
connection = null;
}
}
public void restoreSavedConnection() {
JSONObject savedCredentials = getStoredCredentials();
if (savedCredentials != null) {
connection = ApiConnection.getWikidataApiConnection();
try {
connection.login(savedCredentials.getString("username"), savedCredentials.getString("password"));
} catch (LoginFailedException e) {
connection = null;
} catch (JSONException e) {
connection = null;
}
}
}
public JSONObject getStoredCredentials() {
JSONArray array = (JSONArray) prefStore.get(PREFERENCE_STORE_KEY);
if (array != null && array.length() > 0) {
try {
return array.getJSONObject(0);
} catch (JSONException e) {
logger.error(e.getMessage());
}
}
return null;
}
public void logout() {
prefStore.put(PREFERENCE_STORE_KEY, new JSONArray());
if (connection != null) {
try {
connection.logout();
connection = null;
} catch (IOException e) {
logger.error(e.getMessage());
}
}
}
public ApiConnection getConnection() {
return connection;
}
public boolean isLoggedIn() {
return connection != null;
}
public String getUsername() {
if (connection != null) {
return connection.getCurrentUser();
} else {
return null;
}
}
}

210
extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java Normal file
View File

@ -0,0 +1,210 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
/**
* Schedules and performs a list of updates to items via the API.
*
* @author Antonin Delpeuch
*
*/
public class EditBatchProcessor {
static final Logger logger = LoggerFactory.getLogger(EditBatchProcessor.class);
private WikibaseDataFetcher fetcher;
private WikibaseDataEditor editor;
private NewItemLibrary library;
private List<ItemUpdate> scheduled;
private String summary;
private List<ItemUpdate> remainingUpdates;
private List<ItemUpdate> currentBatch;
private int batchCursor;
private int globalCursor;
private Map<String, EntityDocument> currentDocs;
private int batchSize;
/**
* Initiates the process of pushing a batch of updates to Wikibase. This
* schedules the updates and is a prerequisite for calling
* {@link performOneEdit}.
*
* @param fetcher
* the fetcher to use to retrieve the current state of items
* @param editor
* the object to use to perform the edits
* @param updates
* the list of item updates to perform
* @param library
* the library to use to keep track of new item creation
* @param summary
* the summary to append to all edits
* @param batchSize
* the number of items that should be retrieved in one go from the
* API
*/
public EditBatchProcessor(WikibaseDataFetcher fetcher, WikibaseDataEditor editor, List<ItemUpdate> updates,
NewItemLibrary library, String summary, int batchSize) {
this.fetcher = fetcher;
this.editor = editor;
editor.setEditAsBot(true); // this will not do anything if the user does not
// have a bot flag, and this is generally wanted if they have one.
this.library = library;
this.summary = summary;
this.batchSize = batchSize;
// Schedule the edit batch
WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler();
this.scheduled = scheduler.schedule(updates);
this.globalCursor = 0;
this.batchCursor = 0;
this.remainingUpdates = new ArrayList<>(scheduled);
this.currentBatch = Collections.emptyList();
this.currentDocs = Collections.emptyMap();
}
/**
* Performs the next edit in the batch.
*
* @throws InterruptedException
*/
public void performEdit()
throws InterruptedException {
if (remainingEdits() == 0) {
return;
}
if (batchCursor == currentBatch.size()) {
prepareNewBatch();
}
ItemUpdate update = currentBatch.get(batchCursor);
// Rewrite mentions to new items
ReconEntityRewriter rewriter = new ReconEntityRewriter(library, update.getItemId());
update = rewriter.rewrite(update);
try {
// New item
if (update.isNew()) {
ReconEntityIdValue newCell = (ReconEntityIdValue) update.getItemId();
update = update.normalizeLabelsAndAliases();
ItemDocument itemDocument = Datamodel.makeItemDocument(update.getItemId(),
update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()), update.getAddedStatementGroups(),
Collections.emptyMap());
ItemDocument createdDoc = editor.createItemDocument(itemDocument, summary);
library.setQid(newCell.getReconInternalId(), createdDoc.getItemId().getId());
} else {
// Existing item
ItemDocument currentDocument = (ItemDocument) currentDocs.get(update.getItemId().getId());
editor.updateTermsStatements(currentDocument, update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()),
new ArrayList<MonolingualTextValue>(),
update.getAddedStatements().stream().collect(Collectors.toList()),
update.getDeletedStatements().stream().collect(Collectors.toList()), summary);
}
} catch (MediaWikiApiErrorException e) {
// TODO find a way to report these errors to the user in a nice way
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
batchCursor++;
}
/**
* @return the number of edits that remain to be done in the current batch
*/
public int remainingEdits() {
return scheduled.size() - (globalCursor + batchCursor);
}
/**
* @return the progress, measured as a percentage
*/
public int progress() {
return (100 * (globalCursor + batchCursor)) / scheduled.size();
}
protected void prepareNewBatch()
throws InterruptedException {
// remove the previous batch from the remainingUpdates
globalCursor += currentBatch.size();
currentBatch.clear();
if (remainingUpdates.size() < batchSize) {
currentBatch = remainingUpdates;
remainingUpdates = Collections.emptyList();
} else {
currentBatch = remainingUpdates.subList(0, batchSize);
}
List<String> qidsToFetch = currentBatch.stream().filter(u -> !u.isNew()).map(u -> u.getItemId().getId())
.collect(Collectors.toList());
// Get the current documents for this batch of updates
logger.info("Requesting documents");
currentDocs = null;
int retries = 3;
while (currentDocs == null && retries > 0) {
try {
currentDocs = fetcher.getEntityDocuments(qidsToFetch);
} catch (MediaWikiApiErrorException e) {
e.printStackTrace();
Thread.sleep(5000);
}
retries--;
}
if (currentDocs == null) {
throw new InterruptedException("Fetching current documents failed.");
}
batchCursor = 0;
}
}

159
extensions/wikidata/src/org/openrefine/wikidata/editing/NewItemLibrary.java Normal file
View File

@ -0,0 +1,159 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.ReconStats;
import com.google.refine.model.Row;
/**
* This keeps track of the new items that we have created for each internal
* reconciliation id.
*
* @author Antonin Delpeuch
*
*/
public class NewItemLibrary {
private Map<Long, String> map;
public NewItemLibrary() {
map = new HashMap<>();
}
@JsonCreator
public NewItemLibrary(@JsonProperty("qidMap") Map<Long, String> map) {
this.map = map;
}
/**
* Retrieves the Qid allocated to a given new cell
*
* @param id:
* the fake ItemId generated by the cell
* @return the qid (or null if unallocated yet)
*/
public String getQid(long id) {
return map.get(id);
}
/**
* Stores a Qid associated to a new cell
*
* @param id
* : the internal reconciliation id of the new cell
* @param qid
* : the associated Qid returned by Wikibase
*/
public void setQid(long id, String qid) {
map.put(id, qid);
}
/**
* Changes the "new" reconciled cells to their allocated qids for later use.
*
* @param reset:
* set to true to revert the operation (set cells to "new")
*/
public void updateReconciledCells(Project project, boolean reset) {
Set<Integer> impactedColumns = new HashSet<>();
/*
* Note that there is a slight violation of OpenRefine's model here: if we
* reconcile multiple cells to the same new item, and then perform this
* operation on a subset of the corresponding rows, we are going to modify cells
* that are outside the facet (because they are reconciled to the same cell).
* But I think this is the right thing to do.
*/
for (Row row : project.rows) {
for (int i = 0; i != row.cells.size(); i++) {
Cell cell = row.cells.get(i);
if (cell == null || cell.recon == null) {
continue;
}
Recon recon = cell.recon;
if (Recon.Judgment.New.equals(recon.judgment) && !reset
&& map.containsKey(recon.id)) {
recon.judgment = Recon.Judgment.Matched;
recon.match = new ReconCandidate(map.get(recon.id), cell.value.toString(),
new String[0], 100);
impactedColumns.add(i);
} else if (Recon.Judgment.Matched.equals(recon.judgment) && reset
&& map.containsKey(recon.id)) {
recon.judgment = Recon.Judgment.New;
recon.match = null;
impactedColumns.add(i);
}
}
}
// Update reconciliation statistics for impacted columns
for (Integer colId : impactedColumns) {
Column column = project.columnModel.getColumnByCellIndex(colId);
column.setReconStats(ReconStats.create(project, colId));
}
}
/**
* Getter, only meant to be used by Jackson
*
* @return the underlying map
*/
@JsonProperty("qidMap")
public Map<Long, String> getQidMap() {
return map;
}
@Override
public boolean equals(Object other) {
if (other == null || !NewItemLibrary.class.isInstance(other)) {
return false;
}
NewItemLibrary otherLibrary = (NewItemLibrary) other;
return map.equals(otherLibrary.getQidMap());
}
@Override
public int hashCode() {
return map.hashCode();
}
@Override
public String toString() {
return map.toString();
}
}

102
extensions/wikidata/src/org/openrefine/wikidata/editing/ReconEntityRewriter.java Normal file
View File

@ -0,0 +1,102 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.helpers.DatamodelConverter;
import org.wikidata.wdtk.datamodel.implementation.DataObjectFactoryImpl;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A class that rewrites an {@link ItemUpdate}, replacing reconciled entity id
* values by their concrete values after creation of all the new items involved.
*
* If an item has not been created yet, an {@link IllegalArgumentException} will
* be raised.
*
* The subject is treated as a special case: it is returned unchanged. This is
* because it is guaranteed not to appear in the update (but it does appear in
* the datamodel representation as the subject is passed around to the Claim
* objects its document contains).
*
* @author Antonin Delpeuch
*
*/
public class ReconEntityRewriter extends DatamodelConverter {
private NewItemLibrary library;
private ItemIdValue subject;
/**
* Constructor. Sets up a rewriter which uses the provided library to look up
* qids of new items, and the subject (which should not be rewritten).
*
* @param library
* @param subject
*/
public ReconEntityRewriter(NewItemLibrary library, ItemIdValue subject) {
super(new DataObjectFactoryImpl());
this.library = library;
this.subject = subject;
}
@Override
public ItemIdValue copy(ItemIdValue value) {
if (subject.equals(value)) {
return value;
}
if (value instanceof ReconItemIdValue) {
ReconItemIdValue recon = (ReconItemIdValue) value;
if (recon.isNew()) {
String newId = library.getQid(recon.getReconInternalId());
if (newId == null) {
throw new IllegalArgumentException(
"Trying to rewrite an update where a new item was not created yet.");
}
return Datamodel.makeItemIdValue(newId, recon.getRecon().identifierSpace);
}
}
return super.copy(value);
}
public ItemUpdate rewrite(ItemUpdate update) {
Set<MonolingualTextValue> labels = update.getLabels().stream().map(l -> copy(l)).collect(Collectors.toSet());
Set<MonolingualTextValue> descriptions = update.getDescriptions().stream().map(l -> copy(l))
.collect(Collectors.toSet());
Set<MonolingualTextValue> aliases = update.getAliases().stream().map(l -> copy(l)).collect(Collectors.toSet());
List<Statement> addedStatements = update.getAddedStatements().stream().map(l -> copy(l))
.collect(Collectors.toList());
Set<Statement> deletedStatements = update.getDeletedStatements().stream().map(l -> copy(l))
.collect(Collectors.toSet());
return new ItemUpdate(update.getItemId(), addedStatements, deletedStatements, labels, descriptions, aliases);
}
}

86
extensions/wikidata/src/org/openrefine/wikidata/editing/WikibaseCredentials.java Normal file
View File

@ -0,0 +1,86 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
/**
* This is just the necessary bits to store Wikidata credentials in OpenRefine's
* preference store.
*
* @author Antonin Delpeuch
*
*/
class WikibaseCredentials implements Jsonizable {
private String username;
private String password;
public WikibaseCredentials() {
username = null;
password = null;
}
public WikibaseCredentials(String username, String password) {
this.username = username;
this.password = password;
}
public String getUsername() {
return username;
}
public String getPassword() {
return password;
}
public boolean isNonNull() {
return username != null && password != null && !"null".equals(username) && !"null".equals(password);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("class");
writer.value(this.getClass().getName());
writer.key("username");
writer.value(username);
writer.key("password");
writer.value(password);
writer.endObject();
}
public static WikibaseCredentials load(JSONObject obj)
throws JSONException {
return new WikibaseCredentials(obj.getString("username"), obj.getString("password"));
}
}

106
extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java Normal file
View File

@ -0,0 +1,106 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import java.math.BigDecimal;
import java.util.Locale;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.updates.scheduler.QuickStatementsUpdateScheduler;
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
/**
* Prints a Wikibase value as a string as required by QuickStatements. Format
* documentation: https://www.wikidata.org/wiki/Help:QuickStatements
*
* Any new entity id will be assumed to be the last one created, represented
* with "LAST". It is fine to do this assumption because we are working on edit
* batches previously scheduled by {@link QuickStatementsUpdateScheduler}.
*
* @author Antonin Delpeuch
*
*/
public class QSValuePrinter implements ValueVisitor<String> {
@Override
public String visit(DatatypeIdValue value) {
// impossible case (this is actually a bug in WDTK, DatatypeIdValue should not subclass Value)
throw new IllegalArgumentException();
}
@Override
public String visit(EntityIdValue value) {
if (ReconEntityIdValue.class.isInstance(value) && ((ReconEntityIdValue) value).isNew()) {
return "LAST";
}
return value.getId();
}
@Override
public String visit(GlobeCoordinatesValue value) {
return String.format(Locale.US, "@%f/%f", value.getLatitude(), value.getLongitude());
}
@Override
public String visit(MonolingualTextValue value) {
return String.format("%s:\"%s\"", value.getLanguageCode(), value.getText());
}
@Override
public String visit(QuantityValue value) {
String unitPrefix = "http://www.wikidata.org/entity/Q";
String unitIri = value.getUnit();
String unitRepresentation = "", boundsRepresentation = "";
if (!unitIri.isEmpty()) {
if (!unitIri.startsWith(unitPrefix)) return null; // QuickStatements only accepts Qids as units
unitRepresentation = "U" + unitIri.substring(unitPrefix.length());
}
if (value.getLowerBound() != null) {
// bounds are always null at the same time so we know they are both not null
BigDecimal lowerBound = value.getLowerBound();
BigDecimal upperBound = value.getUpperBound();
boundsRepresentation = String.format(Locale.US, "[%s,%s]", lowerBound.toString(), upperBound.toString());
}
return String.format(Locale.US, "%s%s%s", value.getNumericValue().toString(), boundsRepresentation,
unitRepresentation);
}
@Override
public String visit(StringValue value) {
return "\"" + value.getString() + "\"";
}
@Override
public String visit(TimeValue value) {
return String.format("+%04d-%02d-%02dT%02d:%02d:%02dZ/%d", value.getYear(), value.getMonth(), value.getDay(),
value.getHour(), value.getMinute(), value.getSecond(), value.getPrecision());
}
}

191
extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java Normal file
View File

@ -0,0 +1,191 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import java.io.IOException;
import java.io.Writer;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.ImpossibleSchedulingException;
import org.openrefine.wikidata.updates.scheduler.QuickStatementsUpdateScheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
import com.google.refine.browsing.Engine;
import com.google.refine.exporters.WriterExporter;
import com.google.refine.model.Project;
public class QuickStatementsExporter implements WriterExporter {
final static Logger logger = LoggerFactory.getLogger("QuickStatementsExporter");
public static final String impossibleSchedulingErrorMessage = "This edit batch cannot be performed with QuickStatements due to the structure of its new items.";
public static final String noSchemaErrorMessage = "No schema was provided. You need to align your project with Wikidata first.";
public QuickStatementsExporter() {
}
@Override
public String getContentType() {
return "text/plain";
}
@Override
public void export(Project project, Properties options, Engine engine, Writer writer)
throws IOException {
WikibaseSchema schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema");
if (schema == null) {
writer.write(noSchemaErrorMessage);
} else {
translateSchema(project, engine, schema, writer);
}
}
/**
* Exports a project and a schema to a QuickStatements file
*
* @param project
* the project to translate
* @param engine
* the engine used for evaluation of the edits
* @param schema
* the WikibaseSchema used for translation of tabular data to edits
* @param writer
* the writer to which the QS should be written
* @throws IOException
*/
public void translateSchema(Project project, Engine engine, WikibaseSchema schema, Writer writer)
throws IOException {
List<ItemUpdate> items = schema.evaluate(project, engine);
translateItemList(items, writer);
}
public void translateItemList(List<ItemUpdate> updates, Writer writer)
throws IOException {
QuickStatementsUpdateScheduler scheduler = new QuickStatementsUpdateScheduler();
try {
List<ItemUpdate> scheduled = scheduler.schedule(updates);
for (ItemUpdate item : scheduled) {
translateItem(item, writer);
}
} catch (ImpossibleSchedulingException e) {
writer.write(impossibleSchedulingErrorMessage);
}
}
protected void translateNameDescr(String qid, Set<MonolingualTextValue> values, String prefix, ItemIdValue id,
Writer writer)
throws IOException {
for (MonolingualTextValue value : values) {
writer.write(qid + "\t");
writer.write(prefix);
writer.write(value.getLanguageCode());
writer.write("\t\"");
writer.write(value.getText());
writer.write("\"\n");
}
}
protected void translateItem(ItemUpdate item, Writer writer)
throws IOException {
String qid = item.getItemId().getId();
if (item.isNew()) {
writer.write("CREATE\n");
qid = "LAST";
item = item.normalizeLabelsAndAliases();
}
translateNameDescr(qid, item.getLabels(), "L", item.getItemId(), writer);
translateNameDescr(qid, item.getDescriptions(), "D", item.getItemId(), writer);
translateNameDescr(qid, item.getAliases(), "A", item.getItemId(), writer);
for (Statement s : item.getAddedStatements()) {
translateStatement(qid, s, s.getClaim().getMainSnak().getPropertyId().getId(), true, writer);
}
for (Statement s : item.getDeletedStatements()) {
translateStatement(qid, s, s.getClaim().getMainSnak().getPropertyId().getId(), false, writer);
}
}
protected void translateStatement(String qid, Statement statement, String pid, boolean add, Writer writer)
throws IOException {
Claim claim = statement.getClaim();
Value val = claim.getValue();
ValueVisitor<String> vv = new QSValuePrinter();
String targetValue = val.accept(vv);
if (targetValue != null) {
if (!add) {
writer.write("- ");
}
writer.write(qid + "\t" + pid + "\t" + targetValue);
for (SnakGroup q : claim.getQualifiers()) {
translateSnakGroup(q, false, writer);
}
for (Reference r : statement.getReferences()) {
for (SnakGroup g : r.getSnakGroups()) {
translateSnakGroup(g, true, writer);
}
break; // QS only supports one reference
}
writer.write("\n");
}
}
protected void translateSnakGroup(SnakGroup sg, boolean reference, Writer writer)
throws IOException {
for (Snak s : sg.getSnaks()) {
translateSnak(s, reference, writer);
}
}
protected void translateSnak(Snak s, boolean reference, Writer writer)
throws IOException {
String pid = s.getPropertyId().getId();
if (reference) {
pid = pid.replace('P', 'S');
}
Value val = s.getValue();
ValueVisitor<String> vv = new QSValuePrinter();
String valStr = val.accept(vv);
if (valStr != null) {
writer.write("\t" + pid + "\t" + valStr);
}
}
}

237
extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java Normal file
View File

@ -0,0 +1,237 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.List;
import java.util.Properties;
import java.util.Random;
import org.apache.commons.lang.Validate;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.editing.ConnectionManager;
import org.openrefine.wikidata.editing.EditBatchProcessor;
import org.openrefine.wikidata.editing.NewItemLibrary;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.util.WebResourceFetcherImpl;
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.browsing.Engine;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.process.LongRunningProcess;
import com.google.refine.process.Process;
import com.google.refine.util.Pool;
public class PerformWikibaseEditsOperation extends EngineDependentOperation {
static final Logger logger = LoggerFactory.getLogger(PerformWikibaseEditsOperation.class);
private String summary;
public PerformWikibaseEditsOperation(JSONObject engineConfig, String summary) {
super(engineConfig);
Validate.notNull(summary, "An edit summary must be provided.");
Validate.notEmpty(summary, "An edit summary must be provided.");
this.summary = summary;
}
static public AbstractOperation reconstruct(Project project, JSONObject obj)
throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
String summary = null;
if (obj.has("summary")) {
summary = obj.getString("summary");
}
return new PerformWikibaseEditsOperation(engineConfig, summary);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op");
writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description");
writer.value("Perform Wikibase edits");
writer.key("summary");
writer.value(summary);
writer.key("engineConfig");
writer.value(getEngineConfig());
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Peform edits on Wikidata";
}
@Override
public Process createProcess(Project project, Properties options)
throws Exception {
return new PerformEditsProcess(project, createEngine(project), getBriefDescription(project), summary);
}
static public class PerformWikibaseEditsChange implements Change {
private NewItemLibrary newItemLibrary;
public PerformWikibaseEditsChange(NewItemLibrary library) {
newItemLibrary = library;
}
@Override
public void apply(Project project) {
// we don't re-run changes on Wikidata
newItemLibrary.updateReconciledCells(project, false);
}
@Override
public void revert(Project project) {
// this does not do anything on Wikibase side -
// (we don't revert changes on Wikidata either)
newItemLibrary.updateReconciledCells(project, true);
}
@Override
public void save(Writer writer, Properties options)
throws IOException {
if (newItemLibrary != null) {
writer.write("newItems=");
ObjectMapper mapper = new ObjectMapper();
writer.write(mapper.writeValueAsString(newItemLibrary) + "\n");
}
writer.write("/ec/\n"); // end of change
}
static public Change load(LineNumberReader reader, Pool pool)
throws Exception {
NewItemLibrary library = new NewItemLibrary();
String line = null;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("newItems".equals(field)) {
ObjectMapper mapper = new ObjectMapper();
library = mapper.readValue(value, NewItemLibrary.class);
}
}
return new PerformWikibaseEditsChange(library);
}
}
public class PerformEditsProcess extends LongRunningProcess implements Runnable {
protected Project _project;
protected Engine _engine;
protected WikibaseSchema _schema;
protected String _summary;
protected final long _historyEntryID;
protected PerformEditsProcess(Project project, Engine engine, String description, String summary) {
super(description);
this._project = project;
this._engine = engine;
this._schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema");
this._summary = summary;
this._historyEntryID = HistoryEntry.allocateID();
}
@Override
public void run() {
WebResourceFetcherImpl.setUserAgent("OpenRefine Wikidata extension");
ConnectionManager manager = ConnectionManager.getInstance();
if (!manager.isLoggedIn()) {
return;
}
ApiConnection connection = manager.getConnection();
WikibaseDataFetcher wbdf = new WikibaseDataFetcher(connection, _schema.getBaseIri());
WikibaseDataEditor wbde = new WikibaseDataEditor(connection, _schema.getBaseIri());
// Generate batch token
long token = (new Random()).nextLong();
String summary = _summary + String.format(" ([[:toollabs:editgroups/b/OR/%s|details]])",
(Long.toHexString(token).substring(0, 7)));
// Evaluate the schema
List<ItemUpdate> itemDocuments = _schema.evaluate(_project, _engine);
// Prepare the edits
NewItemLibrary newItemLibrary = new NewItemLibrary();
EditBatchProcessor processor = new EditBatchProcessor(wbdf, wbde, itemDocuments, newItemLibrary, summary,
50);
// Perform edits
logger.info("Performing edits");
while (processor.remainingEdits() > 0) {
try {
processor.performEdit();
} catch (InterruptedException e) {
_canceled = true;
}
_progress = processor.progress();
if (_canceled) {
break;
}
}
_progress = 100;
if (!_canceled) {
Change change = new PerformWikibaseEditsChange(newItemLibrary);
HistoryEntry historyEntry = new HistoryEntry(_historyEntryID, _project, _description,
PerformWikibaseEditsOperation.this, change);
_project.history.addEntry(historyEntry);
_project.processManager.onDoneProcess(this);
}
}
@Override
protected Runnable getRunnable() {
return this;
}
}
}

161
extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java Normal file
View File

@ -0,0 +1,161 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.schema.WikibaseSchema;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
public class SaveWikibaseSchemaOperation extends AbstractOperation {
final public String operationDescription = "Save Wikibase schema";
final protected WikibaseSchema _schema;
public SaveWikibaseSchemaOperation(WikibaseSchema schema) {
this._schema = schema;
}
static public AbstractOperation reconstruct(Project project, JSONObject obj)
throws Exception {
return new SaveWikibaseSchemaOperation(WikibaseSchema.reconstruct(obj.getJSONObject("schema")));
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op");
writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description");
writer.value(operationDescription);
writer.key("schema");
_schema.write(writer, options);
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Save Wikibase schema skelton";
}
@Override
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID)
throws Exception {
String description = operationDescription;
Change change = new WikibaseSchemaChange(_schema);
return new HistoryEntry(historyEntryID, project, description, SaveWikibaseSchemaOperation.this, change);
}
static public class WikibaseSchemaChange implements Change {
final protected WikibaseSchema _newSchema;
protected WikibaseSchema _oldSchema = null;
public final static String overlayModelKey = "wikibaseSchema";
public WikibaseSchemaChange(WikibaseSchema newSchema) {
_newSchema = newSchema;
}
public void apply(Project project) {
synchronized (project) {
_oldSchema = (WikibaseSchema) project.overlayModels.get(overlayModelKey);
project.overlayModels.put(overlayModelKey, _newSchema);
}
}
public void revert(Project project) {
synchronized (project) {
if (_oldSchema == null) {
project.overlayModels.remove(overlayModelKey);
} else {
project.overlayModels.put(overlayModelKey, _oldSchema);
}
}
}
public void save(Writer writer, Properties options)
throws IOException {
writer.write("newSchema=");
writeWikibaseSchema(_newSchema, writer);
writer.write('\n');
writer.write("oldSchema=");
writeWikibaseSchema(_oldSchema, writer);
writer.write('\n');
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool)
throws Exception {
WikibaseSchema oldSchema = null;
WikibaseSchema newSchema = null;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("oldSchema".equals(field) && value.length() > 0) {
oldSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
} else if ("newSchema".equals(field) && value.length() > 0) {
newSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
}
}
WikibaseSchemaChange change = new WikibaseSchemaChange(newSchema);
change._oldSchema = oldSchema;
return change;
}
static protected void writeWikibaseSchema(WikibaseSchema s, Writer writer)
throws IOException {
if (s != null) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
s.write(jsonWriter, new Properties());
} catch (JSONException e) {
e.printStackTrace();
}
}
}
}
}

94
extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java Normal file
View File

@ -0,0 +1,94 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.Set;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
/**
* An object that fetches constraints about properties.
*
* @author Antonin Delpeuch
*
*/
public interface ConstraintFetcher {
/**
* Retrieves the regular expression for formatting a property, or null if there
* is no such constraint
*
* @param pid
* @return the expression of a regular expression which should be compatible
* with java.util.regex
*/
String getFormatRegex(PropertyIdValue pid);
/**
* Retrieves the property that is the inverse of a given property
*
* @param pid:
* the property to retrieve the inverse for
* @return the pid of the inverse property
*/
PropertyIdValue getInversePid(PropertyIdValue pid);
/**
* Is this property for values only?
*/
boolean isForValuesOnly(PropertyIdValue pid);
/**
* Is this property for qualifiers only?
*/
boolean isForQualifiersOnly(PropertyIdValue pid);
/**
* Is this property for references only?
*/
boolean isForReferencesOnly(PropertyIdValue pid);
/**
* Get the list of allowed qualifiers (as property ids) for this property (null
* if any)
*/
Set<PropertyIdValue> allowedQualifiers(PropertyIdValue pid);
/**
* Get the list of mandatory qualifiers (as property ids) for this property
* (null if any)
*/
Set<PropertyIdValue> mandatoryQualifiers(PropertyIdValue pid);
/**
* Is this property expected to have at most one value per item?
*/
boolean hasSingleValue(PropertyIdValue pid);
/**
* Is this property expected to have distinct values?
*/
boolean hasDistinctValues(PropertyIdValue pid);
}

124
extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java Normal file
View File

@ -0,0 +1,124 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.openrefine.wikidata.qa.scrutinizers.DistinctValuesScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.EditScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.FormatScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.InverseConstraintScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.NewItemScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.NoEditsMadeScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.QualifierCompatibilityScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.RestrictedPositionScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.SelfReferentialScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.SingleValueScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.UnsourcedScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.WhitespaceScrutinizer;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
/**
* Runs a collection of edit scrutinizers on an edit batch.
*
* @author Antonin Delpeuch
*
*/
public class EditInspector {
private Map<String, EditScrutinizer> scrutinizers;
private QAWarningStore warningStore;
private ConstraintFetcher fetcher;
public EditInspector(QAWarningStore warningStore) {
this.scrutinizers = new HashMap<>();
this.fetcher = new WikidataConstraintFetcher();
this.warningStore = warningStore;
// Register all known scrutinizers here
register(new NewItemScrutinizer());
register(new FormatScrutinizer());
register(new InverseConstraintScrutinizer());
register(new SelfReferentialScrutinizer());
register(new UnsourcedScrutinizer());
register(new RestrictedPositionScrutinizer());
register(new QualifierCompatibilityScrutinizer());
register(new SingleValueScrutinizer());
register(new DistinctValuesScrutinizer());
register(new NoEditsMadeScrutinizer());
register(new WhitespaceScrutinizer());
}
/**
* Adds a new scrutinizer to the inspector
*
* @param scrutinizer
*/
public void register(EditScrutinizer scrutinizer) {
String key = scrutinizer.getClass().getName();
scrutinizers.put(key, scrutinizer);
scrutinizer.setStore(warningStore);
scrutinizer.setFetcher(fetcher);
}
/**
* Inspect a batch of edits with the registered scrutinizers
*
* @param editBatch
*/
public void inspect(List<ItemUpdate> editBatch) {
// First, schedule them with some scheduler,
// so that all newly created entities appear in the batch
WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler();
editBatch = scheduler.schedule(editBatch);
Map<EntityIdValue, ItemUpdate> updates = ItemUpdate.groupBySubject(editBatch);
List<ItemUpdate> mergedUpdates = updates.values().stream().collect(Collectors.toList());
for (EditScrutinizer scrutinizer : scrutinizers.values()) {
scrutinizer.batchIsBeginning();
}
for(ItemUpdate update : mergedUpdates) {
if(!update.isNull()) {
for (EditScrutinizer scrutinizer : scrutinizers.values()) {
scrutinizer.scrutinize(update);
}
}
}
for(EditScrutinizer scrutinizer : scrutinizers.values()) {
scrutinizer.batchIsFinished();
}
if (warningStore.getNbWarnings() == 0) {
warningStore.addWarning(new QAWarning("no-issue-detected", null, QAWarning.Severity.INFO, 0));
}
}
}

167
extensions/wikidata/src/org/openrefine/wikidata/qa/QAWarning.java Normal file
View File

@ -0,0 +1,167 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.utils.JacksonJsonizable;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A class to represent a QA warning emitted by the Wikidata schema This could
* probably be reused at a broader scale, for instance for Data Package
* validation.
*
* @author Antonin Delpeuch
*
*/
public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning> {
public enum Severity {
INFO, // We just report something to the user but it is probably fine
WARNING, // Edits that look wrong but in some cases they are actually fine
IMPORTANT, // There is almost surely something wrong about the edit but in rare cases we
// might want to allow it
CRITICAL, // We should never edit if there is a critical issue
}
/// The type of QA warning emitted
private final String type;
// The key for aggregation of other QA warnings together - this specializes the
// id
private final String bucketId;
// The severity of the issue
private final Severity severity;
// The number of times this issue was found
private final int count;
// Other details about the warning, that can be displayed to the user
private final Map<String, Object> properties;
public QAWarning(String type, String bucketId, Severity severity, int count) {
Validate.notNull(type);
this.type = type;
this.bucketId = bucketId;
Validate.notNull(severity);
this.severity = severity;
this.count = count;
this.properties = new HashMap<>();
}
/**
* @return the full key for aggregation of QA warnings
*/
@JsonIgnore
public String getAggregationId() {
if (this.bucketId != null) {
return this.type + "_" + this.bucketId;
} else {
return this.type;
}
}
/**
* Aggregates another QA warning of the same aggregation id.
*
* @param other
*/
public QAWarning aggregate(QAWarning other) {
assert other.getAggregationId().equals(getAggregationId());
int newCount = count + other.getCount();
Severity newSeverity = severity;
if (other.getSeverity().compareTo(severity) > 0) {
newSeverity = other.getSeverity();
}
QAWarning merged = new QAWarning(getType(), getBucketId(), newSeverity, newCount);
for (Entry<String, Object> entry : properties.entrySet()) {
merged.setProperty(entry.getKey(), entry.getValue());
}
for (Entry<String, Object> entry : other.getProperties().entrySet()) {
merged.setProperty(entry.getKey(), entry.getValue());
}
return merged;
}
/**
* Sets a property of the QA warning, to be used by the front-end for display.
*
* @param key:
* the name of the property
* @param value
* should be Jackson-serializable
*/
public void setProperty(String key, Object value) {
this.properties.put(key, value);
}
@JsonProperty("type")
public String getType() {
return type;
}
@JsonProperty("bucketId")
public String getBucketId() {
return bucketId;
}
@JsonProperty("severity")
public Severity getSeverity() {
return severity;
}
@JsonProperty("count")
public int getCount() {
return count;
}
@JsonProperty("properties")
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public Map<String, Object> getProperties() {
return properties;
}
/**
* Warnings are sorted by decreasing severity.
*/
@Override
public int compareTo(QAWarning other) {
return -severity.compareTo(other.getSeverity());
}
@Override
public boolean equals(Object other) {
if (other == null || !QAWarning.class.isInstance(other)) {
return false;
}
QAWarning otherWarning = (QAWarning) other;
return type.equals(otherWarning.getType()) && bucketId.equals(otherWarning.getBucketId())
&& severity.equals(otherWarning.getSeverity()) && count == otherWarning.getCount()
&& properties.equals(otherWarning.getProperties());
}
}

99
extensions/wikidata/src/org/openrefine/wikidata/qa/QAWarningStore.java Normal file
View File

@ -0,0 +1,99 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A store for QA warnings which aggregates them by type.
*
* @author Antonin Delpeuch
*/
public class QAWarningStore {
@JsonIgnore
private Map<String, QAWarning> map;
@JsonIgnore
private QAWarning.Severity maxSeverity;
@JsonIgnore
private int totalWarnings;
public QAWarningStore() {
this.map = new HashMap<>();
this.maxSeverity = QAWarning.Severity.INFO;
}
/**
* Stores a warning, aggregating it with any existing
*
* @param warning
*/
public void addWarning(QAWarning warning) {
String aggregationKey = warning.getAggregationId();
QAWarning.Severity severity = warning.getSeverity();
if (severity.compareTo(maxSeverity) > 0) {
maxSeverity = severity;
}
totalWarnings += warning.getCount();
if (map.containsKey(aggregationKey)) {
QAWarning existing = map.get(aggregationKey);
map.put(aggregationKey, existing.aggregate(warning));
} else {
map.put(aggregationKey, warning);
}
}
/**
* Returns the list of aggregated warnings, ordered by decreasing severity
*/
@JsonProperty("warnings")
public List<QAWarning> getWarnings() {
List<QAWarning> result = new ArrayList<>(map.values());
Collections.sort(result);
return result;
}
/**
* Returns the maximum severity of the stored warnings (INFO if empty)
*/
@JsonProperty("max_severity")
public QAWarning.Severity getMaxSeverity() {
return maxSeverity;
}
/**
* Returns the total number of warnings
*/
@JsonProperty("nb_warnings")
public int getNbWarnings() {
return totalWarnings;
}
}

221
extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java Normal file
View File

@ -0,0 +1,221 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.openrefine.wikidata.utils.EntityCache;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* This class provides an abstraction over the way constraint definitions are
* stored in Wikidata.
*
* @author Antonin Delpeuch
*
*/
public class WikidataConstraintFetcher implements ConstraintFetcher {
public static String WIKIDATA_CONSTRAINT_PID = "P2302";
public static String FORMAT_CONSTRAINT_QID = "Q21502404";
public static String FORMAT_REGEX_PID = "P1793";
public static String INVERSE_CONSTRAINT_QID = "Q21510855";
public static String INVERSE_PROPERTY_PID = "P2306";
public static String USED_ONLY_AS_VALUES_CONSTRAINT_QID = "Q21528958";
public static String USED_ONLY_AS_QUALIFIER_CONSTRAINT_QID = "Q21510863";
public static String USED_ONLY_AS_REFERENCE_CONSTRAINT_QID = "Q21528959";
public static String ALLOWED_QUALIFIERS_CONSTRAINT_QID = "Q21510851";
public static String ALLOWED_QUALIFIERS_CONSTRAINT_PID = "P2306";
public static String MANDATORY_QUALIFIERS_CONSTRAINT_QID = "Q21510856";
public static String MANDATORY_QUALIFIERS_CONSTRAINT_PID = "P2306";
public static String SINGLE_VALUE_CONSTRAINT_QID = "Q19474404";
public static String DISTINCT_VALUES_CONSTRAINT_QID = "Q21502410";
// The following constraints still need to be implemented:
public static String TYPE_CONSTRAINT_QID = "Q21503250";
@Override
public String getFormatRegex(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, FORMAT_CONSTRAINT_QID);
if (specs != null) {
List<Value> regexes = findValues(specs, FORMAT_REGEX_PID);
if (!regexes.isEmpty()) {
return ((StringValue) regexes.get(0)).getString();
}
}
return null;
}
@Override
public PropertyIdValue getInversePid(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, INVERSE_CONSTRAINT_QID);
if (specs != null) {
List<Value> inverses = findValues(specs, INVERSE_PROPERTY_PID);
if (!inverses.isEmpty()) {
return (PropertyIdValue) inverses.get(0);
}
}
return null;
}
@Override
public boolean isForValuesOnly(PropertyIdValue pid) {
return getSingleConstraint(pid, USED_ONLY_AS_VALUES_CONSTRAINT_QID) != null;
}
@Override
public boolean isForQualifiersOnly(PropertyIdValue pid) {
return getSingleConstraint(pid, USED_ONLY_AS_QUALIFIER_CONSTRAINT_QID) != null;
}
@Override
public boolean isForReferencesOnly(PropertyIdValue pid) {
return getSingleConstraint(pid, USED_ONLY_AS_REFERENCE_CONSTRAINT_QID) != null;
}
@Override
public Set<PropertyIdValue> allowedQualifiers(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, ALLOWED_QUALIFIERS_CONSTRAINT_QID);
if (specs != null) {
List<Value> properties = findValues(specs, ALLOWED_QUALIFIERS_CONSTRAINT_PID);
return properties.stream().map(e -> (PropertyIdValue) e).collect(Collectors.toSet());
}
return null;
}
@Override
public Set<PropertyIdValue> mandatoryQualifiers(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, MANDATORY_QUALIFIERS_CONSTRAINT_QID);
if (specs != null) {
List<Value> properties = findValues(specs, MANDATORY_QUALIFIERS_CONSTRAINT_PID);
return properties.stream().map(e -> (PropertyIdValue) e).collect(Collectors.toSet());
}
return null;
}
@Override
public boolean hasSingleValue(PropertyIdValue pid) {
return getSingleConstraint(pid, SINGLE_VALUE_CONSTRAINT_QID) != null;
}
@Override
public boolean hasDistinctValues(PropertyIdValue pid) {
return getSingleConstraint(pid, DISTINCT_VALUES_CONSTRAINT_QID) != null;
}
/**
* Returns a single constraint for a particular type and a property, or null if
* there is no such constraint
*
* @param pid:
* the property to retrieve the constraints for
* @param qid:
* the type of the constraints
* @return the list of qualifiers for the constraint, or null if it does not
* exist
*/
protected List<SnakGroup> getSingleConstraint(PropertyIdValue pid, String qid) {
Statement statement = getConstraintsByType(pid, qid).findFirst().orElse(null);
if (statement != null) {
return statement.getClaim().getQualifiers();
}
return null;
}
/**
* Gets the list of constraints of a particular type for a property
*
* @param pid:
* the property to retrieve the constraints for
* @param qid:
* the type of the constraints
* @return the stream of matching constraint statements
*/
protected Stream<Statement> getConstraintsByType(PropertyIdValue pid, String qid) {
Stream<Statement> allConstraints = getConstraintStatements(pid).stream()
.filter(s -> ((EntityIdValue) s.getValue()).getId().equals(qid));
return allConstraints;
}
/**
* Gets all the constraint statements for a given property
*
* @param pid
* : the id of the property to retrieve the constraints for
* @return the list of constraint statements
*/
protected List<Statement> getConstraintStatements(PropertyIdValue pid) {
PropertyDocument doc = (PropertyDocument) EntityCache.getEntityDocument(pid);
StatementGroup group = doc.findStatementGroup(WIKIDATA_CONSTRAINT_PID);
if (group != null) {
return group.getStatements();
} else {
return new ArrayList<Statement>();
}
}
/**
* Returns the values of a given property in qualifiers
*
* @param groups:
* the qualifiers
* @param pid:
* the property to filter on
* @return
*/
protected List<Value> findValues(List<SnakGroup> groups, String pid) {
List<Value> results = new ArrayList<>();
for (SnakGroup group : groups) {
if (group.getProperty().getId().equals(pid)) {
for (Snak snak : group.getSnaks())
results.add(snak.getValue());
}
}
return results;
}
}

75
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DistinctValuesScrutinizer.java Normal file
View File

@ -0,0 +1,75 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.Map;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that checks for properties using the same value on different
* items.
*
* @author Antonin Delpeuch
*
*/
public class DistinctValuesScrutinizer extends StatementScrutinizer {
public final static String type = "identical-values-for-distinct-valued-property";
private Map<PropertyIdValue, Map<Value, EntityIdValue>> _seenValues;
public DistinctValuesScrutinizer() {
_seenValues = new HashMap<>();
}
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId();
if (_fetcher.hasDistinctValues(pid)) {
Value mainSnakValue = statement.getClaim().getMainSnak().getValue();
Map<Value, EntityIdValue> seen = _seenValues.get(pid);
if (seen == null) {
seen = new HashMap<Value, EntityIdValue>();
_seenValues.put(pid, seen);
}
if (seen.containsKey(mainSnakValue)) {
EntityIdValue otherId = seen.get(mainSnakValue);
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("item1_entity", entityId);
issue.setProperty("item2_entity", otherId);
addIssue(issue);
} else {
seen.put(mainSnakValue, entityId);
}
}
}
}

128
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java Normal file
View File

@ -0,0 +1,128 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.ConstraintFetcher;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.qa.QAWarning.Severity;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.updates.ItemUpdate;
/**
* Inspects an edit batch and emits warnings.
*
* @author Antonin Delpeuch
*/
public abstract class EditScrutinizer {
protected QAWarningStore _store;
protected ConstraintFetcher _fetcher;
public EditScrutinizer() {
_fetcher = null;
_store = null;
}
public void setStore(QAWarningStore store) {
_store = store;
}
public void setFetcher(ConstraintFetcher fetcher) {
_fetcher = fetcher;
}
/**
* Called before an edit batch is scrutinized.
*/
public void batchIsBeginning() {
}
/**
* Reads the candidate edits and emits warnings in the store
*
* @param edit:
* the list of ItemUpdates to scrutinize
*/
public abstract void scrutinize(ItemUpdate edit);
/**
* Method called once the edit batch has been read entirely
*/
public void batchIsFinished() {
}
/**
* Emits an issue that will be reported to the user,
* after merging with other issues of the same kind.
*
* @param warning
* the issue to report
*/
protected void addIssue(QAWarning warning) {
_store.addWarning(warning);
}
protected void addIssue(String type, String aggregationId, Severity severity, int count) {
addIssue(new QAWarning(type, aggregationId, severity, count));
}
/**
* Helper to be used by subclasses to emit simple INFO warnings
*
* @param warning
*/
protected void info(String type) {
addIssue(type, null, QAWarning.Severity.INFO, 1);
}
/**
* Helper to be used by subclasses to emit simple warnings
*
* @param warning
*/
protected void warning(String type) {
addIssue(type, null, QAWarning.Severity.WARNING, 1);
}
/**
* Helper to be used by subclasses to emit simple important warnings
*
* @param warning
*/
protected void important(String type) {
addIssue(type, null, QAWarning.Severity.IMPORTANT, 1);
}
/**
* Helper to be used by subclasses to emit simple critical warnings
*
* @param warning
*/
protected void critical(String type) {
addIssue(type, null, QAWarning.Severity.CRITICAL, 1);
}
}

100
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/FormatScrutinizer.java Normal file
View File

@ -0,0 +1,100 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
/**
* A scrutinizer that detects incorrect formats in text values (mostly
* identifiers).
*
* @author Antonin Delpeuch
*
*/
public class FormatScrutinizer extends SnakScrutinizer {
public static final String type = "add-statements-with-invalid-format";
private Map<PropertyIdValue, Pattern> _patterns;
public FormatScrutinizer() {
_patterns = new HashMap<>();
}
/**
* Loads the regex for a property and compiles it to a pattern (this is cached
* upstream, plus we are doing it only once per property and batch).
*
* @param pid
* the id of the property to fetch the constraints for
* @return
*/
protected Pattern getPattern(PropertyIdValue pid) {
if (_patterns.containsKey(pid)) {
return _patterns.get(pid);
} else {
String regex = _fetcher.getFormatRegex(pid);
Pattern pattern = null;
if (regex != null) {
pattern = Pattern.compile(regex);
}
_patterns.put(pid, pattern);
return pattern;
}
}
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
if (StringValue.class.isInstance(snak.getValue())) {
String value = ((StringValue) snak.getValue()).getString();
PropertyIdValue pid = snak.getPropertyId();
Pattern pattern = getPattern(pid);
if (pattern == null) {
return;
}
if (!pattern.matcher(value).matches()) {
if (added) {
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("regex", pattern.toString());
issue.setProperty("example_value", value);
issue.setProperty("example_item_entity", entityId);
addIssue(issue);
} else {
info("remove-statements-with-invalid-format");
}
}
}
}
}

122
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java Normal file
View File

@ -0,0 +1,122 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that checks for missing inverse statements in edit batches.
*
* @author Antonin Delpeuch
*
*/
public class InverseConstraintScrutinizer extends StatementScrutinizer {
public static final String type = "missing-inverse-statements";
private Map<PropertyIdValue, PropertyIdValue> _inverse;
private Map<PropertyIdValue, Map<EntityIdValue, Set<EntityIdValue>>> _statements;
public InverseConstraintScrutinizer() {
_inverse = new HashMap<>();
_statements = new HashMap<>();
}
protected PropertyIdValue getInverseConstraint(PropertyIdValue pid) {
if (_inverse.containsKey(pid)) {
return _inverse.get(pid);
} else {
PropertyIdValue inversePid = _fetcher.getInversePid(pid);
_inverse.put(pid, inversePid);
_statements.put(pid, new HashMap<EntityIdValue, Set<EntityIdValue>>());
// We are doing this check because we do not have any guarantee that
// the inverse constraints are consistent on Wikidata.
if (inversePid != null && !_inverse.containsKey(inversePid)) {
_inverse.put(inversePid, pid);
_statements.put(inversePid, new HashMap<EntityIdValue, Set<EntityIdValue>>());
}
return inversePid;
}
}
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
if (!added) {
return; // TODO support for deleted statements
}
Value mainSnakValue = statement.getClaim().getMainSnak().getValue();
if (ItemIdValue.class.isInstance(mainSnakValue)) {
PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId();
PropertyIdValue inversePid = getInverseConstraint(pid);
if (inversePid != null) {
EntityIdValue targetEntityId = (EntityIdValue) mainSnakValue;
Set<EntityIdValue> currentValues = _statements.get(pid).get(entityId);
if (currentValues == null) {
currentValues = new HashSet<EntityIdValue>();
_statements.get(pid).put(entityId, currentValues);
}
currentValues.add(targetEntityId);
}
}
}
@Override
public void batchIsFinished() {
// For each pair of inverse properties (in each direction)
for (Entry<PropertyIdValue, PropertyIdValue> propertyPair : _inverse.entrySet()) {
// Get the statements made for the first
PropertyIdValue ourProperty = propertyPair.getKey();
for (Entry<EntityIdValue, Set<EntityIdValue>> itemLinks : _statements.get(ourProperty).entrySet()) {
// For each outgoing link
for (EntityIdValue idValue : itemLinks.getValue()) {
// Check that they are in the statements made for the second
PropertyIdValue missingProperty = propertyPair.getValue();
Set<EntityIdValue> reciprocalLinks = _statements.get(missingProperty).get(idValue);
if (reciprocalLinks == null || !reciprocalLinks.contains(itemLinks.getKey())) {
QAWarning issue = new QAWarning(type, ourProperty.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("added_property_entity", ourProperty);
issue.setProperty("inverse_property_entity", missingProperty);
issue.setProperty("source_entity", itemLinks.getKey());
issue.setProperty("target_entity", idValue);
addIssue(issue);
}
}
}
}
}
}

83
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java Normal file
View File

@ -0,0 +1,83 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
/**
* A scrutinizer that inspects new items.
*
* @author Antonin Delpeuch
*/
public class NewItemScrutinizer extends EditScrutinizer {
public static final String noLabelType = "new-item-without-labels-or-aliases";
public static final String noDescType = "new-item-without-descriptions";
public static final String deletedStatementsType = "new-item-with-deleted-statements";
public static final String noTypeType = "new-item-without-P31-or-P279";
public static final String newItemType = "new-item-created";
@Override
public void scrutinize(ItemUpdate update) {
if (update.isNew()) {
info(newItemType);
if (update.getLabels().isEmpty() && update.getAliases().isEmpty()) {
QAWarning issue = new QAWarning(noLabelType, null, QAWarning.Severity.CRITICAL, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
if (update.getDescriptions().isEmpty()) {
QAWarning issue = new QAWarning(noDescType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
if (!update.getDeletedStatements().isEmpty()) {
QAWarning issue = new QAWarning(deletedStatementsType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
// Try to find a "instance of" or "subclass of" claim
boolean typeFound = false;
for (StatementGroup group : update.getAddedStatementGroups()) {
String pid = group.getProperty().getId();
if ("P31".equals(pid) || "P279".equals(pid)) {
typeFound = true;
break;
}
}
if (!typeFound) {
QAWarning issue = new QAWarning(noTypeType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
}
}
}

51
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java Normal file
View File

@ -0,0 +1,51 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.updates.ItemUpdate;
public class NoEditsMadeScrutinizer extends EditScrutinizer {
public static final String type = "no-edit-generated";
private boolean nonNullUpdateSeen = false;
@Override
public void batchIsBeginning() {
nonNullUpdateSeen = false;
}
@Override
public void scrutinize(ItemUpdate edit) {
nonNullUpdateSeen = true;
}
@Override
public void batchIsFinished() {
if(!nonNullUpdateSeen) {
info(type);
}
}
}

110
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QualifierCompatibilityScrutinizer.java Normal file
View File

@ -0,0 +1,110 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer that checks the compatibility of the qualifiers and the
* property of a statement, and looks for mandatory qualifiers.
*
* @author Antonin Delpeuch
*/
public class QualifierCompatibilityScrutinizer extends StatementScrutinizer {
public static final String missingMandatoryQualifiersType = "missing-mandatory-qualifiers";
public static final String disallowedQualifiersType = "disallowed-qualifiers";
private Map<PropertyIdValue, Set<PropertyIdValue>> _allowedQualifiers;
private Map<PropertyIdValue, Set<PropertyIdValue>> _mandatoryQualifiers;
public QualifierCompatibilityScrutinizer() {
_allowedQualifiers = new HashMap<>();
_mandatoryQualifiers = new HashMap<>();
}
protected boolean qualifierIsAllowed(PropertyIdValue statementProperty, PropertyIdValue qualifierProperty) {
Set<PropertyIdValue> allowed = null;
if (_allowedQualifiers.containsKey(statementProperty)) {
allowed = _allowedQualifiers.get(statementProperty);
} else {
allowed = _fetcher.allowedQualifiers(statementProperty);
_allowedQualifiers.put(statementProperty, allowed);
}
return allowed == null || allowed.contains(qualifierProperty);
}
protected Set<PropertyIdValue> mandatoryQualifiers(PropertyIdValue statementProperty) {
Set<PropertyIdValue> mandatory = null;
if (_mandatoryQualifiers.containsKey(statementProperty)) {
mandatory = _mandatoryQualifiers.get(statementProperty);
} else {
mandatory = _fetcher.mandatoryQualifiers(statementProperty);
if (mandatory == null) {
mandatory = new HashSet<>();
}
_mandatoryQualifiers.put(statementProperty, mandatory);
}
return mandatory;
}
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
PropertyIdValue statementProperty = statement.getClaim().getMainSnak().getPropertyId();
Set<PropertyIdValue> qualifiers = statement.getClaim().getQualifiers().stream().map(e -> e.getProperty())
.collect(Collectors.toSet());
Set<PropertyIdValue> missingQualifiers = mandatoryQualifiers(statementProperty).stream()
.filter(p -> !qualifiers.contains(p)).collect(Collectors.toSet());
Set<PropertyIdValue> disallowedQualifiers = qualifiers.stream()
.filter(p -> !qualifierIsAllowed(statementProperty, p)).collect(Collectors.toSet());
for (PropertyIdValue missing : missingQualifiers) {
QAWarning issue = new QAWarning(missingMandatoryQualifiersType,
statementProperty.getId() + "-" + missing.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("statement_property_entity", statementProperty);
issue.setProperty("missing_property_entity", missing);
issue.setProperty("example_item_entity", entityId);
addIssue(issue);
}
for (PropertyIdValue disallowed : disallowedQualifiers) {
QAWarning issue = new QAWarning(disallowedQualifiersType,
statementProperty.getId() + "-" + disallowed.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("statement_property_entity", statementProperty);
issue.setProperty("disallowed_property_entity", disallowed);
issue.setProperty("example_item_entity", entityId);
addIssue(issue);
}
}
}

114
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedPositionScrutinizer.java Normal file
View File

@ -0,0 +1,114 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
public class RestrictedPositionScrutinizer extends StatementScrutinizer {
protected enum SnakPosition {
MAINSNAK, QUALIFIER, REFERENCE
}
private Map<PropertyIdValue, SnakPosition> _restrictedPids;
private Set<PropertyIdValue> _unrestrictedPids;
public RestrictedPositionScrutinizer() {
_restrictedPids = new HashMap<>();
_unrestrictedPids = new HashSet<>();
}
SnakPosition positionRestriction(PropertyIdValue pid) {
if (_unrestrictedPids.contains(pid)) {
return null;
}
SnakPosition restriction = _restrictedPids.get(pid);
if (restriction != null) {
return restriction;
} else {
if (_fetcher.isForValuesOnly(pid)) {
restriction = SnakPosition.MAINSNAK;
} else if (_fetcher.isForQualifiersOnly(pid)) {
restriction = SnakPosition.QUALIFIER;
} else if (_fetcher.isForReferencesOnly(pid)) {
restriction = SnakPosition.REFERENCE;
}
// Cache these results:
if (restriction != null) {
_restrictedPids.put(pid, restriction);
} else {
_unrestrictedPids.add(pid);
}
return restriction;
}
}
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
// Skip the main snak
scrutinize(statement.getClaim().getMainSnak(), entityId, SnakPosition.MAINSNAK, added);
// Qualifiers
scrutinizeSnakSet(statement.getClaim().getAllQualifiers(), entityId, SnakPosition.QUALIFIER, added);
// References
for (Reference ref : statement.getReferences()) {
scrutinizeSnakSet(ref.getAllSnaks(), entityId, SnakPosition.REFERENCE, added);
}
}
protected void scrutinizeSnakSet(Iterator<Snak> snaks, EntityIdValue entityId, SnakPosition position,
boolean added) {
while (snaks.hasNext()) {
Snak snak = snaks.next();
scrutinize(snak, entityId, position, added);
}
}
public void scrutinize(Snak snak, EntityIdValue entityId, SnakPosition position, boolean added) {
SnakPosition restriction = positionRestriction(snak.getPropertyId());
if (restriction != null && position != restriction) {
String positionStr = position.toString().toLowerCase();
String restrictionStr = restriction.toString().toLowerCase();
QAWarning issue = new QAWarning("property-restricted-to-" + restrictionStr + "-found-in-" + positionStr,
snak.getPropertyId().getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", snak.getPropertyId());
addIssue(issue);
}
}
}

50
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SelfReferentialScrutinizer.java Normal file
View File

@ -0,0 +1,50 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
/**
* A scrutinizer that checks for self-referential statements. These statements
* are flagged by Wikibase as suspicious.
*
* @author Antonin Delpeuch
*
*/
public class SelfReferentialScrutinizer extends SnakScrutinizer {
public static final String type = "self-referential-statements";
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
if (entityId.equals(snak.getValue())) {
QAWarning issue = new QAWarning(type, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", entityId);
addIssue(issue);
}
}
}

63
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java Normal file
View File

@ -0,0 +1,63 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashSet;
import java.util.Set;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* For now this scrutinizer only checks for uniqueness at the item level (it
* ignores qualifiers and references).
*
* @author Antonin Delpeuch
*
*/
public class SingleValueScrutinizer extends EditScrutinizer {
public static final String type = "single-valued-property-added-more-than-once";
@Override
public void scrutinize(ItemUpdate update) {
Set<PropertyIdValue> seenSingleProperties = new HashSet<>();
for (Statement statement : update.getAddedStatements()) {
PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId();
if (seenSingleProperties.contains(pid)) {
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
} else if (_fetcher.hasSingleValue(pid)) {
seenSingleProperties.add(pid);
}
}
}
}

74
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SnakScrutinizer.java Normal file
View File

@ -0,0 +1,74 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.Iterator;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer that inspects snaks individually, no matter whether they appear
* as main snaks, qualifiers or references.
*
* @author Antonin Delpeuch
*
*/
public abstract class SnakScrutinizer extends StatementScrutinizer {
/**
* This is the method that subclasses should override to implement their checks.
*
* @param snak:
* the snak to inspect
* @param entityId:
* the item on which it is going to (dis)appear
* @param added:
* whether this snak is going to be added or deleted
*/
public abstract void scrutinize(Snak snak, EntityIdValue entityId, boolean added);
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
// Main snak
scrutinize(statement.getClaim().getMainSnak(), entityId, added);
// Qualifiers
scrutinizeSnakSet(statement.getClaim().getAllQualifiers(), entityId, added);
// References
for (Reference ref : statement.getReferences()) {
scrutinizeSnakSet(ref.getAllSnaks(), entityId, added);
}
}
protected void scrutinizeSnakSet(Iterator<Snak> snaks, EntityIdValue entityId, boolean added) {
while (snaks.hasNext()) {
Snak snak = snaks.next();
scrutinize(snak, entityId, added);
}
}
}

55
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java Normal file
View File

@ -0,0 +1,55 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
public abstract class StatementScrutinizer extends EditScrutinizer {
@Override
public void scrutinize(ItemUpdate update) {
EntityIdValue currentEntityId = update.getItemId();
for (Statement statement : update.getAddedStatements()) {
scrutinize(statement, currentEntityId, true);
}
for (Statement statement : update.getDeletedStatements()) {
scrutinize(statement, currentEntityId, false);
}
}
/**
* The method that should be overridden by subclasses, implementing the checks
* on one statement
*
* @param statement:
* the statement to scrutinize
* @param entityId:
* the id of the entity on which this statement is made or removed
* @param added:
* whether this statement was added or deleted
*/
public abstract void scrutinize(Statement statement, EntityIdValue entityId, boolean added);
}

46
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/UnsourcedScrutinizer.java Normal file
View File

@ -0,0 +1,46 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer checking for unsourced statements
*
* @author Antonin Delpeuch
*
*/
public class UnsourcedScrutinizer extends StatementScrutinizer {
public static final String type = "unsourced-statements";
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
if (statement.getReferences().isEmpty() && added) {
warning(type);
}
}
}

62
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java Normal file
View File

@ -0,0 +1,62 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that inspects the values of snaks and terms
*
* @author Antonin Delpeuch
*
*/
public abstract class ValueScrutinizer extends SnakScrutinizer {
@Override
public void scrutinize(ItemUpdate update) {
super.scrutinize(update);
for (MonolingualTextValue label : update.getLabels()) {
scrutinize(label);
}
for (MonolingualTextValue alias : update.getAliases()) {
scrutinize(alias);
}
for (MonolingualTextValue description : update.getDescriptions()) {
scrutinize(description);
}
}
public abstract void scrutinize(Value value);
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
scrutinize(snak.getValue());
}
}

85
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/WhitespaceScrutinizer.java Normal file
View File

@ -0,0 +1,85 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* Scrutinizes strings for trailing / leading whitespace, and others
*
* @author Antonin Delpeuch
*
*/
public class WhitespaceScrutinizer extends ValueScrutinizer {
private Map<String, Pattern> _issuesMap;
public static final String leadingWhitespaceType = "leading-whitespace";
public static final String trailingWhitespaceType = "trailing-whitespace";
public static final String duplicateWhitespaceType = "duplicate-whitespace";
public static final String nonPrintableCharsType = "non-printable-characters";
public WhitespaceScrutinizer() {
_issuesMap = new HashMap<>();
_issuesMap.put(leadingWhitespaceType, Pattern.compile("^\\s"));
_issuesMap.put(trailingWhitespaceType, Pattern.compile("\\s$"));
_issuesMap.put(duplicateWhitespaceType, Pattern.compile("\\s\\s"));
// https://stackoverflow.com/questions/14565934/regular-expression-to-remove-all-non-printable-characters
_issuesMap.put(nonPrintableCharsType, Pattern.compile("[\\x00\\x03\\x08\\x0B\\x0C\\x0E-\\x1F]"));
}
@Override
public void scrutinize(Value value) {
String str = null;
if (MonolingualTextValue.class.isInstance(value)) {
str = ((MonolingualTextValue) value).getText();
} else if (StringValue.class.isInstance(value)) {
str = ((StringValue) value).getString();
}
if (str != null) {
for (Entry<String, Pattern> entry : _issuesMap.entrySet()) {
if (entry.getValue().matcher(str).find()) {
emitWarning(entry.getKey(), str);
}
}
}
}
private void emitWarning(String type, String example) {
QAWarning warning = new QAWarning(type, null, QAWarning.Severity.WARNING, 1);
warning.setProperty("example_string", example);
addIssue(warning);
}
}

107
extensions/wikidata/src/org/openrefine/wikidata/schema/ExpressionContext.java Normal file
View File

@ -0,0 +1,107 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.qa.QAWarningStore;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ColumnModel;
import com.google.refine.model.Row;
/**
* A class holding all the necessary information about the context in which a
* schema expression is evaluated.
*
* @author Antonin Delpeuch
*
*/
public class ExpressionContext {
private String baseIRI;
private int rowId;
private Row row;
private ColumnModel columnModel;
private QAWarningStore warningStore;
/**
* Builds an expression context to evaluate a schema on a row
*
* @param baseIRI
* the siteIRI of the schema
* @param rowId
* the id of the row currently visited
* @param row
* the row itself
* @param columnModel
* lets us access cells by column name
* @param warningStore
* where to store the issues encountered when evaluating (can be set
* to null if these issues should be ignored)
*/
public ExpressionContext(String baseIRI, int rowId, Row row, ColumnModel columnModel, QAWarningStore warningStore) {
Validate.notNull(baseIRI);
this.baseIRI = baseIRI;
this.rowId = rowId;
Validate.notNull(row);
this.row = row;
Validate.notNull(columnModel);
this.columnModel = columnModel;
this.warningStore = warningStore;
}
public String getBaseIRI() {
return baseIRI;
}
/**
* Retrieves a cell in the current row, by column name. If the column does not
* exist, null is returned.
*
* @param name
* the name of the column to retrieve the cell from
* @return the cell
*/
public Cell getCellByName(String name) {
Column column = columnModel.getColumnByName(name);
if (column != null) {
int idx = column.getCellIndex();
return row.getCell(idx);
} else {
return null;
}
}
public int getRowId() {
return rowId;
}
public void addWarning(QAWarning warning) {
if (warningStore != null) {
warningStore.addWarning(warning);
}
}
}

158
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java Normal file
View File

@ -0,0 +1,158 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
import java.util.Map.Entry;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableMap;
/**
* A constant for a time value, accepting a number of formats which determine
* the precision of the parsed value.
*
* @author Antonin Delpeuch
*
*/
public class WbDateConstant implements WbExpression<TimeValue> {
/**
* Map of formats accepted by the parser. Each format is associated to the time
* precision it induces (an integer according to Wikibase's data model).
*/
public static Map<SimpleDateFormat, Integer> acceptedFormats = ImmutableMap.<SimpleDateFormat, Integer> builder()
.put(new SimpleDateFormat("yyyy"), 9).put(new SimpleDateFormat("yyyy-MM"), 10)
.put(new SimpleDateFormat("yyyy-MM-dd"), 11).put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"), 13)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), 14).build();
private TimeValue parsed;
private String origDatestamp;
/**
* Constructor. Used for deserialization from JSON. The object will be
* constructed even if the time cannot be parsed (it will evaluate to null) in
* {@link evaluate}.
*
* @param origDatestamp
* the date value as a string
*/
@JsonCreator
public WbDateConstant(@JsonProperty("value") String origDatestamp) {
Validate.notNull(origDatestamp);
this.setOrigDatestamp(origDatestamp);
}
@Override
public TimeValue evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
return parsed;
}
/**
* Parses a timestamp into a Wikibase {@link TimeValue}. The precision is
* automatically inferred from the format.
*
* @param datestamp
* the time to parse
* @return
* @throws ParseException
* if the time cannot be parsed
*/
public static TimeValue parse(String datestamp)
throws ParseException {
Date date = null;
int precision = 9; // default precision (will be overridden)
for (Entry<SimpleDateFormat, Integer> entry : acceptedFormats.entrySet()) {
ParsePosition position = new ParsePosition(0);
String trimmedDatestamp = datestamp.trim();
date = entry.getKey().parse(trimmedDatestamp, position);
// Ignore parses which failed or do not consume all the input
if (date != null && position.getIndex() == trimmedDatestamp.length()) {
precision = entry.getValue();
break;
}
}
if (date == null) {
throw new ParseException("Invalid date.", 0);
} else {
Calendar calendar = Calendar.getInstance();
calendar = Calendar.getInstance();
calendar.setTime(date);
return Datamodel.makeTimeValue(calendar.get(Calendar.YEAR), (byte) (calendar.get(Calendar.MONTH) + 1), // java
// starts
// at
// 0
(byte) calendar.get(Calendar.DAY_OF_MONTH), (byte) calendar.get(Calendar.HOUR_OF_DAY),
(byte) calendar.get(Calendar.MINUTE), (byte) calendar.get(Calendar.SECOND), (byte) precision, 0, 1,
0, TimeValue.CM_GREGORIAN_PRO);
}
}
/**
* @return the original datestamp
*/
@JsonProperty("value")
public String getOrigDatestamp() {
return origDatestamp;
}
private void setOrigDatestamp(String origDatestamp) {
this.origDatestamp = origDatestamp;
try {
this.parsed = parse(origDatestamp);
} catch (ParseException e) {
throw new IllegalArgumentException("Invalid datestamp provided: " + origDatestamp);
}
}
@Override
public boolean equals(Object other) {
if (other == null || !WbDateConstant.class.isInstance(other)) {
return false;
}
WbDateConstant otherConstant = (WbDateConstant) other;
return origDatestamp.equals(otherConstant.getOrigDatestamp());
}
@Override
public int hashCode() {
return origDatestamp.hashCode();
}
}

68
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java Normal file
View File

@ -0,0 +1,68 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
/**
* An expression that represents a time value, extracted from a string. A number
* of formats are recognized, see {@link WbDateConstant} for details.
*
* @author Antonin Delpeuch
*
*/
public class WbDateVariable extends WbVariableExpr<TimeValue> {
@JsonCreator
public WbDateVariable() {
}
public WbDateVariable(String columnName) {
setColumnName(columnName);
}
@Override
public TimeValue fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException {
try {
// TODO accept parsed dates (without converting them to strings)
return WbDateConstant.parse(cell.value.toString());
} catch (ParseException e) {
throw new SkipSchemaExpressionException();
}
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbDateVariable.class);
}
}

60
extensions/wikidata/src/org/openrefine/wikidata/schema/WbExpression.java Normal file
View File

@ -0,0 +1,60 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* The base interface for all expressions, which evaluate to a particular type T
* in an ExpressionContext.
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
@JsonSubTypes({ @Type(value = WbStringConstant.class, name = "wbstringconstant"),
@Type(value = WbStringVariable.class, name = "wbstringvariable"),
@Type(value = WbLocationConstant.class, name = "wblocationconstant"),
@Type(value = WbLocationVariable.class, name = "wblocationvariable"),
@Type(value = WbItemConstant.class, name = "wbitemconstant"),
@Type(value = WbItemVariable.class, name = "wbitemvariable"),
@Type(value = WbLanguageConstant.class, name = "wblanguageconstant"),
@Type(value = WbLanguageVariable.class, name = "wblanguagevariable"),
@Type(value = WbDateConstant.class, name = "wbdateconstant"),
@Type(value = WbDateVariable.class, name = "wbdatevariable"),
@Type(value = WbMonolingualExpr.class, name = "wbmonolingualexpr"),
@Type(value = WbPropConstant.class, name = "wbpropconstant"),
@Type(value = WbLanguageConstant.class, name = "wblanguageconstant"),
@Type(value = WbLanguageVariable.class, name = "wblanguagevariable"),
@Type(value = WbQuantityExpr.class, name = "wbquantityexpr"), })
public interface WbExpression<T> {
/**
* Evaluates the value expression in a given context, returns a Wikibase value
* suitable to be the target of a claim.
*/
public T evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException;
}

77
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java Normal file
View File

@ -0,0 +1,77 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.entityvalues.SuggestedItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* Represents an item that does not vary, it is independent of the row.
*/
public class WbItemConstant implements WbExpression<ItemIdValue> {
private String qid;
private String label;
@JsonCreator
public WbItemConstant(@JsonProperty("qid") String qid, @JsonProperty("label") String label) {
Validate.notNull(qid);
this.qid = qid;
Validate.notNull(label);
this.label = label;
}
@Override
public ItemIdValue evaluate(ExpressionContext ctxt) {
return new SuggestedItemIdValue(qid, ctxt.getBaseIRI(), label);
}
@JsonProperty("qid")
public String getQid() {
return qid;
}
@JsonProperty("label")
public String getLabel() {
return label;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbItemConstant.class.isInstance(other)) {
return false;
}
WbItemConstant otherConstant = (WbItemConstant) other;
return (qid.equals(otherConstant.getQid()) && label.equals(otherConstant.getLabel()));
}
@Override
public int hashCode() {
return qid.hashCode() + label.hashCode();
}
}

122
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java Normal file
View File

@ -0,0 +1,122 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.Collections;
import java.util.List;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.openrefine.wikidata.utils.JacksonJsonizable;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* The representation of an item document, which can contain variables both for
* its own id and in its contents.
*
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
public class WbItemDocumentExpr extends JacksonJsonizable implements WbExpression<ItemUpdate> {
private WbExpression<? extends ItemIdValue> subject;
private List<WbNameDescExpr> nameDescs;
private List<WbStatementGroupExpr> statementGroups;
@JsonCreator
public WbItemDocumentExpr(@JsonProperty("subject") WbExpression<? extends ItemIdValue> subjectExpr,
@JsonProperty("nameDescs") List<WbNameDescExpr> nameDescExprs,
@JsonProperty("statementGroups") List<WbStatementGroupExpr> statementGroupExprs) {
Validate.notNull(subjectExpr);
this.subject = subjectExpr;
if (nameDescExprs == null) {
nameDescExprs = Collections.emptyList();
}
this.nameDescs = nameDescExprs;
if (statementGroupExprs == null) {
statementGroupExprs = Collections.emptyList();
}
this.statementGroups = statementGroupExprs;
}
@Override
public ItemUpdate evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
ItemIdValue subjectId = getSubject().evaluate(ctxt);
ItemUpdateBuilder update = new ItemUpdateBuilder(subjectId);
for (WbStatementGroupExpr expr : getStatementGroups()) {
try {
for (Statement s : expr.evaluate(ctxt, subjectId).getStatements()) {
update.addStatement(s);
}
} catch (SkipSchemaExpressionException e) {
continue;
}
}
for (WbNameDescExpr expr : getNameDescs()) {
expr.contributeTo(update, ctxt);
}
return update.build();
}
@JsonProperty("subject")
public WbExpression<? extends ItemIdValue> getSubject() {
return subject;
}
@JsonProperty("nameDescs")
public List<WbNameDescExpr> getNameDescs() {
return nameDescs;
}
@JsonProperty("statementGroups")
public List<WbStatementGroupExpr> getStatementGroups() {
return statementGroups;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbItemDocumentExpr.class.isInstance(other)) {
return false;
}
WbItemDocumentExpr otherExpr = (WbItemDocumentExpr) other;
return subject.equals(otherExpr.getSubject()) && nameDescs.equals(otherExpr.getNameDescs())
&& statementGroups.equals(otherExpr.getStatementGroups());
}
@Override
public int hashCode() {
return subject.hashCode() + nameDescs.hashCode() + statementGroups.hashCode();
}
}

81
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java Normal file
View File

@ -0,0 +1,81 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
import com.google.refine.model.Recon.Judgment;
/**
* An item that depends on a reconciled value in a column.
*
* @author Antonin Delpeuch
*
*/
public class WbItemVariable extends WbVariableExpr<ItemIdValue> {
@JsonCreator
public WbItemVariable() {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly used as a
* convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from
*/
public WbItemVariable(String columnName) {
setColumnName(columnName);
}
@Override
public ItemIdValue fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException {
if (cell.recon != null
&& (Judgment.Matched.equals(cell.recon.judgment) || Judgment.New.equals(cell.recon.judgment))) {
if (!cell.recon.identifierSpace.equals(Datamodel.SITE_WIKIDATA)) {
QAWarning warning = new QAWarning("invalid-identifier-space", null, QAWarning.Severity.INFO, 1);
warning.setProperty("example_cell", cell.value.toString());
ctxt.addWarning(warning);
throw new SkipSchemaExpressionException();
}
return new ReconItemIdValue(cell.recon, cell.value.toString());
}
throw new SkipSchemaExpressionException();
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbItemVariable.class);
}
}

105
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageConstant.java Normal file
View File

@ -0,0 +1,105 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.interfaces.WikimediaLanguageCodes;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A constant that represents a Wikimedia language code.
*
* @author Antonin Delpeuch
*
*/
public class WbLanguageConstant implements WbExpression<String> {
protected String _langId;
protected String _langLabel;
@JsonCreator
public WbLanguageConstant(@JsonProperty("id") String langId, @JsonProperty("label") String langLabel) {
_langId = normalizeLanguageCode(langId);
Validate.notNull(_langId, "A valid language code must be provided.");
Validate.notNull(langLabel);
_langLabel = langLabel;
}
/**
* Checks that a language code is valid and returns its preferred version
* (converting deprecated language codes to their better values).
*
* @param lang
* a Wikimedia language code
* @return the normalized code, or null if the code is invalid.
*/
public static String normalizeLanguageCode(String lang) {
try {
WikimediaLanguageCodes.getLanguageCode(lang);
return WikimediaLanguageCodes.fixLanguageCodeIfDeprecated(lang);
} catch (IllegalArgumentException e) {
return null;
}
}
@Override
public String evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
return _langId;
}
/**
* @return the language code for this language
*/
@JsonProperty("id")
public String getLang() {
return _langId;
}
/**
* @return the name of the language in itself
*/
@JsonProperty("label")
public String getLabel() {
return _langLabel;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbLanguageConstant.class.isInstance(other)) {
return false;
}
WbLanguageConstant otherConstant = (WbLanguageConstant) other;
return _langId.equals(otherConstant.getLang()) && _langLabel.equals(otherConstant.getLabel());
}
@Override
public int hashCode() {
return _langId.hashCode();
}
}

71
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageVariable.java Normal file
View File

@ -0,0 +1,71 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
/**
* A language variable generates a language code from a cell. It checks its
* values against a known list of valid language codes and fixes on the fly the
* deprecated ones (see {@link WbLanguageConstant}).
*/
public class WbLanguageVariable extends WbVariableExpr<String> {
@JsonCreator
public WbLanguageVariable() {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly used as a
* convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from
*/
public WbLanguageVariable(String columnName) {
setColumnName(columnName);
}
@Override
public String fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException {
if (cell.value != null && !cell.value.toString().isEmpty()) {
String code = cell.value.toString().trim();
String normalized = WbLanguageConstant.normalizeLanguageCode(code);
if (normalized != null) {
return normalized;
}
}
throw new SkipSchemaExpressionException();
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbLanguageVariable.class);
}
}

114
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationConstant.java Normal file
View File

@ -0,0 +1,114 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
import org.apache.commons.lang.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A constant for a geographical location. The accepted format is lat,lng or
* lat/lng.
*
* @author Antonin Delpeuch
*
*/
public class WbLocationConstant implements WbExpression<GlobeCoordinatesValue> {
public static final double defaultPrecision = GlobeCoordinatesValue.PREC_TEN_MICRO_DEGREE;
private String value;
private GlobeCoordinatesValue parsed;
@JsonCreator
public WbLocationConstant(@JsonProperty("value") String origValue) throws ParseException {
this.value = origValue;
Validate.notNull(origValue);
this.parsed = parse(origValue);
Validate.notNull(this.parsed);
}
/**
* Parses a string to a location.
*
* @param expr
* the string to parse
* @return the parsed location
* @throws ParseException
*/
public static GlobeCoordinatesValue parse(String expr)
throws ParseException {
double lat = 0;
double lng = 0;
double precision = defaultPrecision;
String[] parts = expr.split("[,/]");
if (parts.length >= 2 && parts.length <= 3) {
try {
lat = Double.parseDouble(parts[0]);
lng = Double.parseDouble(parts[1]);
if (parts.length == 3) {
precision = Double.parseDouble(parts[2]);
}
return Datamodel.makeGlobeCoordinatesValue(lat, lng, precision, GlobeCoordinatesValue.GLOBE_EARTH);
} catch (NumberFormatException e) {
;
}
}
throw new ParseException("Invalid globe coordinates", 0);
}
@Override
public GlobeCoordinatesValue evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
return parsed;
}
/**
* @return the original value as a string.
*/
@JsonProperty("value")
public String getValue() {
return value;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbLocationConstant.class.isInstance(other)) {
return false;
}
WbLocationConstant otherConstant = (WbLocationConstant) other;
return value.equals(otherConstant.getValue());
}
@Override
public int hashCode() {
return value.hashCode();
}
}

61
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java Normal file
View File

@ -0,0 +1,61 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
public class WbLocationVariable extends WbVariableExpr<GlobeCoordinatesValue> {
@JsonCreator
public WbLocationVariable() {
}
public WbLocationVariable(String columnName) {
setColumnName(columnName);
}
@Override
public GlobeCoordinatesValue fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException {
String expr = cell.value.toString();
try {
return WbLocationConstant.parse(expr);
} catch (ParseException e) {
throw new SkipSchemaExpressionException();
}
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbLocationVariable.class);
}
}

89
extensions/wikidata/src/org/openrefine/wikidata/schema/WbMonolingualExpr.java Normal file
View File

@ -0,0 +1,89 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class WbMonolingualExpr implements WbExpression<MonolingualTextValue> {
private WbExpression<? extends String> languageExpr;
private WbExpression<? extends StringValue> valueExpr;
@JsonCreator
public WbMonolingualExpr(@JsonProperty("language") WbExpression<? extends String> languageExpr,
@JsonProperty("value") WbExpression<? extends StringValue> valueExpr) {
Validate.notNull(languageExpr);
this.languageExpr = languageExpr;
Validate.notNull(valueExpr);
this.valueExpr = valueExpr;
}
@Override
public MonolingualTextValue evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
String text = getValueExpr().evaluate(ctxt).getString();
try {
String lang = getLanguageExpr().evaluate(ctxt);
return Datamodel.makeMonolingualTextValue(text, lang);
} catch (SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning("monolingual-text-without-language", null, QAWarning.Severity.WARNING, 1);
warning.setProperty("example_text", text);
ctxt.addWarning(warning);
throw new SkipSchemaExpressionException();
}
}
@JsonProperty("language")
public WbExpression<? extends String> getLanguageExpr() {
return languageExpr;
}
@JsonProperty("value")
public WbExpression<? extends StringValue> getValueExpr() {
return valueExpr;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbMonolingualExpr.class.isInstance(other)) {
return false;
}
WbMonolingualExpr otherExpr = (WbMonolingualExpr) other;
return languageExpr.equals(otherExpr.getLanguageExpr()) && valueExpr.equals(otherExpr.getValueExpr());
}
@Override
public int hashCode() {
return languageExpr.hashCode() + valueExpr.hashCode();
}
}

112
extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java Normal file
View File

@ -0,0 +1,112 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* An expression that represent a term (label, description or alias). The
* structure is slightly different from other expressions because we need to
* call different methods on {@link ItemUpdateBuilder}.
*
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class WbNameDescExpr {
enum NameDescrType {
LABEL, DESCRIPTION, ALIAS,
}
private NameDescrType type;
private WbMonolingualExpr value;
@JsonCreator
public WbNameDescExpr(@JsonProperty("name_type") NameDescrType type,
@JsonProperty("value") WbMonolingualExpr value) {
Validate.notNull(type);
this.type = type;
Validate.notNull(value);
this.value = value;
}
/**
* Evaluates the expression and adds the result to the item update.
*
* @param item
* the item update where the term should be stored
* @param ctxt
* the evaluation context for the expression
*/
public void contributeTo(ItemUpdateBuilder item, ExpressionContext ctxt) {
try {
MonolingualTextValue val = getValue().evaluate(ctxt);
switch (getType()) {
case LABEL:
item.addLabel(val);
break;
case DESCRIPTION:
item.addDescription(val);
break;
case ALIAS:
item.addAlias(val);
break;
}
} catch (SkipSchemaExpressionException e) {
return;
}
}
@JsonProperty("name_type")
public NameDescrType getType() {
return type;
}
@JsonProperty("value")
public WbMonolingualExpr getValue() {
return value;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbNameDescExpr.class.isInstance(other)) {
return false;
}
WbNameDescExpr otherExpr = (WbNameDescExpr) other;
return type.equals(otherExpr.getType()) && value.equals(otherExpr.getValue());
}
@Override
public int hashCode() {
return type.hashCode() + value.hashCode();
}
}

89
extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java Normal file
View File

@ -0,0 +1,89 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.entityvalues.SuggestedPropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A constant property, that does not change depending on the row
*
* @author Antonin Delpeuch
*
*/
public class WbPropConstant implements WbExpression<PropertyIdValue> {
private String pid;
private String label;
private String datatype;
@JsonCreator
public WbPropConstant(@JsonProperty("pid") String pid, @JsonProperty("label") String label,
@JsonProperty("datatype") String datatype) {
Validate.notNull(pid);
this.pid = pid;
Validate.notNull(label);
this.label = label;
this.datatype = datatype;
}
@Override
public PropertyIdValue evaluate(ExpressionContext ctxt) {
return new SuggestedPropertyIdValue(pid, ctxt.getBaseIRI(), label);
}
@JsonProperty("pid")
public String getPid() {
return pid;
}
@JsonProperty("label")
public String getLabel() {
return label;
}
@JsonProperty("datatype")
public String getDatatype() {
return datatype;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbPropConstant.class.isInstance(other)) {
return false;
}
WbPropConstant otherConstant = (WbPropConstant) other;
return pid.equals(otherConstant.getPid()) && label.equals(otherConstant.getLabel())
&& datatype.equals(otherConstant.getDatatype());
}
@Override
public int hashCode() {
return pid.hashCode() + label.hashCode();
}
}

104
extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java Normal file
View File

@ -0,0 +1,104 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.math.BigDecimal;
import org.apache.commons.lang.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class WbQuantityExpr implements WbExpression<QuantityValue> {
private final WbExpression<? extends StringValue> amountExpr;
private final WbExpression<? extends ItemIdValue> unitExpr;
/**
* Creates an expression for a quantity, which contains two sub-expressions: one
* for the amount (a string with a particular format) and one for the unit,
* which is optional.
*
* Setting unitExpr to null will give quantities without units. Setting it to a
* non-null value will make the unit mandatory: if the unit expression fails to
* evaluate, the whole quantity expression will fail too.
*/
@JsonCreator
public WbQuantityExpr(@JsonProperty("amount") WbExpression<? extends StringValue> amountExpr,
@JsonProperty("unit") WbExpression<? extends ItemIdValue> unitExpr) {
Validate.notNull(amountExpr);
this.amountExpr = amountExpr;
this.unitExpr = unitExpr;
}
@Override
public QuantityValue evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
StringValue amount = getLanguageExpr().evaluate(ctxt);
// we know the amount is nonnull, nonempty here
BigDecimal parsedAmount = null;
BigDecimal lowerBound = null;
BigDecimal upperBound = null;
try {
String originalAmount = amount.getString().toUpperCase();
parsedAmount = new BigDecimal(originalAmount);
if (originalAmount.contains("E")) {
// engineering notation: we derive the precision from
// the expression (feature!)
BigDecimal uncertainty = new BigDecimal("0.5").scaleByPowerOfTen(-parsedAmount.scale());
lowerBound = new BigDecimal(parsedAmount.subtract(uncertainty).toPlainString());
upperBound = new BigDecimal(parsedAmount.add(uncertainty).toPlainString());
}
// workaround for https://github.com/Wikidata/Wikidata-Toolkit/issues/341
parsedAmount = new BigDecimal(parsedAmount.toPlainString());
} catch (NumberFormatException e) {
throw new SkipSchemaExpressionException();
}
if (getUnitExpr() != null) {
ItemIdValue unit = getUnitExpr().evaluate(ctxt);
return Datamodel.makeQuantityValue(parsedAmount, lowerBound, upperBound, unit.getIri());
}
return Datamodel.makeQuantityValue(parsedAmount, lowerBound, upperBound);
}
@JsonProperty("amount")
public WbExpression<? extends StringValue> getLanguageExpr() {
return amountExpr;
}
@JsonProperty("unit")
public WbExpression<? extends ItemIdValue> getUnitExpr() {
return unitExpr;
}
}

97
extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java Normal file
View File

@ -0,0 +1,97 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* An expression for a reference (list of reference snaks).
*
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
public class WbReferenceExpr implements WbExpression<Reference> {
private List<WbSnakExpr> snakExprs;
@JsonCreator
public WbReferenceExpr(@JsonProperty("snaks") List<WbSnakExpr> snakExprs) {
Validate.notNull(snakExprs);
this.snakExprs = snakExprs;
}
@Override
public Reference evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
List<SnakGroup> snakGroups = new ArrayList<SnakGroup>();
for (WbSnakExpr expr : getSnaks()) {
List<Snak> snakList = new ArrayList<Snak>(1);
try {
snakList.add(expr.evaluate(ctxt));
snakGroups.add(Datamodel.makeSnakGroup(snakList));
} catch (SkipSchemaExpressionException e) {
continue;
}
}
if (!snakGroups.isEmpty()) {
return Datamodel.makeReference(snakGroups);
} else {
throw new SkipSchemaExpressionException();
}
}
@JsonProperty("snaks")
public List<WbSnakExpr> getSnaks() {
return snakExprs;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbReferenceExpr.class.isInstance(other)) {
return false;
}
WbReferenceExpr otherExpr = (WbReferenceExpr) other;
return snakExprs.equals(otherExpr.getSnaks());
}
@Override
public int hashCode() {
return snakExprs.hashCode();
}
}

92
extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java Normal file
View File

@ -0,0 +1,92 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.entityvalues.FullyPropertySerializingValueSnak;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Value;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* An expression for a snak (pair of property and value).
*
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
public class WbSnakExpr implements WbExpression<Snak> {
private WbExpression<? extends PropertyIdValue> prop;
private WbExpression<? extends Value> value;
@JsonCreator
public WbSnakExpr(@JsonProperty("prop") WbExpression<? extends PropertyIdValue> propExpr,
@JsonProperty("value") WbExpression<? extends Value> valueExpr) {
Validate.notNull(propExpr);
this.prop = propExpr;
Validate.notNull(valueExpr);
this.value = valueExpr;
}
@Override
public Snak evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
PropertyIdValue propertyId = getProp().evaluate(ctxt);
Value evaluatedValue = value.evaluate(ctxt);
return new FullyPropertySerializingValueSnak(propertyId, evaluatedValue);
}
@JsonProperty("prop")
public WbExpression<? extends PropertyIdValue> getProp() {
return prop;
}
@JsonProperty("value")
public WbExpression<? extends Value> getValue() {
return value;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbSnakExpr.class.isInstance(other)) {
return false;
}
WbSnakExpr otherExpr = (WbSnakExpr) other;
return prop.equals(otherExpr.getProp()) && value.equals(otherExpr.getValue());
}
@Override
public int hashCode() {
return prop.hashCode() + value.hashCode();
}
}

147
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java Normal file
View File

@ -0,0 +1,147 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
import org.wikidata.wdtk.datamodel.interfaces.Value;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
@JsonIgnoreProperties(ignoreUnknown = true)
public class WbStatementExpr {
private WbExpression<? extends Value> mainSnakValueExpr;
private List<WbSnakExpr> qualifierExprs;
private List<WbReferenceExpr> referenceExprs;
@JsonCreator
public WbStatementExpr(@JsonProperty("value") WbExpression<? extends Value> mainSnakValueExpr,
@JsonProperty("qualifiers") List<WbSnakExpr> qualifierExprs,
@JsonProperty("references") List<WbReferenceExpr> referenceExprs) {
Validate.notNull(mainSnakValueExpr);
this.mainSnakValueExpr = mainSnakValueExpr;
if (qualifierExprs == null) {
qualifierExprs = Collections.emptyList();
}
this.qualifierExprs = qualifierExprs;
if (referenceExprs == null) {
referenceExprs = Collections.emptyList();
}
this.referenceExprs = referenceExprs;
}
public static List<SnakGroup> groupSnaks(List<Snak> snaks) {
List<SnakGroup> snakGroups = new ArrayList<SnakGroup>();
for (Snak snak : snaks) {
List<Snak> singleton = new ArrayList<Snak>();
singleton.add(snak);
snakGroups.add(Datamodel.makeSnakGroup(singleton));
}
return snakGroups;
}
public Statement evaluate(ExpressionContext ctxt, ItemIdValue subject, PropertyIdValue propertyId)
throws SkipSchemaExpressionException {
Value mainSnakValue = getMainsnak().evaluate(ctxt);
Snak mainSnak = Datamodel.makeValueSnak(propertyId, mainSnakValue);
// evaluate qualifiers
List<Snak> qualifiers = new ArrayList<Snak>(getQualifiers().size());
for (WbSnakExpr qExpr : getQualifiers()) {
try {
qualifiers.add(qExpr.evaluate(ctxt));
} catch (SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning("ignored-qualifiers", null, QAWarning.Severity.INFO, 1);
warning.setProperty("example_entity", subject);
warning.setProperty("example_property_entity", mainSnak.getPropertyId());
ctxt.addWarning(warning);
}
}
List<SnakGroup> groupedQualifiers = groupSnaks(qualifiers);
Claim claim = Datamodel.makeClaim(subject, mainSnak, groupedQualifiers);
// evaluate references
List<Reference> references = new ArrayList<Reference>();
for (WbReferenceExpr rExpr : getReferences()) {
try {
references.add(rExpr.evaluate(ctxt));
} catch (SkipSchemaExpressionException e) {
QAWarning warning = new QAWarning("ignored-references", null, QAWarning.Severity.INFO, 1);
warning.setProperty("example_entity", subject);
warning.setProperty("example_property_entity", mainSnak.getPropertyId());
ctxt.addWarning(warning);
}
}
StatementRank rank = StatementRank.NORMAL;
return Datamodel.makeStatement(claim, references, rank, "");
}
@JsonProperty("value")
public WbExpression<? extends Value> getMainsnak() {
return mainSnakValueExpr;
}
@JsonProperty("qualifiers")
public List<WbSnakExpr> getQualifiers() {
return qualifierExprs;
}
@JsonProperty("references")
public List<WbReferenceExpr> getReferences() {
return referenceExprs;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbStatementExpr.class.isInstance(other)) {
return false;
}
WbStatementExpr otherExpr = (WbStatementExpr) other;
return mainSnakValueExpr.equals(otherExpr.getMainsnak()) && qualifierExprs.equals(otherExpr.getQualifiers())
&& referenceExprs.equals(otherExpr.getReferences());
}
@Override
public int hashCode() {
return mainSnakValueExpr.hashCode() + qualifierExprs.hashCode() + referenceExprs.hashCode();
}
}

98
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java Normal file
View File

@ -0,0 +1,98 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
import java.util.List;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
@JsonIgnoreProperties(ignoreUnknown = true)
public class WbStatementGroupExpr {
private WbExpression<? extends PropertyIdValue> propertyExpr;
private List<WbStatementExpr> statementExprs;
@JsonCreator
public WbStatementGroupExpr(@JsonProperty("property") WbExpression<? extends PropertyIdValue> propertyExpr,
@JsonProperty("statements") List<WbStatementExpr> claimExprs) {
Validate.notNull(propertyExpr);
this.propertyExpr = propertyExpr;
Validate.notNull(claimExprs);
Validate.isTrue(!claimExprs.isEmpty());
this.statementExprs = claimExprs;
}
public StatementGroup evaluate(ExpressionContext ctxt, ItemIdValue subject)
throws SkipSchemaExpressionException {
PropertyIdValue propertyId = propertyExpr.evaluate(ctxt);
List<Statement> statements = new ArrayList<Statement>(statementExprs.size());
for (WbStatementExpr expr : statementExprs) {
try {
statements.add(expr.evaluate(ctxt, subject, propertyId));
} catch (SkipSchemaExpressionException e) {
continue;
}
}
if (!statements.isEmpty()) {
return Datamodel.makeStatementGroup(statements);
} else {
throw new SkipSchemaExpressionException();
}
}
@JsonProperty("property")
public WbExpression<? extends PropertyIdValue> getProperty() {
return propertyExpr;
}
@JsonProperty("statements")
public List<WbStatementExpr> getStatements() {
return statementExprs;
}
@Override
public boolean equals(Object other) {
if (other == null || !WbStatementGroupExpr.class.isInstance(other)) {
return false;
}
WbStatementGroupExpr otherExpr = (WbStatementGroupExpr) other;
return propertyExpr.equals(otherExpr.getProperty()) && statementExprs.equals(otherExpr.getStatements());
}
@Override
public int hashCode() {
return propertyExpr.hashCode() + statementExprs.hashCode();
}
}

67
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java Normal file
View File

@ -0,0 +1,67 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class WbStringConstant implements WbExpression<StringValue> {
private String value;
@JsonCreator
public WbStringConstant(@JsonProperty("value") String value) {
Validate.notNull(value);
Validate.isTrue(!value.isEmpty()); // for now we don't accept empty strings
// because in the variable counterpart of this expression, they are skipped
this.value = value;
}
@Override
public StringValue evaluate(ExpressionContext ctxt) {
return Datamodel.makeStringValue(value);
}
@JsonProperty("value")
public String getValue() {
return value;
}
@Override
public boolean equals(Object other) {
if(other == null || !WbStringConstant.class.isInstance(other)) {
return false;
}
return value.equals(((WbStringConstant)other).getValue());
}
@Override
public int hashCode() {
return value.hashCode();
}
}

70
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringVariable.java Normal file
View File

@ -0,0 +1,70 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
/**
* A variable that returns a simple string value.
*
* @author Antonin Delpeuch
*
*/
public class WbStringVariable extends WbVariableExpr<StringValue> {
@JsonCreator
public WbStringVariable() {
}
/**
* Constructs a variable and sets the column it is bound to. Mostly used as a
* convenience method for testing.
*
* @param columnName
* the name of the column the expression should draw its value from
*/
public WbStringVariable(String columnName) {
setColumnName(columnName);
}
@Override
public StringValue fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException {
if (!cell.value.toString().isEmpty()) {
return Datamodel.makeStringValue(cell.value.toString());
}
throw new SkipSchemaExpressionException();
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbStringVariable.class);
}
}

125
extensions/wikidata/src/org/openrefine/wikidata/schema/WbVariableExpr.java Normal file
View File

@ -0,0 +1,125 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.model.Cell;
/**
* A base class for expressions which draw their values from a particular
* column.
*
* @author Antonin Delpeuch
*
* @param <T>
* the type of Wikibase value returned by the expression.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public abstract class WbVariableExpr<T> implements WbExpression<T> {
private String columnName;
/**
* Constructs a variable without setting the column name yet.
*/
@JsonCreator
public WbVariableExpr() {
columnName = null;
}
/**
* Returns the column name used by the variable.
*
* @return the OpenRefine column name
*/
@JsonProperty("columnName")
public String getColumnName() {
return columnName;
}
/**
* Changes the column name used by the variable. This is useful for
* deserialization, as well as updates when column names change.
*/
@JsonProperty("columnName")
public void setColumnName(String columnName) {
this.columnName = columnName;
}
/**
* Evaluates the expression in a given context, returning
*/
@Override
public T evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
Cell cell = ctxt.getCellByName(columnName);
if (cell != null) {
return fromCell(cell, ctxt);
}
throw new SkipSchemaExpressionException();
}
/**
* Method that should be implemented by subclasses, converting an OpenRefine
* cell to a Wikibase value. Access to other values and emiting warnings is
* possible via the supplied EvaluationContext object.
*
* @param cell
* the cell to convert
* @param ctxt
* the evaluation context
* @return the corresponding Wikibase value
*/
public abstract T fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException;
/**
* Helper for equality methods of subclasses.
*
* @param other
* the object to compare
* @param columnName
* the column name to compare to
* @param targetClass
* the target class for equality
* @return
*/
protected boolean equalAsVariables(Object other, Class<? extends WbVariableExpr<?>> targetClass) {
if (other == null || !targetClass.isInstance(other)) {
return false;
}
return columnName.equals(targetClass.cast(other).getColumnName());
}
@Override
public int hashCode() {
return columnName.hashCode();
}
}

241
extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java Normal file
View File

@ -0,0 +1,241 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.model.OverlayModel;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
/**
* Main class representing a skeleton of Wikibase edits with OpenRefine columns
* as variables.
*
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class WikibaseSchema implements OverlayModel {
final static Logger logger = LoggerFactory.getLogger("RdfSchema");
protected List<WbItemDocumentExpr> itemDocumentExprs = new ArrayList<WbItemDocumentExpr>();
protected String baseIri = "http://www.wikidata.org/entity/";
/**
* Constructor.
*/
public WikibaseSchema() {
}
/**
* Constructor for deserialization via Jackson
*/
@JsonCreator
public WikibaseSchema(@JsonProperty("itemDocuments") List<WbItemDocumentExpr> exprs) {
this.itemDocumentExprs = exprs;
}
/**
* @return the site IRI of the Wikibase instance referenced by this schema
*/
public String getBaseIri() {
return baseIri;
}
/**
* @return the list of document expressions for this schema
*/
public List<WbItemDocumentExpr> getItemDocumentExpressions() {
return itemDocumentExprs;
}
public void setItemDocumentExpressions(List<WbItemDocumentExpr> exprs) {
this.itemDocumentExprs = exprs;
}
/**
* Evaluates all item documents in a particular expression context. This
* specifies, among others, a row where the values of the variables will be
* read.
*
* @param ctxt
* the context in which the schema should be evaluated.
* @return
*/
public List<ItemUpdate> evaluateItemDocuments(ExpressionContext ctxt) {
List<ItemUpdate> result = new ArrayList<>();
for (WbItemDocumentExpr expr : itemDocumentExprs) {
try {
result.add(expr.evaluate(ctxt));
} catch (SkipSchemaExpressionException e) {
continue;
}
}
return result;
}
/**
* Evaluates the schema on a project, returning a list of ItemUpdates generated
* by the schema.
*
* Some warnings will be emitted in the warning store: those are only the ones
* that are generated at evaluation time (such as invalid formats for dates).
* Issues detected on candidate statements (such as constraint violations) are
* not included at this stage.
*
* @param project
* the project on which the schema should be evaluated
* @param engine
* the engine, which gives access to the current facets
* @param warningStore
* a store in which issues will be emitted
* @return item updates are stored in their generating order (not merged yet).
*/
public List<ItemUpdate> evaluate(Project project, Engine engine, QAWarningStore warningStore) {
List<ItemUpdate> result = new ArrayList<>();
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, new EvaluatingRowVisitor(result, warningStore));
return result;
}
/**
* Same as above, ignoring any warnings.
*/
public List<ItemUpdate> evaluate(Project project, Engine engine) {
return evaluate(project, engine, null);
}
protected class EvaluatingRowVisitor implements RowVisitor {
private List<ItemUpdate> result;
private QAWarningStore warningStore;
public EvaluatingRowVisitor(List<ItemUpdate> result, QAWarningStore warningStore) {
this.result = result;
this.warningStore = warningStore;
}
@Override
public void start(Project project) {
;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
ExpressionContext ctxt = new ExpressionContext(baseIri, rowIndex, row, project.columnModel, warningStore);
result.addAll(evaluateItemDocuments(ctxt));
return false;
}
@Override
public void end(Project project) {
;
}
}
static public WikibaseSchema reconstruct(JSONObject o)
throws JSONException {
return reconstruct(o.toString());
}
static public WikibaseSchema reconstruct(String json) throws JSONException {
ObjectMapper mapper = new ObjectMapper();
try {
return mapper.readValue(json, WikibaseSchema.class);
} catch (JsonParseException e) {
throw new JSONException(e.toString());
} catch (JsonMappingException e) {
throw new JSONException(e.toString());
} catch (IOException e) {
throw new JSONException(e.toString());
}
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("itemDocuments");
writer.array();
for (WbItemDocumentExpr changeExpr : itemDocumentExprs) {
changeExpr.write(writer, options);
}
writer.endArray();
writer.endObject();
}
static public WikibaseSchema load(Project project, JSONObject obj)
throws Exception {
return reconstruct(obj);
}
@Override
public void onBeforeSave(Project project) {
}
@Override
public void onAfterSave(Project project) {
}
@Override
public void dispose(Project project) {
}
@Override
public boolean equals(Object other) {
if (other == null || !WikibaseSchema.class.isInstance(other)) {
return false;
}
WikibaseSchema otherSchema = (WikibaseSchema) other;
return itemDocumentExprs.equals(otherSchema.getItemDocumentExpressions());
}
}

28
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/FullyPropertySerializingValueSnak.java Normal file
View File

@ -0,0 +1,28 @@
package org.openrefine.wikidata.schema.entityvalues;
import org.wikidata.wdtk.datamodel.implementation.ValueSnakImpl;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A tweaked version of {@link SnakImpl} that serializes
* the full property (not just its PID), so that we can also
* get the label for that property and display it in the UI
* without having to query the remove server.
*
* @author Antonin Delpeuch
*
*/
public class FullyPropertySerializingValueSnak extends ValueSnakImpl {
public FullyPropertySerializingValueSnak(PropertyIdValue property, Value value) {
super(property, value);
}
@JsonProperty("full_property")
public PropertyIdValue getFullPropertyId() {
return getPropertyId();
}
}

60
extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/PrefetchedEntityIdValue.java Normal file
View File

@ -0,0 +1,60 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema.entityvalues;
import java.util.List;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
/**
* An entity id value that also comes with a label and possibly types.
*
* The rationale behind this classes is that OpenRefine already stores labels
* and types for the Wikidata items it knows about (in the reconciliation data),
* so it is worth keeping this data to avoid re-fetching it when we need it.
*
* @author Antonin Delpeuch
*
*/
public interface PrefetchedEntityIdValue extends EntityIdValue {
/**
* This should return the label "as we got it", with no guarantee that it is
* current or that its language matches that of the user. In general though,
* that should be the case if the user always uses OpenRefine with the same
* language settings.
*
* @return the preferred label of the entity
*/
public String getLabel();
/**
* Returns a list of types for this item. Again these are the types as they were
* originally fetched from the reconciliation interface: they can diverge from
* what is currently on the item.
*
* Empty lists should be returned for
*/
public List<String> getTypes();
}

Some files were not shown because too many files have changed in this diff Show More