s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into fix/allow/nonnumerical/exportname

Browse Source
This commit is contained in:
Jacky 2018-07-07 14:18:04 -04:00 committed by GitHub
commit 5f4f2861de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
400 changed files with 23524 additions and 1977 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.classpath
View File

@ -4,13 +4,15 @@
<classpathentry kind="src" path="main/resources"/>
<classpathentry kind="src" path="extensions/jython/tests/src"/>
<classpathentry kind="src" path="server/src"/>
<classpathentry kind="src" path="main/tests/server/src"/>
<classpathentry kind="src" path="extensions/gdata/src"/>
<classpathentry kind="src" path="extensions/database/src"/>
<classpathentry kind="src" path="extensions/database/test"/>
<classpathentry kind="src" path="extensions/jython/src"/>
<classpathentry kind="src" path="extensions/pc-axis/src"/>
<classpathentry kind="src" path="extensions/sample/src"/>
<classpathentry kind="src" path="main/tests/server/src"/>
<classpathentry kind="src" path="extensions/wikidata/src"/>
<classpathentry kind="src" path="extensions/wikidata/tests/src"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/ant-tools-1.8.0.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/arithcode-1.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/butterfly-1.0.1.jar" sourcepath="main/webapp/WEB-INF/lib-src/butterfly-1.0.1-sources.jar"/>
@ -44,14 +46,14 @@
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/serializer.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/xercesImpl-2.11.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/xml-apis.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpclient-4.2.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpcore-4.2.4.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpclient-4.5.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpcore-4.4.9.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/signpost-commonshttp4-1.2.1.2.jar" sourcepath="main/webapp/WEB-INF/lib-src/signpost-commonshttp4-1.2.1.2-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/signpost-core-1.2.1.2.jar" sourcepath="main/webapp/WEB-INF/lib-src/signpost-core-1.2.1.2-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/guava-13.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/jsr305-1.3.9.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/fluent-hc-4.2.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpmime-4.2.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/fluent-hc-4.5.5.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpmime-4.5.5.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/transaction-api-1.1.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-3.13-20150929.jar"/>
@ -127,5 +129,7 @@
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jena-tdb2-3.6.0.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jsonld-java-0.11.1.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/libthrift-0.10.0.jar"/>
<classpathentry kind="lib" path="extensions/wikidata/module/MOD-INF/lib/wdtk-datamodel-0.9.0-SNAPSHOT-jar-with-dependencies.jar" sourcepath="extensions/wikidata/module/MOD-INF/lib-src/wdtk-datamodel-0.9.0-SNAPSHOT-sources.jar"/>
<classpathentry kind="lib" path="extensions/wikidata/module/MOD-INF/lib/wdtk-wikibaseapi-0.9.0-SNAPSHOT.jar" sourcepath="extensions/wikidata/module/MOD-INF/lib-src/wdtk-wikibaseapi-0.9.0-SNAPSHOT-sources.jar"/>
<classpathentry kind="output" path="main/webapp/WEB-INF/classes"/>
</classpath>

22
.github/ISSUE_TEMPLATE.md vendored
View File

@ -1,22 +0,0 @@
When reporting a bug please provide the following information to help reproduce the bug:
#### Version of OpenRefine used (Google Refine 2.6, OpenRefine2.8, an other distribution?):
#### Operating Systems and version:
#### Browser + version used - Please note that OpenRefine doesn't support Internet Explorer but works OK in most cases:
#### Steps followed to create the issue:
#### If you are allowed and are OK with making your data public, it would be awesome if you can include the data causing the issue or a URL pointing to where the data is (if your concerned about keeping your data private, ping us on our mailing list):
#### Current Results:
#### Expected Results:

39
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View File

@ -0,0 +1,39 @@
---
name: Bug report
about: Create a report to help us improve OpenRefine
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Current Results**
What results occured or were shown.
**Expected behavior**
A clear and concise description of what you expected to happen or to show.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS: [e.g. iOS, Windows, Linux]
- Browser [e.g. chrome, firefox, safari, NOTE: OpenRefine does not support IE but works OK in most cases]
- Version [e.g. 22]
**OpenRefine (please complete the following information):**
- Version [e.g. OpenRefine 3.0 Beta]
**Datasets**
If you are allowed and are OK with making your data public, it would be awesome if you can include or attach the data causing the issue or a URL pointing to where the data is.
If you are concerned about keeping your data private, ping us on our [mailing list](https://groups.google.com/forum/#!forum/openrefine)
**Additional context**
Add any other context about the problem here.

17
.github/ISSUE_TEMPLATE/feature_request.md vendored Normal file
View File

@ -0,0 +1,17 @@
---
name: Feature request
about: Suggest an idea for OpenRefine
---
**Is your feature request related to a problem or area of OpenRefine? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] or, It would be easier if OpenRefine did [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

1
.travis.yml
View File

@ -1,6 +1,7 @@
language: java
jdk:
- oraclejdk8
- openjdk8
addons:
mariadb: '10.0'

77
AUTHORS.md Normal file
View File

@ -0,0 +1,77 @@
This file lists the contributors to OpenRefine.
EMERITUS CREATORS (no longer active but honored for bringing OpenRefine to life)
---------------------
- David Huynh
- Stefano Mazzocchi
EMERITUS CONTRIBUTORS (no longer active but honored here for their previous contributions)
---------------------
- Vishal Talwar
- Jeff Fry
- Will Moffat
- James Home
- Heather Campbell
CURRENT CONTRIBUTORS
--------------------
See up to date list at https://github.com/OpenRefine/OpenRefine/graphs/contributors
- Iain Sproat
- Tom Morris
- Thad Guidry
- Martin Magdinier
- Paul Makepeace
- Tomaž Šolc
- Gabriel Sjoberg
- Rod Salazar
- pxb
- Qi
- Antonin Delpeuch
- Owen Stephens
- Ettore Rizza
- Fabio Tacchelli
- noamoss
- ROMitat
- Jesus M. Castagnetto
- Pablo Moyano
- David Leoni
- Cora Johnson-Roberson
- Pei Long Hui
- Rudy Alvarez
- Mateja Verlic Bruncic
- nestorjal
- Alexey Medvetsky
- Remi Rampin
- lispc
- Bob Harper
- Felix Lohmeier
- Shixiong Zhu
- Ankit Sardesai
- Scott Wiedemann
- Ryo Yamazaki
- Ralf Claussnitzer
- Charles Pritchard
- Maxim Galushka
- Evelyn Mitchell
- Andreas Kohn
- Honza
- Ram Ezrach
- Daniel Berthereau
- Gideon Thomas
- José Vitor Hisse
- Vitor Baptista
- Joe Wicentowski
- mpc9000
- Dan Michael O.
- Adi Eyal
- Shrubhra Sharma
- Fadi Maali
- Aubrey Mcfato
- Matthias Findeisen
- Mathieu Saby
- Allan Nordhøy
- Tony Opara

85
README.md
View File

@ -2,9 +2,9 @@
[![Join the chat at https://gitter.im/OpenRefine/OpenRefine](https://badges.gitter.im/OpenRefine/OpenRefine.svg)](https://gitter.im/OpenRefine/OpenRefine?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://travis-ci.org/OpenRefine/OpenRefine.png?branch=master)](https://travis-ci.org/OpenRefine/OpenRefine) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/210578308bba42c5922c767493e83cf4)](https://www.codacy.com/app/OpenRefine/OpenRefine) [![Codacy Badge](https://api.codacy.com/project/badge/Coverage/210578308bba42c5922c767493e83cf4)](https://www.codacy.com/app/OpenRefine/OpenRefine) [![Translation progress](https://hosted.weblate.org/widgets/openrefine/-/svg-badge.svg)](https://hosted.weblate.org/engage/openrefine/?utm_source=widget)
OpenRefine is a power tool that allows you to load data, understand it,
OpenRefine is a Java-based power tool that allows you to load data, understand it,
clean it up, reconcile it, and augment it with data coming from
the web. All with the comfort and privacy of your own computer.
the web. All from a web browser and the comfort and privacy of your own computer.
[<img src="https://github.com/OpenRefine/OpenRefine/blob/master/graphics/icon/open-refine-320px.png" align="right">](http://openrefine.org)
@ -12,6 +12,13 @@ Download
-----------------------
* [OpenRefine Releases](https://github.com/OpenRefine/OpenRefine/releases)
Run from source
------------------
If you have cloned this repository to your computer, you can run OpenRefine with:
* `./refine` on Mac OS and Linux
* `refine.bat` on Window
This requires JDK 8 and Ant.
Documentation and Videos
-------------------------
* [Documentation Wiki](https://github.com/OpenRefine/OpenRefine/wiki/Documentation-For-Users)
@ -43,76 +50,4 @@ was acquired by Google, Inc. in July 2010 and the product was renamed Google Ref
In October 2012, it was renamed OpenRefine as it transitioned to a
community-supported product.
EMERITUS CREATORS (no longer active but honored for bringing OpenRefine to life)
---------------------
- David Huynh <dfhuynh@google.com>
- Stefano Mazzocchi <stefanom@google.com>
EMERITUS CONTRIBUTORS (no longer active but honored here for their previous contributions)
---------------------
- Vishal Talwar <vtalwar@google.com>
- Jeff Fry <jfry@google.com>
- Will Moffat <wdm@google.com>
- James Home <jameshome@google.com>
- Heather Campbell <campbellh@google.com>
CURRENT CONTRIBUTORS
--------------------
- Iain Sproat <iainsproat@gmail.com>
- Tom Morris <tfmorris@gmail.com>
- Thad Guidry <thadguidry@gmail.com>
- Martin Magdinier <martin.magdinier@gmail.com>
- Paul Makepeace <paulm@paulm.com>
- Tomaž Šolc <tomaz.solc@zemanta.com>
- Gabriel Sjoberg <GabrielSjoberg@gmail.com>
- Rod Salazar <rodrod.salazar@gmail.com>
- pxb <pxb1988@gmail.com>
- Qi <jackyq2015@gmail.com>
- Antonin Delpeuch <antonin@delpeuch.eu>
- Owen Stephens <owen@ostephens.com>
- Ettore Rizza <ettorerizza@gmail.com>
- Fabio Tacchelli <fabio.tacchelli@gmail.com>
- noamoss
- ROMitat
- Jesus M. Castagnetto
- Pablo Moyano <ultraklon@gmail.com>
- David Leoni
- Cora Johnson-Roberson
- Pei Long Hui
- Rudy Alvarez <rudygt@gmail.com>
- Mateja Verlic Bruncic
- nestorjal <nbeltran@humboldt.org.co>
- Alexey Medvetsky <am@opendata.by>
- Remi Rampin <remirampin@gmail.com>
- lispc <mycinbrin@gmail.com>
- Bob Harper
- Felix Lohmeier <mail@felixlohmeier.de>
- Shixiong Zhu
- Ankit Sardesai
- Scott Wiedemann
- Ryo Yamazaki
- Ralf Claussnitzer
- Charles Pritchard
- Maxim Galushka
- Evelyn Mitchell
- Andreas Kohn
- Honza
- Ram Ezrach <ram.ezrach@gmail.com>
- Daniel Berthereau <daniel.on.github-no-spam@berthereau.net>
- Gideon Thomas <gideon@mozillafoundation.org>
- José Vitor Hisse
- Vitor Baptista <vitor@vitorbaptista.com>
- Joe Wicentowski <joewiz@gmail.com>
- mpc9000
- Dan Michael O. <d.m.heggo@ub.uio.no>
- Adi Eyal
- Shrubhra Sharma
- Fadi Maali <fadi.maali@zalando.ie>
- Aubrey Mcfato <aubreymcfato@gmail.com>
- Matthias Findeisen
- Mathieu Saby
- Allan Nordhøy <epost@anotheragency.no>
- Tony Opara <tcbuzor@gmail.com>
See `AUTHORS.md` for the list of OpenRefine contributors.

2
WeLoveOurBackers.md
View File

@ -4,4 +4,4 @@ An evergrowing list of our forever immortalized OpenRefine Backers that we LOVE
2. PaulZH -
3. Jens Willmer -
4. mroswell -
5. Google Inc. News Lab
5. Google News Initiative - https://newsinitiative.withgoogle.com/

8
appveyor.yml
View File

@ -11,18 +11,18 @@ clone_depth: 5
skip_branch_with_pr: true
environment:
ANT_HOME: C:\apache-ant-1.10.1
ANT_HOME: C:\apache-ant-1.10.3
JAVA_HOME: C:\Program Files (x86)\Java\jdk1.8.0
matrix:
fast_finish: true
install:
- cmd: if not exist %ANT_HOME% pushd . && cd \ && appveyor DownloadFile http://apache.spinellicreations.com/ant/binaries/apache-ant-1.10.1-bin.zip && 7z x apache-ant-1.10.1-bin.zip && popd
- cmd: if not exist %ANT_HOME% pushd . && cd \ && appveyor DownloadFile http://apache.spinellicreations.com/ant/binaries/apache-ant-1.10.3-bin.zip && 7z x apache-ant-1.10.3-bin.zip && popd
cache:
- C:\apache-ant-1.10.1
- apache-ant-1.10.1-bin.zip
- C:\apache-ant-1.10.3
- apache-ant-1.10.3-bin.zip
build: off
# scripts to run before test

9
extensions/build.xml
View File

@ -13,7 +13,8 @@
<ant dir="jython/" target="build" />
<ant dir="gdata/" target="build" />
<ant dir="pc-axis/" target="build" />
<ant dir="database/" target="build" />
<ant dir="database/" target="build" />
<ant dir="wikidata/" target="build" />
</target>
<target name="clean">
@ -22,12 +23,14 @@
<ant dir="jython/" target="clean" />
<ant dir="gdata/" target="clean" />
<ant dir="pc-axis/" target="clean" />
<ant dir="database/" target="clean" />
<ant dir="database/" target="clean" />
<ant dir="wikidata/" target="clean" />
</target>
<target name="test">
<echo message="Testing extensions" />
<ant dir="jython/" target="test" />
<ant dir="database/" target="test" />
<ant dir="database/" target="test" />
<ant dir="wikidata/" target="test" />
</target>
</project>

54
extensions/database/module/langs/translation-jp.json Normal file
View File

@ -0,0 +1,54 @@
{
"database-import": {
"title": "データベースサーバー",
"preparing": "準備しています...",
"checking": "クエリーをチェックしています...",
"creating": "プロジェクトを作成中..."
},
"database-source": {
"alert-host": "Database Hostを指定してください",
"alert-port": "Database Portを指定してください",
"alert-user": "Database Userを指定してください",
"alert-password": "Database Passwordを指定してください",
"alert-connection-name": "Connection Nameを指定してください",
"alert-initial-database": "第一Databaseを指定してください",
"alert-query": "有効なクエリーを設定してください",
"alert-invalid-query-keyword": "クエリーにデータ操作語があります:",
"alert-invalid-query-select": "クエリーはSELECTで始まります",
"form-validation-failure" : "接続は無効です！",
"alert-connection-edit": "接続の設定に成功しました",
"connectionNameLabel": "接続先:",
"databaseTypeLabel": "Type:",
"databaseHostLabel": "Host:",
"databasePortLabel": "Port:",
"databaseUserLabel": "User:",
"databasePasswordLabel": "Password:",
"databaseNameLabel": "Database:",
"databaseSchemaLabel": "Schema:",
"databaseTestButton": "テスト",
"databaseSaveButton": "保存",
"databaseConnectButton": "接続",
"newConnectionButtonDiv": "新しい接続",
"savedConnectionSpan": "保存済み接続"
},
"database-parsing": {
"start-over": "&laquo; やり直し",
"conf-pars": "パースオプションの設定",
"proj-name": "プロジェクト&nbsp;名",
"create-proj": "プロジェクトを作成 &raquo;",
"updating-preview": "プレビューを更新中...",
"worksheet": "シート名",
"option": "オプション",
"preview-button": "プレビュー&nbsp;更新",
"ignore-first": "最初の",
"ignore": "行を無視",
"parse-next": "その次の",
"parse": "行をカラム名として解釈",
"discard-next": "最初の",
"discard": "行分を破棄",
"limit-next": "最大で",
"limit": "行を読み込む",
"store-row": "空白行を保存",
"store-cell": "空白行をnullとして保存"
}
}

2
extensions/database/src/com/google/refine/extension/database/DatabaseImportController.java
View File

@ -55,7 +55,7 @@ import com.google.refine.importing.ImportingController;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.model.Project;
import com.google.refine.model.medadata.ProjectMetadata;
import com.google.refine.model.metadata.ProjectMetadata;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities;

2
extensions/database/test/com/google/refine/extension/database/DatabaseImportControllerTest.java
View File

@ -25,7 +25,6 @@ import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import com.google.refine.ProjectManager;
import com.google.refine.model.medadata.ProjectMetadata;
import com.google.refine.RefineServlet;
import com.google.refine.extension.database.mysql.MySQLDatabaseService;
import com.google.refine.extension.database.stub.RefineDbServletStub;
@ -33,6 +32,7 @@ import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.io.FileProjectManager;
import com.google.refine.model.Project;
import com.google.refine.model.metadata.ProjectMetadata;

2
extensions/database/test/com/google/refine/extension/database/cmd/SavedConnectionCommandTest.java
View File

@ -25,7 +25,6 @@ import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
import com.google.refine.ProjectManager;
import com.google.refine.model.medadata.ProjectMetadata;
import com.google.refine.RefineServlet;
import com.google.refine.extension.database.DBExtensionTestUtils;
import com.google.refine.extension.database.DBExtensionTests;
@ -36,6 +35,7 @@ import com.google.refine.extension.database.stub.RefineDbServletStub;
import com.google.refine.importing.ImportingManager;
import com.google.refine.io.FileProjectManager;
import com.google.refine.model.Project;
import com.google.refine.model.metadata.ProjectMetadata;
public class SavedConnectionCommandTest extends DBExtensionTests{

2
extensions/gdata/module/langs/translation-jp.json
View File

@ -5,7 +5,7 @@
"title": "一般公開文書(public document)",
"import-by-url": "一般公開している(<em>public</em>)Google SpreadsheetかFusion TableのURLを入力してください:",
"next->": "次へ &raquo;",
"auth-doc": "認証済み文書(authorized document)",
"auth-doc": "制限付き文書(authorized document)",
"please": "まず",
"sign-in": "サインインして認証",
"sign-out": "サインアウト",

2
extensions/gdata/src/com/google/refine/extension/gdata/FusionTableImporter.java
View File

@ -43,7 +43,7 @@ import com.google.refine.importers.TabularImportingParserBase;
import com.google.refine.importers.TabularImportingParserBase.TableDataReader;
import com.google.refine.importing.ImportingJob;
import com.google.refine.model.Project;
import com.google.refine.model.medadata.ProjectMetadata;
import com.google.refine.model.metadata.ProjectMetadata;
import com.google.refine.util.JSONUtilities;
/**

2
extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java
View File

@ -19,7 +19,7 @@ import com.google.refine.importers.TabularImportingParserBase;
import com.google.refine.importers.TabularImportingParserBase.TableDataReader;
import com.google.refine.importing.ImportingJob;
import com.google.refine.model.Project;
import com.google.refine.model.medadata.ProjectMetadata;
import com.google.refine.model.metadata.ProjectMetadata;
import com.google.refine.util.JSONUtilities;
public class GDataImporter {

2
extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java
View File

@ -37,7 +37,7 @@ import com.google.refine.importing.ImportingController;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.model.Project;
import com.google.refine.model.medadata.ProjectMetadata;
import com.google.refine.model.metadata.ProjectMetadata;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities;

40
extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java
View File

@ -26,6 +26,8 @@ import com.google.api.services.fusiontables.FusiontablesScopes;
import com.google.api.services.sheets.v4.Sheets;
import com.google.api.services.sheets.v4.SheetsScopes;
import com.google.refine.ProjectManager;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.util.ParsingUtilities;
import edu.mit.simile.butterfly.ButterflyModule;
@ -43,6 +45,13 @@ abstract public class GoogleAPIExtension {
private static final String[] SCOPES = {DriveScopes.DRIVE, SheetsScopes.SPREADSHEETS, FusiontablesScopes.FUSIONTABLES};
private static PreferenceStore prefStore = ProjectManager.singleton.getPreferenceStore();
private static final String CONNECT_TIME_OUT_KEY = "googleConnectTimeOut";
private static final String READ_TIME_OUT_KEY = "googleReadTimeOut";
private static final int CONNECT_TIME_OUT_DEFAULT = 3 * 60000; // 3 minutes connect timeout
private static final int READ_TIME_OUT_DEFAULT = 3 * 60000; // 3 minutes read timeout
static public String getAuthorizationUrl(ButterflyModule module, HttpServletRequest request)
throws MalformedURLException {
String authorizedUrl = makeRedirectUrl(module, request);
@ -101,8 +110,8 @@ abstract public class GoogleAPIExtension {
@Override
public void initialize(HttpRequest httpRequest) throws IOException {
credential.initialize(httpRequest);
httpRequest.setConnectTimeout(3 * 60000); // 3 minutes connect timeout
httpRequest.setReadTimeout(3 * 60000); // 3 minutes read timeout
httpRequest.setConnectTimeout(3 * 60000);
httpRequest.setReadTimeout(3 * 60000);
}
})
.setApplicationName(SERVICE_APP_NAME).build();
@ -141,10 +150,29 @@ abstract public class GoogleAPIExtension {
* @throws IOException
*/
public static Sheets getSheetsService(String token) throws IOException {
return new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY,
new GoogleCredential().setAccessToken(token))
.setApplicationName(SERVICE_APP_NAME)
.build();
GoogleCredential credential = new GoogleCredential().setAccessToken(token);
int connectTimeout = getConnectTimeout();
int readTimeout = getReadTimeout();
return new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential).setHttpRequestInitializer(new HttpRequestInitializer() {
@Override
public void initialize(HttpRequest httpRequest) throws IOException {
credential.initialize(httpRequest);
httpRequest.setConnectTimeout(connectTimeout);
httpRequest.setReadTimeout(readTimeout); // 3 minutes read timeout
}
})
.setApplicationName(SERVICE_APP_NAME).build();
}
private static int getConnectTimeout() {
return prefStore.get(CONNECT_TIME_OUT_KEY) == null ? CONNECT_TIME_OUT_DEFAULT :
Integer.parseInt((String) prefStore.get(CONNECT_TIME_OUT_KEY));
}
private static int getReadTimeout() {
return prefStore.get(READ_TIME_OUT_KEY) == null ? READ_TIME_OUT_DEFAULT :
Integer.parseInt((String) prefStore.get(READ_TIME_OUT_KEY));
}
public static String extractSpreadSheetId(String url)

2
extensions/pc-axis/src/com/google/refine/pcaxis/PCAxisImporter.java
View File

@ -44,7 +44,7 @@ import org.slf4j.LoggerFactory;
import com.google.refine.importers.TabularImportingParserBase;
import com.google.refine.importing.ImportingJob;
import com.google.refine.model.Project;
import com.google.refine.model.medadata.ProjectMetadata;
import com.google.refine.model.metadata.ProjectMetadata;
import com.google.refine.util.JSONUtilities;
public class PCAxisImporter extends TabularImportingParserBase {

24
extensions/wikidata/.classpath Normal file
View File

@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
<classpathentry kind="src" path="resources"/>
<classpathentry kind="lib" path="/OpenRefine/server/lib/servlet-api-2.5.jar" sourcepath="/OpenRefine/server/lib-src/servlet-api-2.5-sources.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/butterfly-1.0.1.jar" sourcepath="/OpenRefine/main/webapp/WEB-INF/lib-src/butterfly-trunk.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/tests/server/lib/testng-6.8.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/slf4j-api-1.5.6.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/log4j-1.2.15.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/slf4j-log4j12-1.5.6.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/webapp/WEB-INF/lib/velocity-1.5.jar"/>
<classpathentry kind="lib" path="/OpenRefine/server/lib/jetty-6.1.22.jar"/>
<classpathentry kind="lib" path="/OpenRefine/server/lib/jetty-util-6.1.22.jar"/>
<classpathentry kind="lib" path="/OpenRefine/main/tests/server/lib/mockito-all-1.9.5.jar"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6">
<attributes>
<attribute name="owner.project.facets" value="java"/>
</attributes>
</classpathentry>
<classpathentry combineaccessrules="false" kind="src" path="/OpenRefine"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/wdtk-datamodel-0.8.0-SNAPSHOT-jar-with-dependencies.jar" sourcepath="module/MOD-INF/lib-src/wdtk-datamodel-0.8.0-SNAPSHOT-sources.jar"/>
<classpathentry kind="lib" path="module/MOD-INF/lib/wdtk-wikibaseapi-0.8.0-SNAPSHOT.jar" sourcepath="module/MOD-INF/lib-src/wdtk-wikibaseapi-0.8.0-SNAPSHOT-sources.jar"/>
<classpathentry kind="output" path="module/MOD-INF/classes"/>
</classpath>

3
extensions/wikidata/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
tests/classes/
tests/report/
tests/test-output/

133
extensions/wikidata/build.xml Normal file
View File

@ -0,0 +1,133 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Google Refine Wikidata Extension Build File +-->
<project name="refine-wikidata-extension" default="build" basedir="." xmlns:jacoco="antlib:org.jacoco.ant">
<property environment="env"/>
<condition property="version" value="trunk">
<not><isset property="version"/></not>
</condition>
<condition property="revision" value="rXXXX">
<not><isset property="revision"/></not>
</condition>
<condition property="full_version" value="0.0.0.0">
<not><isset property="full_version"/></not>
</condition>
<condition property="dist.dir" value="dist">
<not><isset property="dist.dir"/></not>
</condition>
<property name="fullname" value="${name}-${version}-${revision}" />
<property name="java_version" value="1.8"/>
<property name="refine.dir" value="${basedir}/../../main" />
<property name="refine.webinf.dir" value="${refine.dir}/webapp/WEB-INF" />
<property name="refine.modinf.dir" value="${refine.dir}/webapp/modules/core/MOD-INF" />
<property name="refine.classes.dir" value="${refine.webinf.dir}/classes" />
<property name="refine.lib.dir" value="${refine.webinf.dir}/lib" />
<property name="server.dir" value="${basedir}/../../server" />
<property name="server.lib.dir" value="${server.dir}/lib" />
<property name="refine.tests.lib.dir" value="${refine.dir}/tests/server/lib" />
<property name="refine.tests.classes.dir" value="${refine.dir}/tests/server/classes" />
<property name="src.dir" value="${basedir}/src" />
<property name="module.dir" value="${basedir}/module" />
<property name="modinf.dir" value="${module.dir}/MOD-INF" />
<property name="lib.dir" value="${modinf.dir}/lib" />
<property name="classes.dir" value="${modinf.dir}/classes" />
<property name="extension.tests.dir" value="${basedir}/tests" />
<property name="tests.build.dir" value="${extension.tests.dir}/build" />
<property name="tests.src.dir" value="${extension.tests.dir}/src" />
<property name="tests.classes.dir" value="${extension.tests.dir}/classes" />
<property name="tests.report.dir" value="${extension.tests.dir}/report"/>
<property name="tests.report.html.dir" value="${tests.report.dir}/html"/>
<property name="tests.report.xml.path" value="${tests.report.dir}/jacoco.xml"/>
<path id="class.path">
<fileset dir="${lib.dir}">
<include name="**/*.jar" />
</fileset>
<fileset dir="${refine.lib.dir}">
<include name="**/*.jar" />
</fileset>
<fileset dir="${server.lib.dir}">
<include name="**/*.jar" />
</fileset>
<pathelement path="${refine.classes.dir}"/>
<pathelement path="${classes.dir}" />
</path>
<path id="extension.tests.class.path">
<path refid="class.path"/>
<pathelement location="${tests.classes.dir}"/>
<pathelement location="${refine.tests.classes.dir}"/>
<pathelement location="${refine.tests.lib.dir}/testng-6.8.jar"/>
<fileset dir="${refine.tests.lib.dir}">
<include name="**/*.jar" />
</fileset>
</path>
<taskdef resource="testngtasks" classpath="${refine.tests.lib.dir}/testng-6.8.jar"/>
<taskdef uri="antlib:org.jacoco.ant" resource="org/jacoco/ant/antlib.xml">
<classpath path="${refine.tests.lib.dir}/jacocoant.jar"/>
</taskdef>
<target name="build_java">
<mkdir dir="${classes.dir}" />
<javac source="${java_version}" target="${java_version}" encoding="utf-8" destdir="${classes.dir}" debug="true" includeAntRuntime="no">
<src path="${src.dir}"/>
<classpath refid="class.path" />
</javac>
</target>
<target name="build" depends="build_java"/>
<target name="build_tests" depends="build">
<mkdir dir="${tests.classes.dir}" />
<javac source="${java_version}" target="${java_version}" encoding="utf-8" srcdir="${tests.src.dir}" destdir="${tests.classes.dir}" debug="true" includeAntRuntime="no">
<classpath refid="extension.tests.class.path" />
</javac>
<copy file="${tests.src.dir}/tests.log4j.properties" tofile="${tests.classes.dir}/tests.log4j.properties"/>
</target>
<target name="test" depends="build_tests">
<jacoco:coverage destfile="${tests.report.dir}/jacoco.exec">
<testng verbose="2" haltOnFailure="true" workingdir="${extension.tests.dir}"
listener="org.testng.reporters.DotTestListener" excludedgroups="broken"
classpathref="extension.tests.class.path">
<xmlfileset file="${extension.tests.dir}/conf/tests.xml"/>
</testng>
</jacoco:coverage>
<jacoco:report>
<executiondata>
<file file="${tests.report.dir}/jacoco.exec"/>
</executiondata>
<structure name="Wikidata extension">
<classfiles>
<fileset dir="${classes.dir}" />
</classfiles>
<sourcefiles encoding="UTF-8">
<fileset dir="${src.dir}"/>
</sourcefiles>
</structure>
<html destdir="${tests.report.html.dir}"/>
<xml destfile="${tests.report.xml.path}"/>
</jacoco:report>
</target>
<target name="clean">
<delete dir="${classes.dir}" />
<delete dir="${tests.classes.dir}" />
<delete dir="${tests.report.dir}" />
<delete dir="${tests.build.dir}" />
</target>
</project>

32
extensions/wikidata/credits.txt Normal file
View File

@ -0,0 +1,32 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2018 Antonin Delpeuch
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#-------------------------------------------------------------------------------
The following images were obtained from Wikimedia Commons:
- Information.svg by User:Bobarino: (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Information.svg
- Warning.svg by User:Ezekiel63745 (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Information_orange.svg
- Important.svg originally from David Vignoni (GNU LGPL)
https://commons.wikimedia.org/wiki/File:Nuvola_apps_important.svg
- Critical.svg by User:Stannered (CC BY-SA)
https://commons.wikimedia.org/wiki/File:Stop_x_nuvola.svg

78
extensions/wikidata/module/MOD-INF/controller.js Normal file
View File

@ -0,0 +1,78 @@
importPackage(org.openrefine.wikidata.commands);
/*
* Function invoked to initialize the extension.
*/
function init() {
var RefineServlet = Packages.com.google.refine.RefineServlet;
RefineServlet.registerClassMapping(
"org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation$WikibaseSchemaChange",
"org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation$WikibaseSchemaChange");
RefineServlet.registerClassMapping(
"org.openrefine.wikidata.operations.PerformWikibaseEditsOperation$PerformWikibaseEditsChange",
"org.openrefine.wikidata.operations.PerformWikibaseEditsOperation$PerformWikibaseEditsChange");
RefineServlet.cacheClass(Packages.org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation$WikibaseSchemaChange);
RefineServlet.cacheClass(Packages.org.openrefine.wikidata.operations.PerformWikibaseEditsOperation$PerformWikibaseEditsChange);
/*
* Attach a Wikibase schema to each project.
*/
Packages.com.google.refine.model.Project.registerOverlayModel(
"wikibaseSchema",
Packages.org.openrefine.wikidata.schema.WikibaseSchema);
/*
* Operations
*/
Packages.com.google.refine.operations.OperationRegistry.registerOperation(
module, "save-wikibase-schema", Packages.org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation);
Packages.com.google.refine.operations.OperationRegistry.registerOperation(
module, "perform-wikibase-edits", Packages.org.openrefine.wikidata.operations.PerformWikibaseEditsOperation);
/*
* Exporters
*/
var ExporterRegistry = Packages.com.google.refine.exporters.ExporterRegistry;
var QSExporter = Packages.org.openrefine.wikidata.exporters.QuickStatementsExporter;
ExporterRegistry.registerExporter("quickstatements", new QSExporter());
/*
* Commands
*/
RefineServlet.registerCommand(module, "save-wikibase-schema", new SaveWikibaseSchemaCommand());
RefineServlet.registerCommand(module, "preview-wikibase-schema", new PreviewWikibaseSchemaCommand());
RefineServlet.registerCommand(module, "perform-wikibase-edits", new PerformWikibaseEditsCommand());
RefineServlet.registerCommand(module, "login", new LoginCommand());
/*
* Resources
*/
ClientSideResourceManager.addPaths(
"project/scripts",
module,
[
"scripts/menu-bar-extension.js",
"scripts/warningsrenderer.js",
"scripts/langsuggest.js",
"scripts/bettersuggest.js",
"scripts/previewrenderer.js",
"scripts/dialogs/schema-alignment-dialog.js",
"scripts/dialogs/manage-account-dialog.js",
"scripts/dialogs/perform-edits-dialog.js",
"scripts/jquery.uls.data.js",
]);
ClientSideResourceManager.addPaths(
"project/styles",
module,
[
"styles/dialogs/schema-alignment-dialog.css",
"styles/dialogs/manage-account-dialog.less",
"styles/dialogs/perform-edits.less",
]);
}

BIN
extensions/wikidata/module/MOD-INF/lib-src/wdtk-datamodel-0.9.0-SNAPSHOT-sources.jar Normal file
View File

Binary file not shown.

BIN
extensions/wikidata/module/MOD-INF/lib-src/wdtk-wikibaseapi-0.9.0-SNAPSHOT-sources.jar Normal file
View File

Binary file not shown.

BIN
extensions/wikidata/module/MOD-INF/lib/wdtk-datamodel-0.9.0-SNAPSHOT-jar-with-dependencies.jar Normal file
View File

Binary file not shown.

BIN
extensions/wikidata/module/MOD-INF/lib/wdtk-wikibaseapi-0.9.0-SNAPSHOT.jar Normal file
View File

Binary file not shown.

3
extensions/wikidata/module/MOD-INF/module.properties Normal file
View File

@ -0,0 +1,3 @@
name = wikidata
description = OpenRefine Wikidata export
requires = core

BIN
extensions/wikidata/module/images/Critical.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.2 KiB

269
extensions/wikidata/module/images/Critical.svg Normal file
View File

@ -0,0 +1,269 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="240"
height="240"
id="svg4846">
<defs
id="defs4848">
<linearGradient
id="linearGradient5032">
<stop
style="stop-color:#ce1212;stop-opacity:1"
offset="0"
id="stop5034" />
<stop
style="stop-color:#ff1212;stop-opacity:1"
offset="1"
id="stop5036" />
</linearGradient>
<linearGradient
id="linearGradient5024">
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0"
id="stop5026" />
<stop
style="stop-color:#ffffff;stop-opacity:0"
offset="1"
id="stop5028" />
</linearGradient>
<radialGradient
cx="295.47125"
cy="186.09634"
r="179.55"
id="g5"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(1.14292e-8,1.531256,-0.876504,-2.165967e-5,462.7486,-245.0023)">
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0"
id="stop4889" />
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0.5"
id="stop4891" />
<stop
style="stop-color:#d5d5d5;stop-opacity:1"
offset="1"
id="stop4893" />
</radialGradient>
<linearGradient
x1="250.39845"
y1="101.53633"
x2="412.0943"
y2="264.54187"
id="g4"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787565)">
<stop
style="stop-color:#ffd9d9;stop-opacity:1"
offset="0"
id="stop4884" />
<stop
style="stop-color:#ff2727;stop-opacity:1"
offset="1"
id="stop4886" />
</linearGradient>
<linearGradient
x1="187.87357"
y1="224.59892"
x2="581.83746"
y2="483.10001"
id="g3"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787558)">
<stop
style="stop-color:#ec6c60;stop-opacity:1"
offset="0"
id="stop4879" />
<stop
style="stop-color:#d11412;stop-opacity:1"
offset="1"
id="stop4881" />
</linearGradient>
<linearGradient
x1="530.80951"
y1="486.63101"
x2="174.80548"
y2="211.22995"
id="g2"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787565)">
<stop
style="stop-color:#9a0000;stop-opacity:1"
offset="0"
id="stop4874" />
<stop
style="stop-color:#f22803;stop-opacity:1"
offset="1"
id="stop4876" />
</linearGradient>
<linearGradient
x1="41.194874"
y1="616.47717"
x2="118.93135"
y2="527.55511"
id="g1"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(4.556763,0,0,-4.315033,37.49756,2758.519)">
<stop
style="stop-color:#bb0000;stop-opacity:1"
offset="0"
id="stop4869" />
<stop
style="stop-color:#5f0000;stop-opacity:1"
offset="1"
id="stop4871" />
</linearGradient>
<radialGradient
cx="295.47125"
cy="186.09634"
r="179.55"
id="radialGradient4978"
xlink:href="#g5"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0,0.2725026,-0.1559828,-3.8545592e-6,437.26559,613.56411)" />
<linearGradient
x1="250.39845"
y1="101.53633"
x2="412.0943"
y2="264.54187"
id="linearGradient4981"
xlink:href="#g4"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="187.87357"
y1="224.59892"
x2="581.83746"
y2="483.10001"
id="linearGradient4984"
xlink:href="#g3"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="530.80951"
y1="486.63101"
x2="174.80548"
y2="211.22995"
id="linearGradient4987"
xlink:href="#g2"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="41.194874"
y1="616.47717"
x2="118.93135"
y2="527.55511"
id="linearGradient4990"
xlink:href="#g1"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.8109225,0,0,-0.7679041,361.58783,1148.0714)" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5030"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5038"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<filter
id="filter5068">
<feGaussianBlur
id="feGaussianBlur5070"
stdDeviation="0.16424528"
inkscape:collect="always" />
</filter>
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5099"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5101"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5125"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5127"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5153"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(3.7565752,0,0,3.7565752,-1281.1474,-1839.9864)" />
</defs>
<g
transform="translate(-463.84454,-666.74359)"
id="layer1">
<g
id="g5072">
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.5876353,-1.4860452,1.4860452,3.5876353,417.73023,715.73613)"
style="fill:#7e0000;fill-opacity:1"
id="path3717" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.5876353,-1.4860452,1.4860452,3.5876353,425.24338,723.24928)"
style="fill:#000000;fill-opacity:0.25657893;filter:url(#filter5068)"
id="path5042" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.3778534,-1.3991511,1.3991511,3.3778534,427.22392,719.66855)"
style="fill:#e78a8a;fill-opacity:1"
id="path3741" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.0836017,-1.2772686,1.2772686,3.0836017,440.54023,725.18436)"
style="fill:url(#linearGradient5125);fill-opacity:1"
id="path3743" />
<path
d="M 74.625,56.9375 L 56.9375,74.625 L 99.15625,116.84375 L 56.9375,159.0625 L 74.625,176.75 L 116.84375,134.53125 L 159.0625,176.75 L 176.75,159.0625 L 134.53125,116.84375 L 176.75,74.625 L 159.0625,56.9375 L 116.84375,99.15625 L 74.625,56.9375 z "
transform="translate(463.84454,666.74359)"
style="fill:#fdeeee;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:25;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path5139" />
<path
d="M 535.2741,674.9233 L 471.99929,738.19812 L 471.99929,765.37946 C 558.38932,785.44115 593.76927,739.43264 688.23715,764.37675 L 688.23715,738.19812 L 624.84494,674.9233 L 535.2741,674.9233 z "
style="fill:url(#linearGradient5153);fill-opacity:1"
id="path5021" />
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 9.0 KiB

BIN
extensions/wikidata/module/images/Important.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.5 KiB

269
extensions/wikidata/module/images/Important.svg Normal file
View File

@ -0,0 +1,269 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="240"
height="240"
id="svg4846">
<defs
id="defs4848">
<linearGradient
id="linearGradient5032">
<stop
style="stop-color:#ce1212;stop-opacity:1"
offset="0"
id="stop5034" />
<stop
style="stop-color:#ff1212;stop-opacity:1"
offset="1"
id="stop5036" />
</linearGradient>
<linearGradient
id="linearGradient5024">
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0"
id="stop5026" />
<stop
style="stop-color:#ffffff;stop-opacity:0"
offset="1"
id="stop5028" />
</linearGradient>
<radialGradient
cx="295.47125"
cy="186.09634"
r="179.55"
id="g5"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(1.14292e-8,1.531256,-0.876504,-2.165967e-5,462.7486,-245.0023)">
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0"
id="stop4889" />
<stop
style="stop-color:#ffffff;stop-opacity:1"
offset="0.5"
id="stop4891" />
<stop
style="stop-color:#d5d5d5;stop-opacity:1"
offset="1"
id="stop4893" />
</radialGradient>
<linearGradient
x1="250.39845"
y1="101.53633"
x2="412.0943"
y2="264.54187"
id="g4"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787565)">
<stop
style="stop-color:#ffd9d9;stop-opacity:1"
offset="0"
id="stop4884" />
<stop
style="stop-color:#ff2727;stop-opacity:1"
offset="1"
id="stop4886" />
</linearGradient>
<linearGradient
x1="187.87357"
y1="224.59892"
x2="581.83746"
y2="483.10001"
id="g3"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787558)">
<stop
style="stop-color:#ec6c60;stop-opacity:1"
offset="0"
id="stop4879" />
<stop
style="stop-color:#d11412;stop-opacity:1"
offset="1"
id="stop4881" />
</linearGradient>
<linearGradient
x1="530.80951"
y1="486.63101"
x2="174.80548"
y2="211.22995"
id="g2"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.960006,0,0,0.960006,11.68071,9.787565)">
<stop
style="stop-color:#9a0000;stop-opacity:1"
offset="0"
id="stop4874" />
<stop
style="stop-color:#f22803;stop-opacity:1"
offset="1"
id="stop4876" />
</linearGradient>
<linearGradient
x1="41.194874"
y1="616.47717"
x2="118.93135"
y2="527.55511"
id="g1"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(4.556763,0,0,-4.315033,37.49756,2758.519)">
<stop
style="stop-color:#bb0000;stop-opacity:1"
offset="0"
id="stop4869" />
<stop
style="stop-color:#5f0000;stop-opacity:1"
offset="1"
id="stop4871" />
</linearGradient>
<radialGradient
cx="295.47125"
cy="186.09634"
r="179.55"
id="radialGradient4978"
xlink:href="#g5"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0,0.2725026,-0.1559828,-3.8545592e-6,437.26559,613.56411)" />
<linearGradient
x1="250.39845"
y1="101.53633"
x2="412.0943"
y2="264.54187"
id="linearGradient4981"
xlink:href="#g4"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="187.87357"
y1="224.59892"
x2="581.83746"
y2="483.10001"
id="linearGradient4984"
xlink:href="#g3"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="530.80951"
y1="486.63101"
x2="174.80548"
y2="211.22995"
id="linearGradient4987"
xlink:href="#g2"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.1708429,0,0,0.1708429,356.99346,658.90657)" />
<linearGradient
x1="41.194874"
y1="616.47717"
x2="118.93135"
y2="527.55511"
id="linearGradient4990"
xlink:href="#g1"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(0.8109225,0,0,-0.7679041,361.58783,1148.0714)" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5030"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5038"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<filter
id="filter5068">
<feGaussianBlur
id="feGaussianBlur5070"
stdDeviation="0.16424528"
inkscape:collect="always" />
</filter>
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5099"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5101"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="-6.6274176"
y1="48.000011"
x2="15.999988"
y2="-6.6274176"
id="linearGradient5125"
xlink:href="#linearGradient5032"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5127"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="466.6875"
y1="664.79169"
x2="466.6875"
y2="709.06689"
id="linearGradient5153"
xlink:href="#linearGradient5024"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(3.7565752,0,0,3.7565752,-1281.1474,-1839.9864)" />
</defs>
<g
transform="translate(-463.84454,-666.74359)"
id="layer1">
<g
id="g5072">
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.5876353,-1.4860452,1.4860452,3.5876353,417.73023,715.73613)"
style="fill:#7e0000;fill-opacity:1"
id="path3717" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.5876353,-1.4860452,1.4860452,3.5876353,425.24338,723.24928)"
style="fill:#000000;fill-opacity:0.25657893;filter:url(#filter5068)"
id="path5042" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.3778534,-1.3991511,1.3991511,3.3778534,427.22392,719.66855)"
style="fill:#e78a8a;fill-opacity:1"
id="path3741" />
<path
d="M 32.000001,1.0658141e-014 L 54.627418,9.3725836 L 64,32.000001 L 54.627416,54.627418 L 31.999999,64 L 9.3725824,54.627416 L 1.0658141e-014,31.999999 L 9.3725836,9.3725824 L 32.000001,1.0658141e-014 z "
transform="matrix(3.0836017,-1.2772686,1.2772686,3.0836017,440.54023,725.18436)"
style="fill:url(#linearGradient5125);fill-opacity:1"
id="path3743" />
<path
d="M 74.625,56.9375 L 56.9375,74.625 L 99.15625,116.84375 L 56.9375,159.0625 L 74.625,176.75 L 116.84375,134.53125 L 159.0625,176.75 L 176.75,159.0625 L 134.53125,116.84375 L 176.75,74.625 L 159.0625,56.9375 L 116.84375,99.15625 L 74.625,56.9375 z "
transform="translate(463.84454,666.74359)"
style="fill:#fdeeee;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:25;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path5139" />
<path
d="M 535.2741,674.9233 L 471.99929,738.19812 L 471.99929,765.37946 C 558.38932,785.44115 593.76927,739.43264 688.23715,764.37675 L 688.23715,738.19812 L 624.84494,674.9233 L 535.2741,674.9233 z "
style="fill:url(#linearGradient5153);fill-opacity:1"
id="path5021" />
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 9.0 KiB

BIN
extensions/wikidata/module/images/Information.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.2 KiB

252
extensions/wikidata/module/images/Information.svg Normal file
View File

@ -0,0 +1,252 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="256"
height="256"
id="svg2">
<defs
id="defs4">
<linearGradient
id="linearGradient2820">
<stop
style="stop-color:black;stop-opacity:0.3137255"
offset="0"
id="stop2822" />
<stop
style="stop-color:black;stop-opacity:0.19607843"
offset="0.92982459"
id="stop2826" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop2824" />
</linearGradient>
<linearGradient
id="linearGradient10053">
<stop
style="stop-color:white;stop-opacity:1"
offset="0"
id="stop10055" />
<stop
style="stop-color:white;stop-opacity:0.47959185"
offset="1"
id="stop10057" />
</linearGradient>
<linearGradient
id="linearGradient7070">
<stop
style="stop-color:white;stop-opacity:0.94897962"
offset="0"
id="stop7072" />
<stop
style="stop-color:white;stop-opacity:0.74489796"
offset="1"
id="stop7074" />
</linearGradient>
<linearGradient
id="linearGradient4153">
<stop
style="stop-color:#2a4fe4;stop-opacity:1"
offset="0"
id="stop4155" />
<stop
style="stop-color:#447bfa;stop-opacity:1"
offset="0.2"
id="stop7080" />
<stop
style="stop-color:#69a3ff;stop-opacity:1"
offset="0.625"
id="stop7084" />
<stop
style="stop-color:#89beff;stop-opacity:1"
offset="0.75"
id="stop7082" />
<stop
style="stop-color:#80beff;stop-opacity:1"
offset="1"
id="stop4157" />
</linearGradient>
<linearGradient
id="linearGradient4145">
<stop
style="stop-color:black;stop-opacity:1"
offset="0"
id="stop4147" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop4149" />
</linearGradient>
<linearGradient
id="linearGradient25332">
<stop
style="stop-color:#385ae5;stop-opacity:1"
offset="0"
id="stop25334" />
<stop
style="stop-color:#66a1ff;stop-opacity:1"
offset="0"
id="stop26309" />
<stop
style="stop-color:#66a1f0;stop-opacity:0.94117647"
offset="1"
id="stop25336" />
</linearGradient>
<linearGradient
id="linearGradient23386">
<stop
style="stop-color:#2b51e4;stop-opacity:0"
offset="0"
id="stop23388" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop23390" />
</linearGradient>
<linearGradient
id="linearGradient15560">
<stop
style="stop-color:#1864ed;stop-opacity:1"
offset="0"
id="stop15562" />
<stop
style="stop-color:#3a7bef;stop-opacity:0.25510204"
offset="1"
id="stop15564" />
</linearGradient>
<linearGradient
id="linearGradient15536">
<stop
style="stop-color:white;stop-opacity:1"
offset="0"
id="stop15538" />
<stop
style="stop-color:white;stop-opacity:0.59090906"
offset="1"
id="stop15540" />
</linearGradient>
<filter
id="filter9681">
<feGaussianBlur
inkscape:collect="always"
stdDeviation="1.7392633"
id="feGaussianBlur9683" />
</filter>
<filter
id="filter22413">
<feGaussianBlur
inkscape:collect="always"
stdDeviation="0.33355481"
id="feGaussianBlur22415" />
</filter>
<linearGradient
x1="168.97348"
y1="226.79765"
x2="94.360741"
y2="42.124634"
id="linearGradient4159"
xlink:href="#linearGradient4153"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="162.12244"
y1="228.19299"
x2="101.21178"
y2="40.729256"
id="linearGradient7076"
xlink:href="#linearGradient7070"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="127.8125"
y1="31.59375"
x2="127.8125"
y2="118.62509"
id="linearGradient10059"
xlink:href="#linearGradient10053"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="150.76942"
y1="72.579872"
x2="162.2932"
y2="131.31783"
id="linearGradient2818"
xlink:href="#linearGradient2820"
gradientUnits="userSpaceOnUse" />
<radialGradient
cx="130.61937"
cy="132.01637"
r="115.95089"
fx="130.61937"
fy="132.01637"
id="radialGradient3721"
xlink:href="#linearGradient2820"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="127.8125"
y1="31.59375"
x2="127.8125"
y2="118.62509"
id="linearGradient4614"
xlink:href="#linearGradient15536"
gradientUnits="userSpaceOnUse" />
</defs>
<g
style="opacity:1;fill:url(#linearGradient2818);fill-opacity:1;display:inline"
id="layer2">
<path
d="M 246.57026,132.01637 C 246.57026,196.05428 194.65728,247.96726 130.61937,247.96726 C 66.581461,247.96726 14.668479,196.05428 14.66848,132.01637 C 14.668479,67.978461 66.581461,16.065479 130.61937,16.065479 C 194.65728,16.065479 246.57026,67.978461 246.57026,132.01637 L 246.57026,132.01637 z "
transform="matrix(0.995962,0,0,0.995962,-0.171109,-0.165469)"
style="opacity:1;fill:url(#radialGradient3721);fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path12053" />
</g>
<g
style="opacity:1;display:inline"
id="layer4">
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
transform="matrix(1.082583,0,0,1.082583,-14.54063,-17.56537)"
style="opacity:1;fill:#3a73ef;fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path2175" />
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
transform="matrix(1.042113,0,0,1.042113,-9.212006,-12.12368)"
style="opacity:1;fill:url(#linearGradient7076);fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path13592" />
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
transform="matrix(0.951055,0,0,0.951055,2.777393,0.12014)"
style="opacity:1;fill:url(#linearGradient4159);fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path14565" />
</g>
<g
style="opacity:1;display:inline"
id="layer5" />
<g
style="opacity:1;display:inline"
id="layer3">
<g
id="g15568">
<path
d="M 123.30585,191.07575 C 129.02124,187.76374 138.24211,176.49437 141.6732,168.65732 C 141.6732,168.65732 138.1399,167.1567 138.1399,167.1567 C 131.0411,179.5796 120.02693,188.70016 123.71066,173.32458 C 123.71066,173.32458 145.45687,105.04964 145.45687,105.04964 C 139.77784,104.57639 111.00423,109.17626 106.51327,110.10356 C 106.51327,110.10356 105.79723,113.88169 105.79723,113.88169 C 115.61397,114.21756 116.94651,118.2341 116.23663,121.5238 C 116.23663,121.5238 99.927949,173.57586 99.927949,173.57586 C 99.927949,173.57586 97.881571,179.46474 97.947071,183.84902 C 98.105167,193.86006 111.11196,197.7125 123.30585,191.07575 z "
style="opacity:1;fill:white;fill-opacity:1;stroke:none;stroke-width:0.1;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;display:inline"
id="path3339" />
<path
d="M 151.9268,75.963402 C 151.93091,83.589999 145.74948,89.774755 138.12288,89.774755 C 130.49628,89.774755 124.31485,83.589999 124.31896,75.963402 C 124.31485,68.336805 130.49628,62.152049 138.12288,62.152049 C 145.74948,62.152049 151.93091,68.336805 151.9268,75.963402 L 151.9268,75.963402 z "
transform="matrix(1.006031,0,0,1.006031,-1.167682,-0.583638)"
style="opacity:1;fill:white;fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;display:inline"
id="path7182" />
</g>
</g>
<g
style="opacity:1;fill:url(#linearGradient4614);fill-opacity:1;display:inline"
id="layer7">
<path
d="M 128,31.59375 C 86.871341,31.59375 51.74836,57.401715 37.90625,93.6875 C 56.732365,110.35912 96.245675,118.37197 128.03125,118.625 C 128.07285,118.62467 128.02111,118.62533 128.0625,118.625 C 160.09769,118.36974 199.12268,109.67594 217.71875,92.6875 C 203.61988,56.922098 168.75455,31.593751 128,31.59375 z "
style="opacity:0.45;fill:url(#linearGradient4614);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1"
id="path18502" />
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

BIN
extensions/wikidata/module/images/Warning.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.7 KiB

348
extensions/wikidata/module/images/Warning.svg Normal file
View File

@ -0,0 +1,348 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="256"
height="256"
id="svg2"
sodipodi:version="0.32"
inkscape:version="0.48.0 r9654"
sodipodi:docname="Information red.svg"
inkscape:output_extension="org.inkscape.output.svg.inkscape">
<metadata
id="metadata60">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<sodipodi:namedview
inkscape:window-height="851"
inkscape:window-width="1440"
inkscape:pageshadow="2"
inkscape:pageopacity="0.0"
guidetolerance="10.0"
gridtolerance="10.0"
objecttolerance="10.0"
borderopacity="1.0"
bordercolor="#666666"
pagecolor="#ffffff"
id="base"
showgrid="false"
inkscape:zoom="2.8047763"
inkscape:cx="65.457674"
inkscape:cy="135.60528"
inkscape:window-x="-8"
inkscape:window-y="-8"
inkscape:current-layer="layer4"
inkscape:window-maximized="1"
showguides="true"
inkscape:guide-bbox="true">
<sodipodi:guide
orientation="1,0"
position="-224.9697,26.597403"
id="guide3113" />
</sodipodi:namedview>
<defs
id="defs4">
<linearGradient
inkscape:collect="always"
id="linearGradient3139">
<stop
style="stop-color:#ff6600;stop-opacity:1;"
offset="0"
id="stop3141" />
<stop
style="stop-color:#ffffff;stop-opacity:1;"
offset="1"
id="stop3143" />
</linearGradient>
<linearGradient
id="linearGradient3105">
<stop
style="stop-color:#ff2a2a;stop-opacity:1;"
offset="0"
id="stop3107" />
<stop
style="stop-color:#ffffff;stop-opacity:1;"
offset="1"
id="stop3109" />
</linearGradient>
<inkscape:perspective
sodipodi:type="inkscape:persp3d"
inkscape:vp_x="0 : 128 : 1"
inkscape:vp_y="0 : 1000 : 0"
inkscape:vp_z="256 : 128 : 1"
inkscape:persp3d-origin="128 : 85.333333 : 1"
id="perspective62" />
<linearGradient
id="linearGradient2820">
<stop
style="stop-color:black;stop-opacity:0.3137255"
offset="0"
id="stop2822" />
<stop
style="stop-color:black;stop-opacity:0.19607843"
offset="0.92982459"
id="stop2826" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop2824" />
</linearGradient>
<linearGradient
id="linearGradient10053">
<stop
style="stop-color:white;stop-opacity:1"
offset="0"
id="stop10055" />
<stop
style="stop-color:white;stop-opacity:0.47959185"
offset="1"
id="stop10057" />
</linearGradient>
<linearGradient
id="linearGradient7070">
<stop
style="stop-color:white;stop-opacity:0.94897962"
offset="0"
id="stop7072" />
<stop
style="stop-color:white;stop-opacity:0.74489796"
offset="1"
id="stop7074" />
</linearGradient>
<linearGradient
id="linearGradient4153">
<stop
style="stop-color:#2a4fe4;stop-opacity:1"
offset="0"
id="stop4155" />
<stop
style="stop-color:#447bfa;stop-opacity:1"
offset="0.2"
id="stop7080" />
<stop
style="stop-color:#69a3ff;stop-opacity:1"
offset="0.625"
id="stop7084" />
<stop
style="stop-color:#89beff;stop-opacity:1"
offset="0.75"
id="stop7082" />
<stop
style="stop-color:#80beff;stop-opacity:1"
offset="1"
id="stop4157" />
</linearGradient>
<linearGradient
id="linearGradient4145">
<stop
style="stop-color:black;stop-opacity:1"
offset="0"
id="stop4147" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop4149" />
</linearGradient>
<linearGradient
id="linearGradient25332">
<stop
style="stop-color:#385ae5;stop-opacity:1"
offset="0"
id="stop25334" />
<stop
style="stop-color:#66a1ff;stop-opacity:1"
offset="0"
id="stop26309" />
<stop
style="stop-color:#66a1f0;stop-opacity:0.94117647"
offset="1"
id="stop25336" />
</linearGradient>
<linearGradient
id="linearGradient23386">
<stop
style="stop-color:#2b51e4;stop-opacity:0"
offset="0"
id="stop23388" />
<stop
style="stop-color:black;stop-opacity:0"
offset="1"
id="stop23390" />
</linearGradient>
<linearGradient
id="linearGradient15560">
<stop
style="stop-color:#1864ed;stop-opacity:1"
offset="0"
id="stop15562" />
<stop
style="stop-color:#3a7bef;stop-opacity:0.25510204"
offset="1"
id="stop15564" />
</linearGradient>
<linearGradient
id="linearGradient15536">
<stop
style="stop-color:white;stop-opacity:1"
offset="0"
id="stop15538" />
<stop
style="stop-color:white;stop-opacity:0.59090906"
offset="1"
id="stop15540" />
</linearGradient>
<filter
id="filter9681">
<feGaussianBlur
inkscape:collect="always"
stdDeviation="1.7392633"
id="feGaussianBlur9683" />
</filter>
<filter
id="filter22413">
<feGaussianBlur
inkscape:collect="always"
stdDeviation="0.33355481"
id="feGaussianBlur22415" />
</filter>
<linearGradient
x1="168.97348"
y1="226.79765"
x2="94.360741"
y2="42.124634"
id="linearGradient4159"
xlink:href="#linearGradient4153"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="162.12244"
y1="228.19299"
x2="101.21178"
y2="40.729256"
id="linearGradient7076"
xlink:href="#linearGradient7070"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="127.8125"
y1="31.59375"
x2="127.8125"
y2="118.62509"
id="linearGradient10059"
xlink:href="#linearGradient10053"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="150.76942"
y1="72.579872"
x2="162.2932"
y2="131.31783"
id="linearGradient2818"
xlink:href="#linearGradient2820"
gradientUnits="userSpaceOnUse" />
<radialGradient
cx="130.61937"
cy="132.01637"
r="115.95089"
fx="130.61937"
fy="132.01637"
id="radialGradient3721"
xlink:href="#linearGradient2820"
gradientUnits="userSpaceOnUse" />
<linearGradient
x1="127.8125"
y1="31.59375"
x2="127.8125"
y2="118.62509"
id="linearGradient4614"
xlink:href="#linearGradient15536"
gradientUnits="userSpaceOnUse" />
<linearGradient
inkscape:collect="always"
xlink:href="#linearGradient3139"
id="linearGradient3145"
x1="186.39575"
y1="170.82901"
x2="-165.99458"
y2="78.232819"
gradientUnits="userSpaceOnUse" />
<linearGradient
inkscape:collect="always"
xlink:href="#linearGradient3139"
id="linearGradient3147"
gradientUnits="userSpaceOnUse"
x1="184.89622"
y1="178.70155"
x2="-117.25975"
y2="71.859802" />
</defs>
<g
style="opacity:1;fill:url(#linearGradient2818);fill-opacity:1;display:inline"
id="layer2">
<path
d="M 246.57026,132.01637 C 246.57026,196.05428 194.65728,247.96726 130.61937,247.96726 C 66.581461,247.96726 14.668479,196.05428 14.66848,132.01637 C 14.668479,67.978461 66.581461,16.065479 130.61937,16.065479 C 194.65728,16.065479 246.57026,67.978461 246.57026,132.01637 L 246.57026,132.01637 z "
transform="matrix(0.995962,0,0,0.995962,-0.171109,-0.165469)"
style="opacity:1;fill:url(#radialGradient3721);fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path12053" />
</g>
<g
style="opacity:1;display:inline"
id="layer4">
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
style="opacity:1;fill:#d45500;fill-opacity:1;stroke:none;stroke-width:0.10000000000000001;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path2175"
transform="matrix(1.082583,0,0,1.082583,-14.54063,-17.56537)" />
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
style="opacity:1;fill:#ffb380;fill-opacity:1;stroke:none;stroke-width:0.10000000000000001;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path13592"
transform="matrix(1.042113,0,0,1.042113,-9.212006,-12.12368)" />
<path
d="M 230.50477,134.46112 C 230.50477,189.04765 186.25364,233.29877 131.66712,233.29877 C 77.080585,233.29878 32.82946,189.04765 32.82946,134.46112 C 32.82946,79.87459 77.080585,35.623465 131.66711,35.623465 C 186.25364,35.623465 230.50477,79.87459 230.50477,134.46112 L 230.50477,134.46112 z "
style="opacity:1;fill:url(#linearGradient3147);fill-opacity:1;stroke:none;stroke-width:0.10000000000000001;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
id="path14565"
transform="matrix(0.951055,0,0,0.951055,2.777393,0.12014)" />
</g>
<g
style="opacity:1;display:inline"
id="layer5" />
<g
style="opacity:1;display:inline"
id="layer3">
<g
id="g15568">
<path
d="M 123.30585,191.07575 C 129.02124,187.76374 138.24211,176.49437 141.6732,168.65732 C 141.6732,168.65732 138.1399,167.1567 138.1399,167.1567 C 131.0411,179.5796 120.02693,188.70016 123.71066,173.32458 C 123.71066,173.32458 145.45687,105.04964 145.45687,105.04964 C 139.77784,104.57639 111.00423,109.17626 106.51327,110.10356 C 106.51327,110.10356 105.79723,113.88169 105.79723,113.88169 C 115.61397,114.21756 116.94651,118.2341 116.23663,121.5238 C 116.23663,121.5238 99.927949,173.57586 99.927949,173.57586 C 99.927949,173.57586 97.881571,179.46474 97.947071,183.84902 C 98.105167,193.86006 111.11196,197.7125 123.30585,191.07575 z "
style="opacity:1;fill:white;fill-opacity:1;stroke:none;stroke-width:0.1;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;display:inline"
id="path3339" />
<path
d="M 151.9268,75.963402 C 151.93091,83.589999 145.74948,89.774755 138.12288,89.774755 C 130.49628,89.774755 124.31485,83.589999 124.31896,75.963402 C 124.31485,68.336805 130.49628,62.152049 138.12288,62.152049 C 145.74948,62.152049 151.93091,68.336805 151.9268,75.963402 L 151.9268,75.963402 z "
transform="matrix(1.006031,0,0,1.006031,-1.167682,-0.583638)"
style="opacity:1;fill:white;fill-opacity:1;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;display:inline"
id="path7182" />
</g>
</g>
<g
style="opacity:1;fill:url(#linearGradient4614);fill-opacity:1;display:inline"
id="layer7">
<path
d="M 128,31.59375 C 86.871341,31.59375 51.74836,57.401715 37.90625,93.6875 C 56.732365,110.35912 96.245675,118.37197 128.03125,118.625 C 128.07285,118.62467 128.02111,118.62533 128.0625,118.625 C 160.09769,118.36974 199.12268,109.67594 217.71875,92.6875 C 203.61988,56.922098 168.75455,31.593751 128,31.59375 z "
style="opacity:0.45;fill:url(#linearGradient4614);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1"
id="path18502" />
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

7
extensions/wikidata/module/images/Wikidata-logo-en.svg Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" version="1.2" width="1052.36" height="744.09003">
<path d="m 119.4207,543.01679 h 29.16277 V 43.052497 H 119.4207 V 543.01679 z m 60.27256,0 h 89.42926 V 43.052497 H 179.69326 V 543.01679 z M 298.29136,43.052497 V 542.99895 h 89.42926 V 43.052497 h -89.42926 z" style="fill:#990000" />
<path d="m 838.97933,543.0524 h 29.16883 V 43.052437 H 838.97933 V 543.0524 z M 899.25795,43.052437 V 543.0524 h 29.16277 V 43.052437 H 899.25795 z M 417.85387,543.05247 h 29.16277 V 43.052497 H 417.85387 V 543.05247 z M 478.13249,43.052497 V 543.03463 h 29.16884 V 43.052497 h -29.16884 z" style="fill:#339966" />
<path d="m 537.42244,543.05247 h 89.4414 V 43.052497 h -89.4414 V 543.05247 z m 118.5981,0 h 31.10372 V 43.052497 H 656.02054 V 543.05247 z M 716.2931,43.052497 V 543.03463 h 89.42926 V 43.052497 H 716.2931 z" style="fill:#006699" />
<path d="m 883.98859,598.08811 46.01562,103.68359 -27.75781,0 -8.53516,-20.48438 -46.38672,0 -7.71875,20.48438 -27.38672,0 41.78516,-103.68359 29.98438,0 m 2.07812,65.08984 -16.69922,-40.30078 -15.21484,40.30078 31.91406,0 m -60.26562,-65.08984 0,19 -37.55469,0 0,84.68359 -25.82813,0 0,-84.68359 -37.55468,0 0,-19 100.9375,0 m -133.59375,0 46.01562,103.68359 -27.75781,0 -8.53516,-20.48438 -46.38672,0 -7.71875,20.48438 -27.38672,0 41.78516,-103.68359 29.98438,0 m 2.07812,65.08984 -16.69922,-40.30078 -15.21484,40.30078 31.91406,0 m -176.56641,-65.08984 45.71875,0 c 17.66399,10e-5 31.6171,4.84905 41.85938,14.54687 10.29156,9.64852 15.43739,22.142 15.4375,37.48047 -1.1e-4,16.08076 -5.22016,28.72268 -15.66016,37.92578 -10.39071,9.15365 -25.23444,13.73047 -44.53125,13.73047 l -42.82422,0 0,-103.68359 m 25.82813,19 0,65.68359 16.84766,0 c 10.93483,2e-5 19.2968,-2.99347 25.08593,-8.98047 5.78898,-6.03642 8.68351,-13.97782 8.6836,-23.82422 -9e-5,-10.19264 -2.9441,-18.20826 -8.83203,-24.04687 -5.83862,-5.88794 -14.25007,-8.83195 -25.23438,-8.83203 l -16.55078,0 m -49.72656,-19 0,103.68359 -25.82813,0 0,-103.68359 25.82813,0 m -48.91016,0 -33.76953,48.6875 45.79297,54.99609 -32.35938,0 -41.04297,-48.61328 0,48.61328 -25.82812,0 0,-103.68359 25.82812,0 0,46.68359 32.95313,-46.68359 28.42578,0 m -111.10547,0 0,103.68359 -25.82812,0 0,-103.68359 25.82812,0 m -127.21094,37.40625 -32.0625,67.61328 -11.13281,0 -45.57031,-105.01953 27.38672,0 24.41797,57.29687 26.64453,-57.29687 20.78125,0 26.64453,57.29687 24.26953,-57.29687 27.38672,0 -45.42188,105.01953 -11.13281,0 -32.21094,-67.61328" style="fill:#484848;fill-opacity:1;stroke:none;font-family:Gill Sans MT" />
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.6 KiB

BIN
extensions/wikidata/module/images/wikidata.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.5 KiB

233
extensions/wikidata/module/langs/translation-en.json Normal file
View File

@ -0,0 +1,233 @@
{
"wikidata-extension": {
"menu-label": "Wikidata",
"edit-wikidata-schema": "Edit Wikidata schema",
"manage-wikidata-account": "Manage Wikidata account",
"perform-edits-on-wikidata": "Upload edits to Wikidata",
"export-to-qs": "Export to QuickStatements",
"quickstatements-export-name": "QuickStatements"
},
"wikidata-schema": {
"dialog-header": "Align to Wikidata",
"dialog-explanation": "The Wikidata schema below specifies how your tabular data will be transformed into Wikidata edits. You can drag and drop the column names below in most input boxes: for each row, edits will be generated with the values in these columns.",
"preview-explanation": "This tab shows the first edits (out of {nb_edits}) that will be made once you upload the changes to Wikidata. You can use facets to inspect the edits on particular items.",
"schema-tab-header": "Schema",
"warnings-tab-header": "Issues",
"edits-preview-tab-header": "Preview",
"statements-header": "Statements",
"terms-header": "Terms",
"empty-statements": "no statements added",
"empty-terms": "no labels, descriptions or aliases added",
"add-item-button": "add item",
"add-term": "add term",
"remove": "remove",
"add-statement": "add statement",
"add-value": "add value",
"add-qualifier": "add qualifier",
"add-reference": "add reference",
"add-reference-snak": "add",
"property-placeholder": "property",
"nb-references": "&nbsp;references",
"remove-column": "remove column",
"label": "Label",
"description": "Description",
"alias": "Alias",
"item-or-reconciled-column": "type item or drag reconciled column here",
"amount": "amount",
"unit": "unit",
"full-url": "full URL including the protocol",
"tabular-data-with-prefix": "filename starting with \"Data:\"",
"commons-media": "filename",
"math-expression": "mathematical expression",
"geoshape-with-prefix": "filename starting with \"Data:\"",
"datatype-not-supported-yet": "This datatype is not supported yet, sorry.",
"invalid-schema-warning-issues": "Your schema is incomplete, fix it to see the issues.",
"invalid-schema-warning-preview": "Your schema is incomplete, fix it to see the preview.",
"discard-button": "Discard changes",
"save-button": "Save schema",
"close-button": "Close",
"unsaved-changes-alt": "You have made unsaved changes to your Wikidata schema.",
"save-schema-alt": "Save the schema to OpenRefine. The changes will not be uploaded to Wikidata yet.",
"discard-schema-changes-alt": "Discard the changes made to the schema.",
"incomplete-schema-could-not-be-saved": "Your schema is incomplete so it cannot be saved yet.",
"unsaved-warning": "You have made unsaved changes to your Wikidata schema. Close anyway?"
},
"wikidata-preview": {
"new-id": "new item"
},
"wikidata-account": {
"dialog-header": "Wikidata account",
"explain-log-in": "Logging in to <a href=\"https://www.wikidata.org/\" target=\"_blank\">Wikidata</a> lets you to upload edits directly from OpenRefine.",
"username-label": "Username:",
"username-placeholder": "Enter your username",
"password-label": "Password:",
"password-placeholder": "Enter your password",
"remember-credentials-label": "Remember credentials (stored unencrypted in OpenRefine's preferences)",
"close": "Close",
"log-in": "Log in",
"logged-in-as": "You are logged in as:",
"log-out": "Log out",
"connecting-to-wikidata": "Connecting to Wikidata..."
},
"perform-wikidata-edits": {
"dialog-header": "Upload edits to Wikidata",
"review-your-edits": "You are about to upload {nb_edits} edits to Wikidata. Please check them carefully. Large edit batches should be submitted for <a href=\"https://www.wikidata.org/wiki/Wikidata:Requests_for_permissions/Bot\" target=\"_blank\">bot review</a> first.",
"logged-in-as": "You are logged in as",
"edit-summary-label": "Edit summary:",
"edit-summary-placeholder": "a few words to describe your edits",
"perform-edits": "Upload edits",
"cancel": "Cancel",
"analyzing-edits": "Analyzing your edits..."
},
"warnings-messages": {
"new-item-created": {
"title": "This edit batch will create new Wikidata items.",
"body": "Please make sure that these items do not exist yet and are <a href=\"https://www.wikidata.org/wiki/Wikidata:Notability\" target=\"_blank\">suitable for inclusion in Wikidata</a>."
},
"new-item-without-labels-or-aliases": {
"title": "New items created without any label or alias.",
"body": "You should at least provide one label for new items such as {example_entity}, so that others can understand what the item is about."
},
"new-item-without-descriptions": {
"title": "New items created without any description.",
"body": "Adding descriptions on new items such as {example_entity} will make it easier to disambiguate the items from namesakes."
},
"new-item-with-deleted-statements": {
"title": "Deleting statements on new items.",
"body": "There is probably something wrong in your schema or project."
},
"new-item-without-P31-or-P279": {
"title": "New items created without any type.",
"body": "You should provide an \"instance of\" (P31) or \"subclass of\" (P279) statement for each item that you create, such as {example_entity}."
},
"add-statements-with-invalid-format": {
"title": "{property_entity} statements with invalid format.",
"body": "Values for this property are expected to match the regular expression <span class=\"wb-issue-preformat\">{regex}</span>, which is not the case for <span class=\"wb-issue-preformat\">{example_value}</span> added on {example_item_entity}."
},
"remove-statements-with-invalid-format": {
"title": "Removed statements with invalid format.",
"body": "If these statements currently exist on Wikidata, this will solve constraint violations."
},
"missing-inverse-statements": {
"title": "Inverse statements missing for {added_property_entity}.",
"body": "Any {added_property_entity} statement such as the one from {source_entity} to {target_entity} should also be added in reverse with {inverse_property_entity}: in this case, {target_entity} {inverse_property_entity} {source_entity}."
},
"self-referential-statements": {
"title": "Self-referential statements.",
"body": "While not forbidden, self-referential statements are generally suspicious. You have some on {example_entity}."
},
"unsourced-statements": {
"title": "Statements without references.",
"body": "Most statements should have references. You can add them easily in the schema."
},
"property-found-in-mainsnak": {
"title": "{property_entity} used as statement.",
"body": "You are using {property_entity} as a main statement but it is not designed for that."
},
"property-found-in-qualifier": {
"title": "{property_entity} used as qualifier.",
"body": "You are using in {property_entity} as a qualifier but it is not designed for that ."
},
"property-found-in-reference": {
"title": "{property_entity} used as reference.",
"body": "You are using {property_entity} in a reference but it is not designed for that."
},
"missing-mandatory-qualifiers": {
"title": "{statement_property_entity} is missing a {missing_property_entity} qualifier.",
"body": "Statements using {statement_property_entity} such as the one on {example_item_entity} are missing a mandatory {missing_property_entity} qualifier."
},
"disallowed-qualifiers": {
"title": "Qualifier {disallowed_property_entity} is incompatible with {statement_property_entity}.",
"body": "Statements using {statement_property_entity} such as the one on {example_item_entity} should not have a {disallowed_property_entity} qualifier as they are incompatible."
},
"single-valued-property-added-more-than-once": {
"title": "{property_entity} added more than once on the same item.",
"body": "This property is expected to be used at most once on each item but has been added multiple times on the same item, for instance on {example_entity}."
},
"identical-values-for-distinct-valued-property": {
"title": "Identical values for {property_entity}",
"body": "This property should have distinct values, but the same value was found on {item1_entity} and {item2_entity} for instance."
},
"no-edit-generated": {
"title": "No edit was generated.",
"body": "There might be something wrong with your schema."
},
"no-issue-detected": {
"title": "No issue was detected in your edits.",
"body": "Note that OpenRefine cannot detect all the types of problems Wikidata edits can have."
},
"ignored-qualifiers": {
"title": "Some qualifiers were ignored.",
"body": "Qualifier values could not be parsed, so they will not be added to the corresponding statements."
},
"ignored-references": {
"title": "Some references were ignored.",
"body": "None of their statements could be parsed, so no reference was added."
},
"monolingual-text-without-language": {
"title": "No language provided for monolingual text.",
"body": "Some label, description, alias or monolingual text value have been skipped because no language was provided. Example value: <span class=\"wb-issue-preformat\">{example_text}</span>."
},
"leading-whitespace": {
"title": "Leading whitespace in strings.",
"body": "Strings such as <span class=\"wb-issue-preformat\">{example_string}</span> have leading whitespace."
},
"trailing-whitespace": {
"title": "Trailing whitespace in strings.",
"body": "Strings such as <span class=\"wb-issue-preformat\">{example_string}</span> have trailing whitespace."
},
"duplicate-whitespace": {
"title": "Duplicate whitespace in strings.",
"body": "Strings such as <span class=\"wb-issue-preformat\">{example_string}</span> contain duplicate whitespace."
},
"non-printable-characters": {
"title": "Non-printable characters in strings.",
"body": "Strings such as <span class=\"wb-issue-preformat\">{example_string}</span> contain non-printable characters."
},
"invalid-identifier-space": {
"title": "Invalid identifier space for reconciled cells.",
"body": "Some reconciled cells such as <span class=\"wb-issue-preformat\">{example_cell}</span> were ignored because they are not reconciled to Wikidata."
},
"ignored-language": {
"title": "Invalid language identifiers.",
"body": "Some language identifiers are invalid, such as <span class=\"wb-issue-preformat\">{example_value}</span>. See the <a href=\"https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Schema_alignment#Languages\" target=\"_blank\">allowed values</a>."
},
"ignored-date": {
"title": "Invalid date formats.",
"body": "Some dates are incorrectly formatted, such as <span class=\"wb-issue-preformat\">{example_value}</span>. See the <a href=\"https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Schema_alignment#Dates\" target=\"_blank\">allowed formats</a>."
},
"ignored-amount": {
"title": "Invalid amount formats.",
"body": "Some amounts are incorrectly formatted, such as <span class=\"wb-issue-preformat\">{example_value}</span>. See the <a href=\"https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Schema_alignment#Quantities\" target=\"_blank\">allowed formats</a>."
},
"ignored-coordinates": {
"title": "Invalid geographic coordinates.",
"body": "Some coordinates are incorrectly formatted, such as <span class=\"wb-issue-preformat\">{example_value}</span>. See the <a href=\"https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Schema_alignment#Globe_coordinates\" target=\"_blank\">allowed formats</a>."
},
"forbidden-value": {
"title": "Invalid values for {property_entity}",
"body": "Items such as {example_value_entity} added on {example_subject_entity} are not allowed as values for {property_entity}."
},
"bounds-disallowed": {
"title": "Quantity bounds supplied for {property_entity}",
"body": "Values are not expected to have uncertainty bounds, but <span class=\"wb-issue-preformat\">{example_value}</span> added on {example_item_entity} has some. See the <a href=\"https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Schema_alignment#Quantities\" target=\"_blank\">manual</a> to learn how to fix their format."
},
"values-should-be-integers": {
"title": "Non-integer values of {property_entity}",
"body": "Values are expected to be integers, but <span class=\"wb-issue-preformat\">{example_value}</span> added on {example_item_entity} has a fractional part. See the <a href=\"https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Schema_alignment#Quantities\" target=\"_blank\">manual</a> to learn how to fix their format."
},
"invalid-unit": {
"title": "{property_entity} with invalid units",
"body": "Units such as {unit_entity} used on {example_item_entity} are invalid for {property_entity}."
},
"no-unit-provided": {
"title": "Unit missing for {property_entity}",
"body": "Values such as <span class=\"wb-issue-preformat\">{example_value}</span> on {example_item_entity} are expected to have units."
},
"invalid-entity-type": {
"title": "{property_entity} used on items",
"body": "Uses of {property_entity} on items such as {example_entity} are invalid."
}
}
}

204
extensions/wikidata/module/langs/translation-jp.json Normal file
View File

@ -0,0 +1,204 @@
{
"wikidata-extension": {
"menu-label": "Wikidata",
"edit-wikidata-schema": "Wikidataスキーマを編集",
"manage-wikidata-account": "Wikidataアカウントの管理",
"perform-edits-on-wikidata": "Wikidataへアップロード",
"export-to-qs": "クイック・ステートメントへ出力",
"quickstatements-export-name": "クイック・ステートメント"
},
"wikidata-schema": {
"dialog-header": "Align to Wikidata",
"dialog-explanation": "以下のスキーマは、あなたの表データをどのようにWikidataの編集項目に変換するかを指定します。カラム名を入力boxにドラッグできます。各行が一つづつ項目に変換されます。",
"preview-explanation": "このタブは、アップロードされる({nb_edits}のうち)最初の編集項目を表示します。ファセットを使って点検できます。",
"schema-tab-header": "スキーマ",
"warnings-tab-header": "問題点",
"edits-preview-tab-header": "プレビュー",
"statements-header": "ステートメント",
"terms-header": "用語(Term)",
"empty-statements": "ステートメントがありません",
"empty-terms": "ラベルも説明も別名もありません",
"add-item-button": "項目追加",
"add-term": "Term追加",
"remove": "削除",
"add-statement": "ステートメント追加",
"add-value": "値の追加",
"add-qualifier": "修飾子追加",
"add-reference": "参照の追加",
"add-reference-snak": "追加",
"property-placeholder": "属性",
"nb-references": "&nbsp;参照",
"remove-column": "カラムの削除",
"label": "ラベル",
"description": "説明",
"alias": "別名",
"item-or-reconciled-column": "項目を入力するか、上のカラムをドラッグしてください",
"amount": "量",
"unit": "単位",
"full-url": "完全URL",
"tabular-data-with-prefix": "\"Data:\"で始まるファイル名",
"commons-media": "ファイル名",
"math-expression": "数学的表現",
"geoshape-with-prefix": "\"Data:\"で始まるファイル名",
"datatype-not-supported-yet": "このデータ型は未対応です",
"invalid-schema-warning-issues": "スキーマが不完全です。修正してください",
"invalid-schema-warning-preview": "スキーマが不完全です。修正してください",
"discard-button": "変更を破棄",
"save-button": "スキーマを保存",
"close-button": "閉じる",
"unsaved-changes-alt": "スキーマの変更は保存されていません",
"save-schema-alt": "スキーマをローカルのOpenRefineに保存。Wikidataには送信されません。",
"discard-schema-changes-alt": "スキーマの変更を破棄",
"incomplete-schema-could-not-be-saved": "スキーマが不完全なので保存できません",
"unsaved-warning": "スキーマの変更を保存できていませんが、とにかく終了しますか？"
},
"wikidata-preview": {
"new-id": "新しい項目"
},
"wikidata-account": {
"dialog-header": "Wikidataアカウント",
"explain-log-in": "OpenRefineから直接データをアップロードするため、<a href=\"https://www.wikidata.org/\" target=\"_blank\">Wikidata</a>にログインする",
"username-label": "ユーザー名:",
"username-placeholder": "ユーザー名を入力",
"password-label": "パスワード:",
"password-placeholder": "パスワードを入力",
"remember-credentials-label": "credentialsをOpenRefineの設定に保存する",
"close": "閉じる",
"log-in": "ログイン",
"logged-in-as": "ログイン名:",
"log-out": "ログアウト",
"connecting-to-wikidata": "Wikidataに接続中..."
},
"perform-wikidata-edits": {
"dialog-header": "Wikidataに編集をアップロードする",
"review-your-edits": "Wikidataに{nb_edits}項目をアップロードします。大量アップロードは<a href=\"https://www.wikidata.org/wiki/Wikidata:Requests_for_permissions/Bot\" target=\"_blank\">bot review</a>を試みてください。",
"logged-in-as": "ログイン名:",
"edit-summary-label": "編集サマリー:",
"edit-summary-placeholder": "編集項目について",
"perform-edits": "編集項目のアップロード",
"cancel": "中止",
"analyzing-edits": "編集項目を分析中..."
},
"warnings-messages": {
"new-item-created": {
"title": "このバッチ編集は新しい項目を作ります",
"body": "Wikidataに既存項目がないか、また、<a href=\"https://www.wikidata.org/wiki/Wikidata:Notability\" target=\"_blank\">Wikidataにふさわしいか</a>、確認してください"
},
"new-item-without-labels-or-aliases": {
"title": "ラベルも別名もない項目が作られます",
"body": "少なくとも一つのラベル、たとえば{example_entity}が必要です"
},
"new-item-without-descriptions": {
"title": "記述のない項目が作られます",
"body": "記述がないと、{example_entity}項目が明確になります"
},
"new-item-with-deleted-statements": {
"title": "新しい項目のステートメントを削除します",
"body": "おそらくスキーマに問題があります"
},
"new-item-without-P31-or-P279": {
"title": "型(Type)のない項目が作られます",
"body": "(P31)のインスタンスか、(P279)のサブクラスが必要です"
},
"add-statements-with-invalid-format": {
"title": "{property_entity} ステートメントが無効なフォーマットです",
"body": "この項目は、正規表現の<span class=\"wb-issue-preformat\">{regex}</span>に合致する必要があります。対象：{example_item_entity}の<span class=\"wb-issue-preformat\">{example_value}</span>"
},
"remove-statements-with-invalid-format": {
"title": "無効なフォーマットのステートメントが削除されました",
"body": "Wikidataにステートメントが存在する場合、問題が解消します"
},
"missing-inverse-statements": {
"title": "{added_property_entity}の逆ステートメントがありません",
"body": "Any {added_property_entity} statement such as the one from {source_entity} to {target_entity} should also be added in reverse with {inverse_property_entity}: in this case, {target_entity} {inverse_property_entity} {source_entity}."
},
"self-referential-statements": {
"title": "自己参照ステートメントです",
"body": "禁止ではありませんが、{example_entity}の自己参照ステートメントは不自然です"
},
"unsourced-statements": {
"title": "参照のないステートメントです",
"body": "ほとんどのステートメントには参照があります。スキーマで簡単に追加できます"
},
"property-restricted-to-reference-found-in-mainsnak": {
"title": "{property_entity}がステートメントに使われています",
"body": "{property_entity}は参照専用です"
},
"property-restricted-to-reference-found-in-qualifier": {
"title": "{property_entity}が修飾子として使われています",
"body": "修飾子に参照専用の{property_entity}が使われています"
},
"property-restricted-to-qualifier-found-in-mainsnak": {
"title": "{property_entity}がステートメントに使われています",
"body": "ステートメントに修飾子専用の{property_entity}が使われています"
},
"property-restricted-to-qualifier-found-in-reference": {
"title": "{property_entity}が参照に使われています",
"body": "参照に修飾子専用の{property_entity}が使われています"
},
"property-restricted-to-mainsnak-found-in-qualifier": {
"title": "{property_entity}が修飾子に使われています",
"body": "修飾子にステートメント専用の{property_entity}が使われています"
},
"property-restricted-to-mainsnak-found-in-reference": {
"title": "{property_entity}が参照に使われています",
"body": "参照にステートメント専用の{property_entity}が使われています"
},
"missing-mandatory-qualifiers": {
"title": "{statement_property_entity}には{missing_property_entity} 修飾子がありません",
"body": "{example_item_entity}の{statement_property_entity}のようなステートメントには、{missing_property_entity}修飾子が必須です"
},
"disallowed-qualifiers": {
"title": "修飾子 {disallowed_property_entity} は {statement_property_entity} と矛盾します",
"body": "{example_item_entity}の{statement_property_entity} 属性は {disallowed_property_entity} 修飾子を持てません"
},
"single-valued-property-added-more-than-once": {
"title": "{property_entity} が同じ項目に複数あります",
"body": "この属性は各項目に一つだけですが、{example_entity}には複数追加されています"
},
"identical-values-for-distinct-valued-property": {
"title": "{property_entity}と同じ値です",
"body": "この属性はユニークでなければなりませんが、{item1_entity} と {item2_entity} が同じです"
},
"no-edit-generated": {
"title": "編集項目が生成されていません",
"body": "スキーマに問題があるかもしれません"
},
"no-issue-detected": {
"title": "問題点は見当たりません",
"body": "ただし、Wikidataの編集項目の問題点をOpenRefineが網羅できるわけではありません"
},
"ignored-qualifiers": {
"title": "無視された修飾子があります",
"body": "修飾子が解析できませんでした。このためステートメントに追加できません"
},
"ignored-references": {
"title": "無視された参照があります",
"body": "参照の記述が解析できませんでした"
},
"monolingual-text-without-language": {
"title": "言語指定がありません",
"body": "言語指定がないので、ラベル・記述・別名・単一言語テキストが無視されました。例えば: <span class=\"wb-issue-preformat\">{example_text}</span>."
},
"leading-whitespace": {
"title": "文頭に空白文字があります",
"body": "<span class=\"wb-issue-preformat\">{example_string}</span>の文頭に空白文字があります"
},
"trailing-whitespace": {
"title": "文末に空白文字があります",
"body": "<span class=\"wb-issue-preformat\">{example_string}</span>の文末に空白文字があります"
},
"duplicate-whitespace": {
"title": "二重の空白文字があります",
"body": "<span class=\"wb-issue-preformat\">{example_string}</span>には二重の空白文字があります"
},
"non-printable-characters": {
"title": "印刷できない文字が含まれています",
"body": "<span class=\"wb-issue-preformat\">{example_string}</span>には印刷できない文字が含まれています"
},
"invalid-identifier-space": {
"title": "照合したセルの識別子が無効です",
"body": "<span class=\"wb-issue-preformat\">{example_cell}</span>のような照合セルはWikidataにないので無視されます"
}
}
}

22
extensions/wikidata/module/scripts/bettersuggest.js Normal file
View File

@ -0,0 +1,22 @@
/**
* Adds a few tweaks to a suggest widget, mostly to indicate
* the status of the validation.
*/
var fixSuggestInput = function(input) {
input.bind("fb-select", function(evt, data) {
input.addClass('wbs-validated-input');
input.blur();
}).bind("fb-textchange", function(evt, data) {
input.removeClass('wbs-validated-input');
}).blur(function() {
setTimeout(function() {
if(! input.hasClass('wbs-validated-input')) {
input.addClass('wbs-unvalidated-input');
}
}, 100);
}).focus(function() {
input.removeClass('wbs-unvalidated-input');
});
}

49
extensions/wikidata/module/scripts/dialogs/manage-account-dialog.html Normal file
View File

@ -0,0 +1,49 @@
<div class="dialog-frame" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<a href="https://www.wikidata.org/" target="_blank">
<img src="extension/wikidata/images/wikidata.png" class="wikidata-logo" alt="Wikidata logo" />
</a>
<div class="right-of-logo">
<p class="body-text" bind="explainLogIn">
</p>
<div class="wikibase-user-management-area">
<div class="wikibase-user-login" bind="loginArea">
<div bind="invalidCredentials" class="wikibase-invalid-credentials"></div>
<form bind="loginForm" class="wikibase-login-form" method="post">
<table>
<tr>
<td><label for="wb-username" bind="usernameLabel"></label></td>
<td><input name="wb-username" type="text" bind="usernameInput" /></td>
</tr>
<tr>
<td><label for="wb-password" bind="passwordLabel"></label></td>
<td><input name="wb-password" type="password" bind="passwordInput" /></td>
</tr>
<tr>
<td><input type="checkbox" name="remember-credentials" /></td>
<td><label for="remember-credentials" bind="rememberCredentialsLabel"></label></td>
</tr>
</table>
</form>
<div class="wikibase-login-buttons">
<button class="button cancel-button" bind="cancelButton1"></button>
<button class="button button-primary" bind="loginButton"></button>
</div>
</div>
<div class="wikibase-user-logout" bind="logoutArea">
<p><span bind="loggedInAs"></span>
<a bind="loggedInUsername" target="_blank"></a></p>
<form bind="logoutForm" method="post">
<input type="hidden" name="wb-username" value="null" />
<input name="wb-password" type="hidden" value="null" />
</form>
<div class="wikibase-login-buttons">
<button class="button cancel-button" bind="cancelButton2"></button>
<button class="button button-primary" bind="logoutButton"></button>
</div>
</div>
</div>
</div>
</div>
</div>

114
extensions/wikidata/module/scripts/dialogs/manage-account-dialog.js Normal file
View File

@ -0,0 +1,114 @@
var ManageAccountDialog = {};
ManageAccountDialog.firstLogin = true;
ManageAccountDialog.launch = function(logged_in_username, callback) {
$.post(
"command/core/get-all-preferences",
null,
function (preferences) {
ManageAccountDialog.display(logged_in_username, preferences.wikidata_credentials, callback);
},
"json"
);
};
ManageAccountDialog.display = function(logged_in_username, saved_credentials, callback) {
var self = this;
var frame = $(DOM.loadHTML("wikidata", "scripts/dialogs/manage-account-dialog.html"));
var elmts = this._elmts = DOM.bind(frame);
ManageAccountDialog.firstLaunch = false;
this._elmts.dialogHeader.text($.i18n._('wikidata-account')["dialog-header"]);
this._elmts.explainLogIn.html($.i18n._('wikidata-account')["explain-log-in"]);
this._elmts.usernameLabel.text($.i18n._('wikidata-account')["username-label"]);
this._elmts.usernameInput.attr("placeholder", $.i18n._('wikidata-account')["username-placeholder"]);
this._elmts.passwordLabel.text($.i18n._('wikidata-account')["password-label"]);
this._elmts.passwordInput.attr("placeholder", $.i18n._('wikidata-account')["password-placeholder"]);
this._elmts.rememberCredentialsLabel.text($.i18n._('wikidata-account')["remember-credentials-label"]);
this._elmts.dialogHeader.text($.i18n._('wikidata-account')["dialog-header"]);
this._elmts.cancelButton1.text($.i18n._('wikidata-account')["close"]);
this._elmts.cancelButton2.text($.i18n._('wikidata-account')["close"]);
this._elmts.loggedInAs.text($.i18n._('wikidata-account')["logged-in-as"]);
this._elmts.logoutButton.text($.i18n._('wikidata-account')["log-out"]);
this._elmts.loginButton.text($.i18n._('wikidata-account')["log-in"]);
this._level = DialogSystem.showDialog(frame);
var dismiss = function() {
DialogSystem.dismissUntil(self._level - 1);
};
if (logged_in_username != null) {
elmts.loginArea.hide();
} else {
elmts.logoutArea.hide();
}
elmts.loggedInUsername
.text(logged_in_username)
.attr('href', 'https://www.wikidata.org/wiki/User:'+logged_in_username);
frame.find('.cancel-button').click(function() {
dismiss();
callback(null);
});
elmts.loginButton.click(function() {
frame.hide();
$.post(
"command/wikidata/login",
elmts.loginForm.serialize(),
function(data) {
if (data.logged_in) {
dismiss();
callback(data.username);
} else {
frame.show();
elmts.invalidCredentials.text("Invalid credentials.");
}
});
});
elmts.logoutButton.click(function() {
$.post(
"command/wikidata/login",
"logout=true",
function(data) {
if (!data.logged_in) {
dismiss();
callback(null);
}
});
});
};
ManageAccountDialog.isLoggedIn = function(callback) {
var discardWaiter = function() { };
if(ManageAccountDialog.firstLogin) {
discardWaiter = DialogSystem.showBusy($.i18n._('wikidata-account')["connecting-to-wikidata"]);
}
$.get(
"command/wikidata/login",
function(data) {
discardWaiter();
ManageAccountDialog.firstLogin = false;
callback(data.username);
});
};
ManageAccountDialog.ensureLoggedIn = function(callback) {
ManageAccountDialog.isLoggedIn(function(logged_in_username) {
if (logged_in_username == null) {
ManageAccountDialog.launch(null, callback);
} else {
callback(logged_in_username);
}
});
};
ManageAccountDialog.checkAndLaunch = function () {
ManageAccountDialog.isLoggedIn(function(logged_in_username) {
ManageAccountDialog.launch(logged_in_username, function(success) { });
});
};

17
extensions/wikidata/module/scripts/dialogs/perform-edits-dialog.html Normal file
View File

@ -0,0 +1,17 @@
<div class="dialog-frame" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<p class="body-text" bind="reviewYourEdits">
</p>
<div class="perform-edits-warnings-area" bind="warningsArea">
</div>
<div class="wikibase-perform-edits-area">
<p><span bind="loggedInAs"></span> <a bind="loggedInUsername" target="_blank"></a>.</p>
<p><span bind="editSummaryLabel"></span> <input type="text" name="editSummary" bind="editSummary" class="edit-summary" value="" /></p>
<div class="wikibase-login-buttons">
<button class="button cancel-button" bind="cancelButton"></button>
<button class="button button-primary" bind="performEditsButton"></button>
</div>
</div>
</div>
</div>

115
extensions/wikidata/module/scripts/dialogs/perform-edits-dialog.js Normal file
View File

@ -0,0 +1,115 @@
var PerformEditsDialog = {};
PerformEditsDialog.launch = function(logged_in_username, max_severity) {
var self = this;
var elmts = this._elmts;
var frame = this.frame;
if (this.missingSchema) {
return;
}
this._level = DialogSystem.showDialog(frame);
this._elmts.dialogHeader.text($.i18n._('perform-wikidata-edits')["dialog-header"]);
this._elmts.loggedInAs.text($.i18n._('perform-wikidata-edits')["logged-in-as"]);
this._elmts.editSummaryLabel.text($.i18n._('perform-wikidata-edits')["edit-summary-label"]);
this._elmts.editSummary.attr('placeholder', $.i18n._('perform-wikidata-edits')["edit-summary-placeholder"]);
this._elmts.performEditsButton.text($.i18n._('perform-wikidata-edits')["perform-edits"]);
this._elmts.cancelButton.text($.i18n._('perform-wikidata-edits')["cancel"]);
var dismiss = function() {
DialogSystem.dismissUntil(self._level - 1);
};
elmts.loggedInUsername
.text(logged_in_username)
.attr('href','https://www.wikidata.org/wiki/User:'+logged_in_username);
frame.find('.cancel-button').click(function() {
dismiss();
});
if (max_severity === 'CRITICAL') {
elmts.performEditsButton.prop("disabled",true).addClass("button-disabled");
} else {
elmts.performEditsButton.click(function() {
if(elmts.editSummary.val().length == 0) {
elmts.editSummary.focus();
} else {
Refine.postProcess(
"wikidata",
"perform-wikibase-edits",
{},
{
summary: elmts.editSummary.val(),
},
{ includeEngine: true, cellsChanged: true, columnStatsChanged: true },
{ onDone:
function() {
dismiss();
}
});
}
});
}
};
PerformEditsDialog.updateEditCount = function(edit_count) {
this._elmts.reviewYourEdits.html(
$.i18n._('perform-wikidata-edits')["review-your-edits"]
.replace('{nb_edits}', edit_count));
}
PerformEditsDialog._updateWarnings = function(data) {
var warnings = data.warnings;
var mainDiv = this._elmts.warningsArea;
// clear everything
mainDiv.empty();
PerformEditsDialog.updateEditCount(data.edit_count);
var table = $('<table></table>').appendTo(mainDiv);
for (var i = 0; i != warnings.length; i++) {
var rendered = WarningsRenderer._renderWarning(warnings[i]);
rendered.appendTo(table);
}
}
PerformEditsDialog.checkAndLaunch = function () {
var self = this;
this.frame = $(DOM.loadHTML("wikidata", "scripts/dialogs/perform-edits-dialog.html"));
this._elmts = DOM.bind(this.frame);
this.missingSchema = false;
var onSaved = function() {
ManageAccountDialog.ensureLoggedIn(function(logged_in_username) {
if (logged_in_username) {
var discardWaiter = DialogSystem.showBusy($.i18n._('perform-wikidata-edits')["analyzing-edits"]);
$.post(
"command/wikidata/preview-wikibase-schema?" + $.param({ project: theProject.id }),
{ engine: JSON.stringify(ui.browsingEngine.getJSON()) },
function(data) {
discardWaiter();
if(data['code'] != 'error') {
PerformEditsDialog._updateWarnings(data);
PerformEditsDialog.launch(logged_in_username, data['max_severity']);
} else {
SchemaAlignmentDialog.launch(
PerformEditsDialog.checkAndLaunch);
}
},
"json"
);
}
});
};
if (SchemaAlignmentDialog.isSetUp() && SchemaAlignmentDialog._hasUnsavedChanges) {
SchemaAlignmentDialog._save(onSaved);
} else {
onSaved();
}
};

49
extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.html Normal file
View File

@ -0,0 +1,49 @@
<div class="dialog-frame" style="width: 800px;">
<div class="dialog-header" bind="dialogHeader"></div>
<div class="dialog-body" bind="dialogBody">
<div id="schema-alignment-tabs" class="refine-tabs">
<ul>
<li><a href="#schema-alignment-tabs-schema" bind="schemaTabHeader"></a></li>
<li><a href="#schema-alignment-tabs-warnings"><span bind="warningsTabHeader"></span> <span class="schema-alignment-total-warning-count" bind="warningsTabCount"></span></a></li>
<li><a href="#schema-alignment-tabs-preview-edits" bind="editsPreviewTabHeader"></a></li>
</ul>
<div id="schema-alignment-tabs-schema">
<div bind="schemaHeader">
<p class="body-text" bind="dialogExplanation"></p>
<div class="schema-alignment-dialog-columns-area" bind="columnsArea">
</div>
</div>
<div class="schema-alignment-dialog-canvas" bind="canvas">
<div class="schema-alignment-dialog-statements-area" bind="statementsArea">
<div id="schema-alignment-statements-container">
</div>
<div class="wbs-toolbar"><a class="wbs-add-item" bind="addItemButton"></a></div>
</div>
</div>
</div>
<div id="schema-alignment-tabs-warnings" style="display: none;">
<div class="invalid-schema-warning" bind="invalidSchemaWarningIssues"></div>
<div class="schema-alignment-dialog-warnings" bind="warningsArea"></div>
</div>
<div id="schema-alignment-tabs-preview-edits" style="display: none;">
<p class="body-text" bind="previewExplanation"></p>
<div class="invalid-schema-warning" bind="invalidSchemaWarningPreview"></div>
<div class="schema-alignment-dialog-preview"></div>
</div>
</div>
</div>
<div class="dialog-footer" bind="dialogFooter"><div class="grid-layout layout-normal layout-full"><table><tr>
<td>
<button class="button" bind="resetButton"></button>
</td>
<td style="text-align:right;">
<button class="button button-primary" bind="saveButton"></button>
</td>
<td style="text-align:center;" width="30%">
<span class="schema-alignment-status-indicator" bind="statusIndicator"></span>
</td>
<td style="text-align:right;" width="1%">
<button class="button" bind="closeButton"></button>
</td>
</tr></table></div></div>
</div>

1192
extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js Normal file
View File

File diff suppressed because it is too large Load Diff

5
extensions/wikidata/module/scripts/issues-tab.html Normal file
View File

@ -0,0 +1,5 @@
<div id="schema-issues-tab">
<div class="invalid-schema-warning" bind="invalidSchemaWarningIssues"></div>
<div class="schema-issues-area" bind="warningsArea"></div>
</div>

11
extensions/wikidata/module/scripts/jquery.uls.data.js Normal file
View File

File diff suppressed because one or more lines are too long

78
extensions/wikidata/module/scripts/langsuggest.js Normal file
View File

@ -0,0 +1,78 @@
/********************
* LANGUAGE SUGGEST *
********************/
// This list was manually copied from https://www.wikidata.org/w/api.php?action=paraminfo&modules=wbsetlabel on 2017-10-06
// I don't think it is worth making every OpenRefine client perform this query at every startup, because it typically does
// not change very often.
// See https://stackoverflow.com/questions/46507037/how-to-get-all-allowed-languages-for-wikidata/46562061#46562061
WIKIDATA_LANGUAGES = [ "aa", "ab", "ace", "ady", "ady-cyrl", "aeb", "aeb-arab", "aeb-latn", "af", "ak", "aln", "als", "am", "an", "ang", "anp", "ar", "arc", "arn", "arq", "ary", "arz",
"as", "ase", "ast", "atj", "av", "avk", "awa", "ay", "az", "azb", "ba", "ban", "bar", "bat-smg", "bbc", "bbc-latn", "bcc", "bcl", "be", "be-tarask", "be-x-old", "bg", "bgn", "bh", "bho",
"bi", "bjn", "bm", "bn", "bo", "bpy", "bqi", "br", "brh", "bs", "bto", "bug", "bxr", "ca", "cbk-zam", "cdo", "ce", "ceb", "ch", "cho", "chr", "chy", "ckb", "co", "cps", "cr", "crh",
"crh-cyrl", "crh-latn", "cs", "csb", "cu", "cv", "cy", "da", "de", "de-at", "de-ch", "de-formal", "din", "diq", "dsb", "dtp", "dty", "dv", "dz", "ee", "egl", "el", "eml", "en", "en-ca",
"en-gb", "eo", "es", "et", "eu", "ext", "fa", "ff", "fi", "fit", "fiu-vro", "fj", "fo", "fr", "frc", "frp", "frr", "fur", "fy", "ga", "gag", "gan", "gan-hans", "gan-hant", "gd", "gl",
"glk", "gn", "gom", "gom-deva", "gom-latn", "gor", "got", "grc", "gsw", "gu", "gv", "ha", "hak", "haw", "he", "hi", "hif", "hif-latn", "hil", "ho", "hr", "hrx", "hsb", "ht", "hu", "hy",
"hz", "ia", "id", "ie", "ig", "ii", "ik", "ike-cans", "ike-latn", "ilo", "inh", "io", "is", "it", "iu", "ja", "jam", "jbo", "jut", "jv", "ka", "kaa", "kab", "kbd", "kbd-cyrl", "kbp",
"kea", "kg", "khw", "ki", "kiu", "kj", "kk", "kk-arab", "kk-cn", "kk-cyrl", "kk-kz", "kk-latn", "kk-tr", "kl", "km", "kn", "ko", "ko-kp", "koi", "kr", "krc", "kri", "krj", "krl", "ks",
"ks-arab", "ks-deva", "ksh", "ku", "ku-arab", "ku-latn", "kv", "kw", "ky", "la", "lad", "lb", "lbe", "lez", "lfn", "lg", "li", "lij", "liv", "lki", "lmo", "ln", "lo", "loz", "lrc", "lt",
"ltg", "lus", "luz", "lv", "lzh", "lzz", "mai", "map-bms", "mdf", "mg", "mh", "mhr", "mi", "min", "mk", "ml", "mn", "mo", "mr", "mrj", "ms", "mt", "mus", "mwl", "my", "myv", "mzn", "na",
"nah", "nan", "nap", "nb", "nds", "nds-nl", "ne", "new", "ng", "niu", "nl", "nl-informal", "nn", "no", "nod", "nov", "nrm", "nso", "nv", "ny", "nys", "oc", "olo", "om", "or", "os", "ota",
"pa", "pag", "pam", "pap", "pcd", "pdc", "pdt", "pfl", "pi", "pih", "pl", "pms", "pnb", "pnt", "prg", "ps", "pt", "pt-br", "qu", "qug", "rgn", "rif", "rm", "rmy", "rn", "ro", "roa-rup",
"roa-tara", "ru", "rue", "rup", "ruq", "ruq-cyrl", "ruq-latn", "rw", "rwr", "sa", "sah", "sat", "sc", "scn", "sco", "sd", "sdc", "sdh", "se", "sei", "ses", "sg", "sgs", "sh", "shi",
"shi-latn", "shi-tfng", "shn", "si", "simple", "sje", "sk", "skr", "skr-arab", "sl", "sli", "sm", "sma", "smj", "sn", "so", "sq", "sr", "sr-ec", "sr-el", "srn", "srq", "ss", "st", "stq",
"su", "sv", "sw", "szl", "ta", "tay", "tcy", "te", "tet", "tg", "tg-cyrl", "tg-latn", "th", "ti", "tk", "tl", "tly", "tn", "to", "tokipona", "tpi", "tr", "tru", "ts", "tt", "tt-cyrl",
"tt-latn", "tum", "tw", "ty", "tyv", "tzm", "udm", "ug", "ug-arab", "ug-latn", "uk", "ur", "uz", "uz-cyrl", "uz-latn", "ve", "vec", "vep", "vi", "vls", "vmf", "vo", "vot", "vro", "wa",
"war", "wo", "wuu", "xal", "xh", "xmf", "yi", "yo", "yue", "za", "zea", "zh", "zh-classical", "zh-cn", "zh-hans", "zh-hant", "zh-hk", "zh-min-nan", "zh-mo", "zh-my", "zh-sg", "zh-tw",
"zh-yue", "zu" ]
$.suggest("langsuggest", {
_init: function() {
this.api_url = "https://www.wikidata.org/w/api.php";
this._status.SELECT = "Select a language from the list:";
},
request: function(val, cursor) {
var self = this;
var ajax_options = {
url: self.api_url,
data: { action: "languagesearch",
search: val,
format: "json", },
success: function(data) {
self.response(self.convertResults(data));
},
dataType: "jsonp",
};
$.ajax(ajax_options);
},
convertResults: function(data) {
var array = [];
for (var key in data.languagesearch) {
if (data.languagesearch.hasOwnProperty(key) && WIKIDATA_LANGUAGES.indexOf(key) != -1) {
array.push({ id: key, name: key, search_name: data.languagesearch[key] });
}
}
return array;
},
create_item: function(data, response_data) {
var css = this.options.css;
var li = $("<li>").addClass(css.item);
var type = $("<div>").addClass("fbs-item-type").text(data.id);
var native_name = this.get_native_name(data.id);
var full_name = native_name ? native_name : data.search_name;
var label = $("<label>").text(full_name);
li.append($("<div>").addClass(css.item_name).append(type).append(label));
return li;
},
get_native_name: function(lang_code) {
var language = $.uls.data.languages[lang_code];
if (language) {
return language[2];
}
},
});

90
extensions/wikidata/module/scripts/menu-bar-extension.js Normal file
View File

@ -0,0 +1,90 @@
// Load the localization file
var dictionary = {};
$.ajax({
url : "command/core/load-language?",
type : "POST",
async : false,
data : {
module : "wikidata",
// lang : lang
},
success : function(data) {
dictionary = data;
}
});
$.i18n.setDictionary(dictionary);
ExporterManager.MenuItems.push({});
ExporterManager.MenuItems.push(
{
"id" : "exportQuickStatements",
"label": $.i18n._('wikidata-extension')["quickstatements-export-name"],
"click": function() { WikibaseExporterMenuBar.exportTo("quickstatements"); }
}
);
WikibaseExporterMenuBar = {};
WikibaseExporterMenuBar.exportTo = function(format) {
var form = document.createElement("form");
$(form).css("display", "none")
.attr("method", "post")
.attr("action", "command/core/export-rows/statements.txt")
.attr("target", "gridworks-export");
$('<input />')
.attr("name", "engine")
.attr("value", JSON.stringify(ui.browsingEngine.getJSON()))
.appendTo(form);
$('<input />')
.attr("name", "project")
.attr("value", theProject.id)
.appendTo(form);
$('<input />')
.attr("name", "format")
.attr("value", format)
.appendTo(form);
document.body.appendChild(form);
window.open("about:blank", "gridworks-export");
form.submit();
document.body.removeChild(form);
};
//extend the column header menu
$(function(){
ExtensionBar.MenuItems.push(
{
"id":"reconcile",
"label": $.i18n._('wikidata-extension')["menu-label"],
"submenu" : [
{
id: "wikidata/edit-schema",
label: $.i18n._('wikidata-extension')["edit-wikidata-schema"],
click: function() { SchemaAlignmentDialog.launch(false); }
},
{
id:"wikidata/manage-account",
label: $.i18n._('wikidata-extension')["manage-wikidata-account"],
click: function() { ManageAccountDialog.checkAndLaunch(); }
},
{
id:"wikidata/perform-edits",
label: $.i18n._('wikidata-extension')["perform-edits-on-wikidata"],
click: function() { PerformEditsDialog.checkAndLaunch(); }
},
{
id:"wikidata/export-qs",
label: $.i18n._('wikidata-extension')["export-to-qs"],
click: function() { WikibaseExporterMenuBar.exportTo("quickstatements"); }
},
]
}
);
});

6
extensions/wikidata/module/scripts/preview-tab.html Normal file
View File

@ -0,0 +1,6 @@
<div id="schema-preview-tab">
<p class="panel-explanation" bind="previewExplanation"></p>
<div class="invalid-schema-warning" bind="invalidSchemaWarningPreview"></div>
<div class="schema-alignment-dialog-preview"></div>
</div>

256
extensions/wikidata/module/scripts/previewrenderer.js Normal file
View File

@ -0,0 +1,256 @@
/**
* renders an item update (an edit on an item) in HTML.
*/
var EditRenderer = {};
// settings
EditRenderer.maxTerms = 15; // max number of terms displayed
EditRenderer.maxStatements = 25; // max number of statements per statement group
// main method: takes a DOM element and a list
// of edits to render there.
EditRenderer.renderEdits = function(edits, container) {
for(var i = 0; i != edits.length; i++) {
EditRenderer._renderItem(edits[i], container);
}
}
/**************/
/*** ITEMS ****/
/**************/
EditRenderer._renderItem = function(json, container) {
var subject = json;
var statementGroups = null;
var nameDescs = null;
if (json) {
subject = json.subject;
statementGroups = json.statementGroups;
nameDescs = json.nameDescs;
}
var item = $('<div></div>').addClass('wbs-item').appendTo(container);
var inputContainer = $('<div></div>').addClass('wbs-item-input').appendTo(item);
EditRenderer._renderEntity(json.subject, inputContainer);
var right = $('<div></div>').addClass('wbs-item-contents').appendTo(item);
// Terms
if ((json.labels && json.labels.length) ||
(json.descriptions && json.descriptions.length) ||
(json.addedAliases && json.addedAliases.length)) {
var termsContainer = $('<div></div>').addClass('wbs-namedesc-container')
.appendTo(right);
this._renderTermsList(json.labels, "label", termsContainer);
this._renderTermsList(json.descriptions, "description", termsContainer);
this._renderTermsList(json.aliases, "alias", termsContainer);
// Clear the float
$('<div></div>').attr('style', 'clear: right').appendTo(right);
}
// Statements
if (json.addedStatementGroups && json.addedStatementGroups.length) {
// $('<div></div>').addClass('wbs-statements-header')
// .text($.i18n._('wikidata-schema')['statements-header']).appendTo(right);
var statementsGroupContainer = $('<div></div>').addClass('wbs-statement-group-container')
.appendTo(right);
for(var i = 0; i != json.addedStatementGroups.length; i++) {
EditRenderer._renderStatementGroup(json.addedStatementGroups[i], statementsGroupContainer);
}
}
}
/**************************
* NAMES AND DESCRIPTIONS *
**************************/
EditRenderer._renderTermsList = function(termList, termType, termsContainer) {
if(!termList) {
return;
}
for(var i = 0; i != Math.min(termList.length, this.maxTerms); i++) {
EditRenderer._renderTerm(termType, termList[i], termsContainer);
}
if(termList.length > this.maxTerms) {
$('<div></div>').addClass('wbs-namedesc').text('...').appendTo(termsContainer);
}
}
EditRenderer._renderTerm = function(termType, json, container) {
var namedesc = $('<div></div>').addClass('wbs-namedesc').appendTo(container);
var type_container = $('<div></div>').addClass('wbs-namedesc-type').appendTo(namedesc);
var type_span = $('<span></span>').appendTo(type_container)
.text($.i18n._('wikidata-schema')[termType]);
var right = $('<div></div>').addClass('wbs-right').appendTo(namedesc);
var value_container = $('<div></div>').addClass('wbs-namedesc-value').appendTo(namedesc);
EditRenderer._renderValue({datavalue:json,datatype:'monolingualtext'}, value_container);
}
/********************
* STATEMENT GROUPS *
********************/
EditRenderer._renderStatementGroup = function(json, container) {
var statementGroup = $('<div></div>').addClass('wbs-statement-group').appendTo(container);
var inputContainer = $('<div></div>').addClass('wbs-prop-input').appendTo(statementGroup);
var right = $('<div></div>').addClass('wbs-right').appendTo(statementGroup);
EditRenderer._renderEntity(json.property, inputContainer);
var statementContainer = $('<div></div>').addClass('wbs-statement-container').appendTo(right);
for (var i = 0; i != json.statements.length; i++) {
EditRenderer._renderStatement(json.statements[i], statementContainer);
}
if(json.statements.length > EditRenderer.maxStatements) {
$('<div></div>')
.text('...')
.addClass('wbs-statement')
.appendTo(statementContainer);
}
}
/**************
* STATEMENTS *
**************/
EditRenderer._renderStatement = function(json, container) {
var statement = $('<div></div>').addClass('wbs-statement').appendTo(container);
var inputContainer = $('<div></div>').addClass('wbs-target-input').appendTo(statement);
EditRenderer._renderValue(json.mainsnak, inputContainer);
// add rank
var rank = $('<div></div>').addClass('wbs-rank-selector-icon').prependTo(inputContainer);
// add qualifiers...
var right = $('<div></div>').addClass('wbs-right').appendTo(statement);
var qualifierContainer = $('<div></div>').addClass('wbs-qualifier-container').appendTo(right);
if (json.qualifiers) {
for (var pid in json.qualifiers) {
if (json.qualifiers.hasOwnProperty(pid)) {
var qualifiers = json.qualifiers[pid];
for (var i = 0; i != qualifiers.length; i++) {
EditRenderer._renderSnak(qualifiers[i], qualifierContainer);
}
}
}
}
// and references
$('<div></div>').attr('style', 'clear: right').appendTo(statement);
var referencesToggleContainer = $('<div></div>').addClass('wbs-references-toggle').appendTo(statement);
var triangle = $('<div></div>').addClass('triangle-icon').addClass('pointing-right').appendTo(referencesToggleContainer);
var referencesToggle = $('<a></a>').appendTo(referencesToggleContainer);
right = $('<div></div>').addClass('wbs-right').appendTo(statement);
var referenceContainer = $('<div></div>').addClass('wbs-reference-container').appendTo(right);
referencesToggle.click(function () {
triangle.toggleClass('pointing-down');
triangle.toggleClass('pointing-right');
referenceContainer.toggle(100);
});
referenceContainer.hide();
if (json.references) {
for (var i = 0; i != json.references.length; i++) {
EditRenderer._renderReference(json.references[i], referenceContainer);
}
}
EditRenderer._updateReferencesNumber(referenceContainer);
}
/*********************************
* QUALIFIER AND REFERENCE SNAKS *
*********************************/
EditRenderer._renderSnak = function(json, container) {
var qualifier = $('<div></div>').addClass('wbs-qualifier').appendTo(container);
var toolbar1 = $('<div></div>').addClass('wbs-toolbar').appendTo(qualifier);
var inputContainer = $('<div></div>').addClass('wbs-prop-input').appendTo(qualifier);
var right = $('<div></div>').addClass('wbs-right').appendTo(qualifier);
var statementContainer = $('<div></div>').addClass('wbs-statement-container').appendTo(right);
EditRenderer._renderEntity(json.full_property, inputContainer);
EditRenderer._renderValue(json, statementContainer);
}
/**************
* REFERENCES *
**************/
EditRenderer._renderReference = function(json, container) {
var reference = $('<div></div>').addClass('wbs-reference').appendTo(container);
var referenceHeader = $('<div></div>').addClass('wbs-reference-header').appendTo(reference);
var right = $('<div></div>').addClass('wbs-right').appendTo(reference);
var qualifierContainer = $('<div></div>').addClass('wbs-qualifier-container').appendTo(right);
for (var pid in json.snaks) {
if (json.snaks.hasOwnProperty(pid)) {
var snaks = json.snaks[pid];
for(var i = 0; i != snaks.length; i++) {
EditRenderer._renderSnak(snaks[i], qualifierContainer);
}
}
}
}
EditRenderer._updateReferencesNumber = function(container) {
var childrenCount = container.children().length;
var statement = container.parents('.wbs-statement');
var a = statement.find('.wbs-references-toggle a').first();
a.html(childrenCount+$.i18n._('wikidata-schema')["nb-references"]);
}
/*******************
* VALUE RENDERING *
*******************/
EditRenderer.renderedValueCache = {};
EditRenderer._renderEntity = function(json, container) {
var html = WarningsRenderer._renderEntity(json);
$(html).appendTo(container);
}
EditRenderer._renderValue = function(json, container) {
var input = $('<span></span>').appendTo(container);
var mode = json.datatype;
if (mode === "wikibase-item" || mode === "wikibase-property") {
EditRenderer._renderEntity(json.datavalue, container);
} else {
var jsonValue = JSON.stringify(json.datavalue);
if (jsonValue in EditRenderer.renderedValueCache) {
$('<span>'+EditRenderer.renderedValueCache[jsonValue]+'</span>').appendTo(container);
} else {
var params = {
action: 'wbformatvalue',
generate: 'text/html',
datavalue: jsonValue,
options: '{"lang":"'+$.i18n._('core-recon')["wd-recon-lang"]+'"}',
format: 'json'
};
if ('property' in json) {
params.property = json.property;
} else {
params.datatype = json.datatype;
}
$.get(
'https://www.wikidata.org/w/api.php',
params,
function (data) {
if('result' in data) {
EditRenderer.renderedValueCache[jsonValue] = data.result;
$('<span>'+data.result+'</span>').appendTo(container);
}
},
'jsonp'
);
}
}
}

17
extensions/wikidata/module/scripts/schema-alignment-tab.html Normal file
View File

@ -0,0 +1,17 @@
<div id="schema-alignment-tab">
<div bind="schemaHeader">
<div class="schema-alignment-save"><button class="button button-primary" bind="saveButton"></button><button class="button button-primary" bind="discardButton"></button></div>
<p class="panel-explanation" bind="dialogExplanation"></p>
<div style="clear: right"></div>
<div class="schema-alignment-dialog-columns-area" bind="columnsArea">
</div>
</div>
<div class="schema-alignment-dialog-canvas" bind="canvas">
<div class="schema-alignment-dialog-statements-area" bind="statementsArea">
<div id="schema-alignment-statements-container">
</div>
<div class="wbs-toolbar"><a class="wbs-add-item" bind="addItemButton"></a></div>
</div>
</div>
</div>

79
extensions/wikidata/module/scripts/warningsrenderer.js Normal file
View File

@ -0,0 +1,79 @@
var WarningsRenderer = {};
// renders a Wikibase entity into a link
WarningsRenderer._renderEntity = function(entity) {
if (!entity.id && entity.value) {
entity.id = entity.value.id;
}
var id = entity.id;
var is_new = entity.siteIri == "http://localhost/entity/";
if (is_new) {
id = $.i18n._('wikidata-preview')['new-id'];
}
var fullLabel = id;
if (entity.label) {
fullLabel = entity.label + ' (' + id + ')';
}
var url = entity.iri;
if (!url && entity.value) {
url = 'http://www.wikidata.org/entity/'+entity.value.id;
}
if (is_new) {
return '<span class="wb-preview-new-entity">'+fullLabel+'</span>';
} else {
return '<a href="'+url+'" class="wb-preview-entity" target="_blank">'+fullLabel+'</a>';
}
}
// replaces the issue properties in localization template
WarningsRenderer._replaceIssueProperties = function(template, properties) {
if (!properties) {
return template;
}
var expanded = template;
for (var key in properties) {
if (properties.hasOwnProperty(key)) {
var rendered = properties[key];
if (key.endsWith('_entity')) {
rendered = WarningsRenderer._renderEntity(properties[key]);
}
expanded = expanded.replace(new RegExp('{'+key+'}', 'g'), rendered);
}
}
return expanded;
}
WarningsRenderer._renderWarning = function(warning) {
var localized = $.i18n._('warnings-messages')[warning.type];
var title = warning.type;
var body = "";
if (localized) {
title = WarningsRenderer._replaceIssueProperties(localized.title, warning.properties);
body = WarningsRenderer._replaceIssueProperties(localized.body, warning.properties);
}
var tr = $('<tr></tr>').addClass('wb-warning');
var severityTd = $('<td></td>')
.addClass('wb-warning-severity')
.addClass('wb-warning-severity-'+warning.severity)
.appendTo(tr);
var bodyTd = $('<td></td>')
.addClass('wb-warning-body')
.appendTo(tr);
var h1 = $('<h1></h1>')
.html(title)
.appendTo(bodyTd);
var p = $('<p></p>')
.html(body)
.appendTo(bodyTd);
var countTd = $('<td></td>')
.addClass('wb-warning-count')
.appendTo(tr);
var countSpan = $('<span></span>')
.text(warning.count)
.appendTo(countTd);
return tr;
}

64
extensions/wikidata/module/styles/dialogs/manage-account-dialog.less Normal file
View File

@ -0,0 +1,64 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@import-less url("../theme.less");
.wikibase-login-form {
text-align: center;
}
.wikibase-login-buttons {
text-align: right;
}
.wikibase-invalid-credentials {
color: red;
}
.wikibase-user-login tr td:first-child {
text-align: right;
}
.wikibase-user-login tr td:last-child {
padding-left: 10px;
text-align: left;
}
.wikidata-logo {
float: left;
margin: -10px;
}
.right-of-logo {
margin-left: 110px;
}

39
extensions/wikidata/module/styles/dialogs/perform-edits.less Normal file
View File

@ -0,0 +1,39 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@import-less url("../theme.less");
.edit-summary {
width: 300px;
}

516
extensions/wikidata/module/styles/dialogs/schema-alignment-dialog.css Normal file
View File

@ -0,0 +1,516 @@
@import-less url("../theme.less");
.main-view-panel-tab-header#summary-bar {
font-size: 1.3em;
font-weight: normal;
position: initial;
}
#wikidata-schema-panel, #wikidata-issues-panel, #wikidata-preview-panel {
display: block;
overflow: hidden;
height: 100%;
background-color: white;
font-size: 1.2em;
}
#schema-preview-tab, #schema-issues-tab, #schema-alignment-tab {
overflow: auto;
height: 100%;
}
#schema-alignment-tab {
overflow: hidden;
}
.main-view-panel-tab-header {
margin-top: 9px;
margin-left: 7px;
font-size: 1.3em;
display: inline-block;
background-color: transparent;
color: #11c;
padding: 3px 7px;
cursor: pointer;
}
.main-view-panel-tab-header.active {
background-color: white;
color: black;
border: 1px solid #818fb7;
border-bottom: none;
}
.main-view-panel-tab-header.active:first-child {
background-color: #f2f2f2;
}
.schema-alignment-dialog-canvas {
background: white;
overflow-y: auto;
}
.schema-alignment-dialog-statements-area {
padding: 10px;
max-width: 900px;
overflow-y: auto;
}
.panel-explanation {
margin: 1em;
}
.schema-alignment-save {
float: right;
padding: 1em;
}
.schema-alignment-save button:disabled {
color: gray;
}
.schema-alignment-save button {
font-weight: normal;
}
.schema-alignment-dialog-columns-area {
border: 1px solid #bcf;
border-left: 0;
border-right: 0;
padding: 5px;
max-height: 100px;
overflow-y: auto;
}
.wbs-draggable-column {
border: 1px solid #aaa;
padding: 2px;
margin: 2px;
background-color: #eee;
display: inline-block;
padding-bottom: 3px;
max-width: 95%;
}
.ui-droppable .wbs-draggable-column {
margin: 0px;
padding: 0px;
padding-left: 2px;
}
.wbs-reconciled-column {
border-bottom: 3px solid #282;
padding-bottom: 1px;
}
.wbs-restricted-column-name {
max-width: -moz-calc(100% - 20px);
max-width: -webkit-calc(100% - 20px);
max-width: -o-calc(100% - 20px);
max-width: calc(100% - 20px);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
display: inline-block;
vertical-align: middle;
white-space: nowrap;
}
.wbs-draggable-column .wbs-remove {
float: right;
}
.wbs-accepting-input {
box-shadow: inset 0 0 10px #a7cdff;
}
.wbs-validated-input {
border: 1px solid green !important;
}
.wbs-unvalidated-input {
border: 1px solid red !important;
background-color: #ffbfbf;
}
.wbs-icon {
background-position: center;
background-size: 20px 20px;
width: 20px;
height: 20px;
display: inline-block;
vertical-align: middle;
opacity: 0.7;
}
.wbs-remove .wbs-icon {
background-image:linear-gradient(transparent,transparent),url("data:image/svg+xml,%3Csvg%20xmlns=%22http://www.w3.org/2000/svg%22%20width=%2224%22%20height=%2224%22%20viewBox=%220%200%2024%2024%22%3E%20%3Cg%20id=%22remove%22%3E%20%3Cpath%20id=%22trash-can%22%20d=%22M12%2010h-1v6h1v-6zm-2%200H9v6h1v-6zm4%200h-1v6h1v-6zm0-4V5H9v1H6v3h1v7.966l1%201.03v-.073V18h6.984l.016-.018v.015l1-1.03V9h1V6h-3zm1%2011H8V9h7v8zm1-9H7V7h9v1z%22/%3E%20%3C/g%3E%20%3C/svg%3E");
}
.wbs-remove span:last-child:hover {
text-decoration: underline;
}
.wbs-rank-selector-icon {
background-position: -36px;
width: 8px;
flex: 0 0 8px;
height: 20px;
margin-right: 2px;
background-image:linear-gradient(transparent,transparent),url("data:image/svg+xml,%3Csvg%20xmlns=%22http://www.w3.org/2000/svg%22%20xmlns:xlink=%22http://www.w3.org/1999/xlink%22%20width=%22107%22%20height=%2220%22%3E%20%3Cdefs%3E%20%3Cpath%20d=%22M3.1%2C0%200%2C3.8%200%2C6%208%2C6%208%2C3.8%204.9%2C0zm8.2%2C7%20-2.3%2C2%200%2C2%202.3%2C2%203.4%2C0%202.3%2C-2%200%2C-2%20-2.3%2C-2zm6.7%2C7%200%2C2.2%203.1%2C3.8%201.8%2C0%203.1%2C-3.8%200%2C-2.2z%22%20id=%22a%22/%3E%20%3Cpath%20d=%22m18.5%2C10.75%200%2C-1.5%202%2C-1.75%203%2C0%202%2C1.75%200%2C1.5%20-2%2C1.75%20-3%2C0zm0%2C-6.75%200%2C1.5%207%2C0%200%2C-1.5%20-2.875%2C-3.5%20-1.25%2C0zm-9%2C12%200%2C-1.5%207%2C0%200%2C1.5%20-2.875%2C3.5%20-1.25%2C0zm0%2C-12%200%2C1.5%207%2C0%200%2C-1.5%20-2.875%2C-3.5%20-1.25%2C0zm-9%2C12%200%2C-1.5%207%2C0%200%2C1.5%20-2.875%2C3.5%20-1.25%2C0zm0%2C-5.25%200%2C-1.5%202%2C-1.75%203%2C0%202%2C1.75%200%2C1.5%20-2%2C1.75%20-3%2C0z%22%20id=%22b%22%20fill=%22none%22/%3E%20%3C/defs%3E%20%3Cuse%20fill=%22%23000%22%20x=%220%22%20y=%220%22%20xlink:href=%22%23a%22/%3E%20%3Cuse%20stroke=%22%23000%22%20x=%220%22%20y=%220%22%20xlink:href=%22%23b%22/%3E%20%3Cuse%20fill=%22%2372777d%22%20x=%2227%22%20y=%220%22%20xlink:href=%22%23a%22/%3E%20%3Cuse%20stroke=%22%2372777d%22%20x=%2227%22%20y=%220%22%20xlink:href=%22%23b%22/%3E%20%3Cuse%20fill=%22%2336c%22%20x=%2254%22%20y=%220%22%20xlink:href=%22%23a%22/%3E%20%3Cuse%20stroke=%22%2336c%22%20x=%2254%22%20y=%220%22%20xlink:href=%22%23b%22/%3E%20%3Cuse%20fill=%22%23447ff5%22%20x=%2281%22%20y=%220%22%20xlink:href=%22%23a%22/%3E%20%3Cuse%20stroke=%22%23447ff5%22%20x=%2281%22%20y=%220%22%20xlink:href=%22%23b%22/%3E%20%3C/svg%3E");
}
.wbs-item-input, .wbs-prop-input, .wbs-target-input {
width: 250px;
min-height: 20px;
display: inline-block;
margin: 5px;
white-space: normal;
word-wrap: break-word;
}
.wbs-target-input {
display: inline-flex;
}
.wbs-prop-input {
width: 120px;
}
.wbs-qualifier .wbs-prop-input {
width: 85px;
}
.wbs-qualifier .wbs-target-input {
width: 120px;
}
/* Fix input rendering for Firefox */
.wbs-item-input input, .wbs-prop-input input, .wbs-target-input input,
.wbs-monolingual-container input {
width: 100%;
border: solid #b5b5b5 1px;
padding: 2px;
}
.wbs-draggable-column img {
float: right;
}
.wbs-toolbar {
float: right;
width: 100px;
padding: 2px;
}
.wbs-toolbar a {
cursor: pointer;
}
.wbs-remove,
.wbs-add-item, .wbs-add-statement-group, .wbs-add-statement,
.wbs-add-qualifier, .wbs-add-reference, .wbs-add-namedesc {
color: #0645ad !important;
font-size: 0.9em;
cursor: pointer;
}
.wbs-add-item:hover, .wbs-add-statement-group:hover, .wbs-add-statement:hover,
.wbs-add-qualifier:hover, .wbs-add-reference:hover, .wbs-add-namedesc:hover {
text-decoration: none;
}
.wbs-add-item span:hover, .wbs-add-statement-group span:hover, .wbs-add-statement span:hover,
.wbs-add-qualifier span:hover, .wbs-add-reference span:hover, .wbs-add-namedesc span:hover {
text-decoration: underline;
}
.wbs-remove {
display: inline-block;
}
.wbs-item-contents {
padding-left: 10px;
margin-left: 20px;
border-left: 3px solid gray;
}
.wbs-add-item b, .wbs-add-statement-group b, .wbs-add-statement b,
.wbs-add-qualifier b, .wbs-add-reference b, .wbs-add-namedesc b {
font-size: 1em;
color: grey;
width: 20px;
height: 20px;
}
.wbs-statement-group-container:empty:before {
content: attr(data-emptyplaceholder);
color: gray;
}
.schema-alignment-dialog-preview .wbs-qualifier .wbs-statement-container {
display: table-cell;
vertical-align: bottom;
padding-left: 10px;
height: 20px;
}
.wbs-statement-group-container, .wbs-statement-container, .wbs-qualifier-container, .wbs-reference-container {
width: 100%;
display: block;
overflow: auto;
}
.wbs-item, .wbs-statement-group, .wbs-statement, .wbs-qualifier, .wbs-reference {
display: block;
overflow: auto;
}
.wbs-item {
margin-bottom: 5px;
padding-right: 5px;
}
.wbs-statement-group {
background: #eaecf0;
margin-bottom: 5px;
border: 1px solid #c8ccd1;
}
.wbs-statement {
background: white;
border-bottom: 1px solid #eaecf0;
}
.wbs-statement:last-child {
border-bottom: 0;
}
.wbs-right {
float: right;
width: 85%;
}
.wbs-statement-group .wbs-right {
width: 75%;
}
.wbs-statement .wbs-right {
width: 90%;
}
.wbs-qualifier, .wbs-reference {
position: relative;
overflow-x: hidden;
}
.wbs-qualifier .wbs-right {
width: auto;
position: absolute;
top: 0px;
margin-left: 100px;
}
.wbs-reference > .wbs-right {
width: 100%;
}
.wbs-references-toggle {
width: 12em;
margin: 5px;
}
.wbs-references-toggle a {
color: #0645ad !important;
}
.wbs-references-toggle .triangle-icon {
background: transparent url(data:image/svg+xml,%3C%3Fxml%20version%3D%221.0%22%20encoding%3D%22UTF-8%22%20standalone%3D%22no%22%3F%3E%0A%3Csvg%0A%20%20%20xmlns%3Adc%3D%22http%3A//purl.org/dc/elements/1.1/%22%0A%20%20%20xmlns%3Acc%3D%22http%3A//creativecommons.org/ns%23%22%0A%20%20%20xmlns%3Ardf%3D%22http%3A//www.w3.org/1999/02/22-rdf-syntax-ns%23%22%0A%20%20%20xmlns%3Asvg%3D%22http%3A//www.w3.org/2000/svg%22%0A%20%20%20xmlns%3D%22http%3A//www.w3.org/2000/svg%22%0A%20%20%20id%3D%22svg8%22%0A%20%20%20version%3D%221.1%22%0A%20%20%20viewBox%3D%220%200%202.6458332%202.6458332%22%0A%20%20%20height%3D%2210%22%0A%20%20%20width%3D%2210%22%3E%0A%20%20%3Cdefs%0A%20%20%20%20%20id%3D%22defs2%22%20/%3E%0A%20%20%3Cmetadata%0A%20%20%20%20%20id%3D%22metadata5%22%3E%0A%20%20%20%20%3Crdf%3ARDF%3E%0A%20%20%20%20%20%20%3Ccc%3AWork%0A%20%20%20%20%20%20%20%20%20rdf%3Aabout%3D%22%22%3E%0A%20%20%20%20%20%20%20%20%3Cdc%3Aformat%3Eimage/svg+xml%3C/dc%3Aformat%3E%0A%20%20%20%20%20%20%20%20%3Cdc%3Atype%0A%20%20%20%20%20%20%20%20%20%20%20rdf%3Aresource%3D%22http%3A//purl.org/dc/dcmitype/StillImage%22%20/%3E%0A%20%20%20%20%20%20%20%20%3Cdc%3Atitle%3E%3C/dc%3Atitle%3E%0A%20%20%20%20%20%20%3C/cc%3AWork%3E%0A%20%20%20%20%3C/rdf%3ARDF%3E%0A%20%20%3C/metadata%3E%0A%20%20%3Cg%0A%20%20%20%20%20transform%3D%22translate%280%2C-294.35416%29%22%0A%20%20%20%20%20id%3D%22layer1%22%3E%0A%20%20%20%20%3Cpath%0A%20%20%20%20%20%20%20id%3D%22path12%22%0A%20%20%20%20%20%20%20d%3D%22m%200.21601809%2C294.5035%202.28763941%2C1.14744%20-2.29850705%2C1.20871%20z%22%0A%20%20%20%20%20%20%20style%3D%22fill%3A%23645eea%3Bfill-opacity%3A1%3Bfill-rule%3Aevenodd%3Bstroke%3A%23645eea%3Bstroke-width%3A0.06161711px%3Bstroke-linecap%3Abutt%3Bstroke-linejoin%3Amiter%3Bstroke-opacity%3A1%22%20/%3E%0A%20%20%3C/g%3E%0A%3C/svg%3E%0A) no-repeat center center;
background-size: 100% auto;
height: 8px;
width: 8px;
margin-right: 4px;
display: inline-block;
}
.pointing-right {
transform: rotate(0deg);
transition: .3s cubic-bezier(.17,.67,.21,1.69);
}
.pointing-down {
transform: rotate(90deg);
transition: .3s cubic-bezier(.17,.67,.21,1.69);
}
.wbs-qualifier .wbs-statement {
overflow: hidden;
background: transparent;
}
.wbs-monolingual-container, .wbs-quantity-container {
display: inline-block;
}
.wbs-reference-header {
height: 22px;
overflow: hidden;
display: block;
background-color: #b6c8ec;
}
.wbs-reference {
background-color: #eaf3ff;
margin-bottom: 5px;
}
.wbs-reference .wbs-statement {
background-color: #eaf3ff;
}
.wbs-namedesc-header, .wbs-statements-header {
font-size: 1.2em;
}
.wbs-namedesc {
border-left: 1px solid #c8ccd1;
border-right: 1px solid #c8ccd1;
}
.wbs-namedesc:first-child {
border-top: 1px solid #c8ccd1;
}
.wbs-namedesc:last-child {
border-bottom: 1px solid #c8ccd1;
}
.wbs-namedesc-container:empty:before {
content: attr(data-emptyplaceholder);
color: gray;
padding: 5px;
border: none;
}
.wbs-namedesc {
background-color: #eaecf0;
padding: 5px;
}
.wbs-namedesc-type, .wbs-namedesc-value {
display: inline-block;
}
.wbs-namedesc-value {
padding-left: 20px;
width: 300px;
}
.wbs-language-input {
width: 100%;
}
.schema-alignment-columns-header {
margin-bottom: 0.3em;
}
/*** Warnings rendering ****/
#wikidata-issues-panel table {
width: 100%;
}
.wb-warning h1 {
font-size: 1.2em;
}
.schema-issues-area table,
.perform-edits-warnings-area table {
width: 100%;
}
tr.wb-warning:nth-of-type(odd) {
background-color: #f2f2f2;
}
.wb-warning-count span,
.schema-alignment-total-warning-count {
color: white;
background-color: #777;
padding: 0px 5px;
border-radius: 0.5em;
}
.wb-warning-severity {
width: 60px;
height: 60px;
background-repeat: no-repeat;
background-position: center;
}
.wb-warning-severity-INFO {
background-image: url('../../images/Information.png');
}
.wb-warning-severity-WARNING {
background-image: url('../../images/Warning.png');
}
.wb-warning-severity-IMPORTANT {
background-image: url('../../images/Important.png');
}
.wb-warning-severity-CRITICAL {
background-image: url('../../images/Critical.png');
}
.wb-warning-body {
padding: 5px;
vertical-align: top;
}
.wb-warning-count {
padding: 5px;
vertical-align: middle;
text-align: center;
}
.wb-warning-end {
float: clear;
}
div.perform-edits-warnings-area {
min-height: 340px;
max-height: 400px;
overflow-x: hidden;
overflow-y: auto;
border: 1px solid #bbb;
}
.wb-issue-preformat {
border: 1px solid #eaecf0;
background-color: #f8f9fa;
padding: 1px 3px;
border-radius: 2px;
font-family: monospace;
}
.wb-preview-new-entity {
color: #11c;
}
/*** QuickStatements Preview ***/
div.schema-alignment-dialog-preview {
min-height: 340px;
max-width: 900px;
overflow: auto;
background: white;
padding: 10px;
margin-top: 3px;
white-space: pre;
font-size: 9pt;
}

32
extensions/wikidata/src/org/openrefine/wikidata/commands/CommandUtilities.java Normal file
View File

@ -0,0 +1,32 @@
package org.openrefine.wikidata.commands;
import java.io.IOException;
import java.io.Writer;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONWriter;
public class CommandUtilities {
/**
* Helper introduced to ease returning error messages from a response. Curiously
* this is not part of {@link Command}: the respond method uses the "status" JSON
* key instead of the "code" one required by the JS code.
*
* @param response
* @param errorMessage
* @throws IOException
*/
public static void respondError(HttpServletResponse response, String errorMessage)
throws IOException {
Writer w = response.getWriter();
JSONWriter writer = new JSONWriter(w);
writer.object();
writer.key("code"); writer.value("error");
writer.key("message"); writer.value(errorMessage);
writer.endObject();
w.flush();
w.close();
}
}

77
extensions/wikidata/src/org/openrefine/wikidata/commands/LoginCommand.java Normal file
View File

@ -0,0 +1,77 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.IOException;
import java.io.StringWriter;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONWriter;
import org.openrefine.wikidata.editing.ConnectionManager;
import com.google.refine.commands.Command;
public class LoginCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String username = request.getParameter("wb-username");
String password = request.getParameter("wb-password");
String remember = request.getParameter("remember-credentials");
ConnectionManager manager = ConnectionManager.getInstance();
if (username != null && password != null) {
manager.login(username, password, "on".equals(remember));
} else if ("true".equals(request.getParameter("logout"))) {
manager.logout();
}
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
StringWriter sb = new StringWriter(2048);
JSONWriter writer = new JSONWriter(sb);
try {
writer.object();
writer.key("logged_in");
writer.value(manager.isLoggedIn());
writer.key("username");
writer.value(manager.getUsername());
writer.endObject();
} catch (JSONException e) {
logger.error(e.getMessage());
}
respond(response, sb.toString());
}
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
doPost(request, response);
}
}

44
extensions/wikidata/src/org/openrefine/wikidata/commands/PerformWikibaseEditsCommand.java Normal file
View File

@ -0,0 +1,44 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONObject;
import org.openrefine.wikidata.operations.PerformWikibaseEditsOperation;
import com.google.refine.commands.EngineDependentCommand;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
public class PerformWikibaseEditsCommand extends EngineDependentCommand {
@Override
protected AbstractOperation createOperation(Project project, HttpServletRequest request, JSONObject engineConfig)
throws Exception {
String summary = request.getParameter("summary");
return new PerformWikibaseEditsOperation(engineConfig, summary);
}
}

140
extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java Normal file
View File

@ -0,0 +1,140 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.IOException;
import java.io.StringWriter;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.qa.EditInspector;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import static org.openrefine.wikidata.commands.CommandUtilities.respondError;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.browsing.Engine;
import com.google.refine.commands.Command;
import com.google.refine.model.Project;
public class PreviewWikibaseSchemaCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String jsonString = request.getParameter("schema");
WikibaseSchema schema = null;
if (jsonString != null) {
try {
schema = WikibaseSchema.reconstruct(jsonString);
} catch (JSONException e) {
respondError(response, "Wikibase schema could not be parsed.");
return;
}
} else {
schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema");
}
if (schema == null) {
respondError(response, "No Wikibase schema provided.");
return;
}
QAWarningStore warningStore = new QAWarningStore();
// Evaluate project
Engine engine = getEngine(request, project);
List<ItemUpdate> editBatch = schema.evaluate(project, engine, warningStore);
StringWriter sb = new StringWriter(2048);
JSONWriter writer = new JSONWriter(sb);
writer.object();
{
// Inspect the edits and generate warnings
EditInspector inspector = new EditInspector(warningStore);
inspector.inspect(editBatch);
writer.key("warnings");
writer.array();
for (QAWarning warning : warningStore.getWarnings()) {
warning.write(writer, new Properties());
}
writer.endArray();
// Add max warning level
writer.key("max_severity");
writer.value(warningStore.getMaxSeverity().toString());
// this is not the length of the warnings array written before,
// but the total number of issues raised (before deduplication)
writer.key("nb_warnings");
writer.value(warningStore.getNbWarnings());
// Dump the first 10 edits, scheduled with the default scheduler
WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler();
List<ItemUpdate> nonNullEdits = scheduler.schedule(editBatch).stream()
.filter(e -> !e.isNull())
.collect(Collectors.toList());
writer.key("edit_count");
writer.value(nonNullEdits.size());
List<ItemUpdate> firstEdits = nonNullEdits.stream()
.limit(10)
.collect(Collectors.toList());
ObjectMapper mapper = new ObjectMapper();
String firstEditsJson = mapper.writeValueAsString(firstEdits);
writer.key("edits_preview");
writer.value(new JSONArray(firstEditsJson));
}
writer.endObject();
respond(response, sb.toString());
} catch (Exception e) {
respondException(response, e);
}
}
}

78
extensions/wikidata/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommand.java Normal file
View File

@ -0,0 +1,78 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.commands;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONException;
import org.json.JSONObject;
import org.openrefine.wikidata.operations.SaveWikibaseSchemaOperation;
import org.openrefine.wikidata.schema.WikibaseSchema;
import static org.openrefine.wikidata.commands.CommandUtilities.respondError;
import com.google.refine.commands.Command;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.process.Process;
import com.google.refine.util.ParsingUtilities;
public class SaveWikibaseSchemaCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
String jsonString = request.getParameter("schema");
if (jsonString == null) {
respondError(response, "No Wikibase schema provided.");
return;
}
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
WikibaseSchema schema = WikibaseSchema.reconstruct(json);
AbstractOperation op = new SaveWikibaseSchemaOperation(schema);
Process process = op.createProcess(project, new Properties());
performProcessAndRespond(request, response, project, process);
} catch (JSONException e) {
// We do not use respondException here because this is an expected
// exception which happens every time a user tries to save an incomplete
// schema - the exception should not be logged.
respondError(response, "Wikibase schema could not be parsed.");
} catch (Exception e) {
// This is an unexpected exception, so we log it.
respondException(response, e);
}
}
}

148
extensions/wikidata/src/org/openrefine/wikidata/editing/ConnectionManager.java Normal file
View File

@ -0,0 +1,148 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.io.IOException;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
import org.wikidata.wdtk.wikibaseapi.LoginFailedException;
import com.google.refine.ProjectManager;
import com.google.refine.preference.PreferenceStore;
/**
* Manages a connection to Wikidata, with login credentials stored in the
* preferences.
*
* Ideally, we should store only the cookies and not the password. But
* Wikidata-Toolkit does not allow for that as cookies are kept private.
*
* This class is also hard-coded for Wikidata: generalization to other Wikibase
* instances should be feasible though.
*
* @author Antonin Delpeuch
*/
public class ConnectionManager {
final static Logger logger = LoggerFactory.getLogger("connection_mananger");
public static final String PREFERENCE_STORE_KEY = "wikidata_credentials";
private PreferenceStore prefStore;
private ApiConnection connection;
private static final ConnectionManager instance = new ConnectionManager();
public static ConnectionManager getInstance() {
return instance;
}
private ConnectionManager() {
prefStore = ProjectManager.singleton.getPreferenceStore();
connection = null;
restoreSavedConnection();
}
public void login(String username, String password, boolean rememberCredentials) {
if (rememberCredentials) {
try {
JSONArray array = new JSONArray();
JSONObject obj = new JSONObject();
obj.put("username", username);
obj.put("password", password);
array.put(obj);
prefStore.put(PREFERENCE_STORE_KEY, array);
} catch (JSONException e) {
logger.error(e.getMessage());
}
}
connection = ApiConnection.getWikidataApiConnection();
try {
connection.login(username, password);
} catch (LoginFailedException e) {
connection = null;
}
}
public void restoreSavedConnection() {
JSONObject savedCredentials = getStoredCredentials();
if (savedCredentials != null) {
connection = ApiConnection.getWikidataApiConnection();
try {
connection.login(savedCredentials.getString("username"), savedCredentials.getString("password"));
} catch (LoginFailedException e) {
connection = null;
} catch (JSONException e) {
connection = null;
}
}
}
public JSONObject getStoredCredentials() {
JSONArray array = (JSONArray) prefStore.get(PREFERENCE_STORE_KEY);
if (array != null && array.length() > 0) {
try {
return array.getJSONObject(0);
} catch (JSONException e) {
logger.error(e.getMessage());
}
}
return null;
}
public void logout() {
prefStore.put(PREFERENCE_STORE_KEY, new JSONArray());
if (connection != null) {
try {
connection.logout();
connection = null;
} catch (IOException e) {
logger.error(e.getMessage());
}
}
}
public ApiConnection getConnection() {
return connection;
}
public boolean isLoggedIn() {
return connection != null;
}
public String getUsername() {
if (connection != null) {
return connection.getCurrentUser();
} else {
return null;
}
}
}

210
extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java Normal file
View File

@ -0,0 +1,210 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
/**
* Schedules and performs a list of updates to items via the API.
*
* @author Antonin Delpeuch
*
*/
public class EditBatchProcessor {
static final Logger logger = LoggerFactory.getLogger(EditBatchProcessor.class);
private WikibaseDataFetcher fetcher;
private WikibaseDataEditor editor;
private NewItemLibrary library;
private List<ItemUpdate> scheduled;
private String summary;
private List<ItemUpdate> remainingUpdates;
private List<ItemUpdate> currentBatch;
private int batchCursor;
private int globalCursor;
private Map<String, EntityDocument> currentDocs;
private int batchSize;
/**
* Initiates the process of pushing a batch of updates to Wikibase. This
* schedules the updates and is a prerequisite for calling
* {@link performOneEdit}.
*
* @param fetcher
* the fetcher to use to retrieve the current state of items
* @param editor
* the object to use to perform the edits
* @param updates
* the list of item updates to perform
* @param library
* the library to use to keep track of new item creation
* @param summary
* the summary to append to all edits
* @param batchSize
* the number of items that should be retrieved in one go from the
* API
*/
public EditBatchProcessor(WikibaseDataFetcher fetcher, WikibaseDataEditor editor, List<ItemUpdate> updates,
NewItemLibrary library, String summary, int batchSize) {
this.fetcher = fetcher;
this.editor = editor;
editor.setEditAsBot(true); // this will not do anything if the user does not
// have a bot flag, and this is generally wanted if they have one.
this.library = library;
this.summary = summary;
this.batchSize = batchSize;
// Schedule the edit batch
WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler();
this.scheduled = scheduler.schedule(updates);
this.globalCursor = 0;
this.batchCursor = 0;
this.remainingUpdates = new ArrayList<>(scheduled);
this.currentBatch = Collections.emptyList();
this.currentDocs = Collections.emptyMap();
}
/**
* Performs the next edit in the batch.
*
* @throws InterruptedException
*/
public void performEdit()
throws InterruptedException {
if (remainingEdits() == 0) {
return;
}
if (batchCursor == currentBatch.size()) {
prepareNewBatch();
}
ItemUpdate update = currentBatch.get(batchCursor);
// Rewrite mentions to new items
ReconEntityRewriter rewriter = new ReconEntityRewriter(library, update.getItemId());
update = rewriter.rewrite(update);
try {
// New item
if (update.isNew()) {
ReconEntityIdValue newCell = (ReconEntityIdValue) update.getItemId();
update = update.normalizeLabelsAndAliases();
ItemDocument itemDocument = Datamodel.makeItemDocument(update.getItemId(),
update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()), update.getAddedStatementGroups(),
Collections.emptyMap());
ItemDocument createdDoc = editor.createItemDocument(itemDocument, summary);
library.setQid(newCell.getReconInternalId(), createdDoc.getItemId().getId());
} else {
// Existing item
ItemDocument currentDocument = (ItemDocument) currentDocs.get(update.getItemId().getId());
editor.updateTermsStatements(currentDocument, update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()),
new ArrayList<MonolingualTextValue>(),
update.getAddedStatements().stream().collect(Collectors.toList()),
update.getDeletedStatements().stream().collect(Collectors.toList()), summary);
}
} catch (MediaWikiApiErrorException e) {
// TODO find a way to report these errors to the user in a nice way
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
batchCursor++;
}
/**
* @return the number of edits that remain to be done in the current batch
*/
public int remainingEdits() {
return scheduled.size() - (globalCursor + batchCursor);
}
/**
* @return the progress, measured as a percentage
*/
public int progress() {
return (100 * (globalCursor + batchCursor)) / scheduled.size();
}
protected void prepareNewBatch()
throws InterruptedException {
// remove the previous batch from the remainingUpdates
globalCursor += currentBatch.size();
currentBatch.clear();
if (remainingUpdates.size() < batchSize) {
currentBatch = remainingUpdates;
remainingUpdates = Collections.emptyList();
} else {
currentBatch = remainingUpdates.subList(0, batchSize);
}
List<String> qidsToFetch = currentBatch.stream().filter(u -> !u.isNew()).map(u -> u.getItemId().getId())
.collect(Collectors.toList());
// Get the current documents for this batch of updates
logger.info("Requesting documents");
currentDocs = null;
int retries = 3;
while (currentDocs == null && retries > 0) {
try {
currentDocs = fetcher.getEntityDocuments(qidsToFetch);
} catch (MediaWikiApiErrorException e) {
e.printStackTrace();
Thread.sleep(5000);
}
retries--;
}
if (currentDocs == null) {
throw new InterruptedException("Fetching current documents failed.");
}
batchCursor = 0;
}
}

159
extensions/wikidata/src/org/openrefine/wikidata/editing/NewItemLibrary.java Normal file
View File

@ -0,0 +1,159 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.ReconStats;
import com.google.refine.model.Row;
/**
* This keeps track of the new items that we have created for each internal
* reconciliation id.
*
* @author Antonin Delpeuch
*
*/
public class NewItemLibrary {
private Map<Long, String> map;
public NewItemLibrary() {
map = new HashMap<>();
}
@JsonCreator
public NewItemLibrary(@JsonProperty("qidMap") Map<Long, String> map) {
this.map = map;
}
/**
* Retrieves the Qid allocated to a given new cell
*
* @param id:
* the fake ItemId generated by the cell
* @return the qid (or null if unallocated yet)
*/
public String getQid(long id) {
return map.get(id);
}
/**
* Stores a Qid associated to a new cell
*
* @param id
* : the internal reconciliation id of the new cell
* @param qid
* : the associated Qid returned by Wikibase
*/
public void setQid(long id, String qid) {
map.put(id, qid);
}
/**
* Changes the "new" reconciled cells to their allocated qids for later use.
*
* @param reset:
* set to true to revert the operation (set cells to "new")
*/
public void updateReconciledCells(Project project, boolean reset) {
Set<Integer> impactedColumns = new HashSet<>();
/*
* Note that there is a slight violation of OpenRefine's model here: if we
* reconcile multiple cells to the same new item, and then perform this
* operation on a subset of the corresponding rows, we are going to modify cells
* that are outside the facet (because they are reconciled to the same cell).
* But I think this is the right thing to do.
*/
for (Row row : project.rows) {
for (int i = 0; i != row.cells.size(); i++) {
Cell cell = row.cells.get(i);
if (cell == null || cell.recon == null) {
continue;
}
Recon recon = cell.recon;
if (Recon.Judgment.New.equals(recon.judgment) && !reset
&& map.containsKey(recon.id)) {
recon.judgment = Recon.Judgment.Matched;
recon.match = new ReconCandidate(map.get(recon.id), cell.value.toString(),
new String[0], 100);
impactedColumns.add(i);
} else if (Recon.Judgment.Matched.equals(recon.judgment) && reset
&& map.containsKey(recon.id)) {
recon.judgment = Recon.Judgment.New;
recon.match = null;
impactedColumns.add(i);
}
}
}
// Update reconciliation statistics for impacted columns
for (Integer colId : impactedColumns) {
Column column = project.columnModel.getColumnByCellIndex(colId);
column.setReconStats(ReconStats.create(project, colId));
}
}
/**
* Getter, only meant to be used by Jackson
*
* @return the underlying map
*/
@JsonProperty("qidMap")
public Map<Long, String> getQidMap() {
return map;
}
@Override
public boolean equals(Object other) {
if (other == null || !NewItemLibrary.class.isInstance(other)) {
return false;
}
NewItemLibrary otherLibrary = (NewItemLibrary) other;
return map.equals(otherLibrary.getQidMap());
}
@Override
public int hashCode() {
return map.hashCode();
}
@Override
public String toString() {
return map.toString();
}
}

102
extensions/wikidata/src/org/openrefine/wikidata/editing/ReconEntityRewriter.java Normal file
View File

@ -0,0 +1,102 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.helpers.DatamodelConverter;
import org.wikidata.wdtk.datamodel.implementation.DataObjectFactoryImpl;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A class that rewrites an {@link ItemUpdate}, replacing reconciled entity id
* values by their concrete values after creation of all the new items involved.
*
* If an item has not been created yet, an {@link IllegalArgumentException} will
* be raised.
*
* The subject is treated as a special case: it is returned unchanged. This is
* because it is guaranteed not to appear in the update (but it does appear in
* the datamodel representation as the subject is passed around to the Claim
* objects its document contains).
*
* @author Antonin Delpeuch
*
*/
public class ReconEntityRewriter extends DatamodelConverter {
private NewItemLibrary library;
private ItemIdValue subject;
/**
* Constructor. Sets up a rewriter which uses the provided library to look up
* qids of new items, and the subject (which should not be rewritten).
*
* @param library
* @param subject
*/
public ReconEntityRewriter(NewItemLibrary library, ItemIdValue subject) {
super(new DataObjectFactoryImpl());
this.library = library;
this.subject = subject;
}
@Override
public ItemIdValue copy(ItemIdValue value) {
if (subject.equals(value)) {
return value;
}
if (value instanceof ReconItemIdValue) {
ReconItemIdValue recon = (ReconItemIdValue) value;
if (recon.isNew()) {
String newId = library.getQid(recon.getReconInternalId());
if (newId == null) {
throw new IllegalArgumentException(
"Trying to rewrite an update where a new item was not created yet.");
}
return Datamodel.makeItemIdValue(newId, recon.getRecon().identifierSpace);
}
}
return super.copy(value);
}
public ItemUpdate rewrite(ItemUpdate update) {
Set<MonolingualTextValue> labels = update.getLabels().stream().map(l -> copy(l)).collect(Collectors.toSet());
Set<MonolingualTextValue> descriptions = update.getDescriptions().stream().map(l -> copy(l))
.collect(Collectors.toSet());
Set<MonolingualTextValue> aliases = update.getAliases().stream().map(l -> copy(l)).collect(Collectors.toSet());
List<Statement> addedStatements = update.getAddedStatements().stream().map(l -> copy(l))
.collect(Collectors.toList());
Set<Statement> deletedStatements = update.getDeletedStatements().stream().map(l -> copy(l))
.collect(Collectors.toSet());
return new ItemUpdate(update.getItemId(), addedStatements, deletedStatements, labels, descriptions, aliases);
}
}

86
extensions/wikidata/src/org/openrefine/wikidata/editing/WikibaseCredentials.java Normal file
View File

@ -0,0 +1,86 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.editing;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
/**
* This is just the necessary bits to store Wikidata credentials in OpenRefine's
* preference store.
*
* @author Antonin Delpeuch
*
*/
class WikibaseCredentials implements Jsonizable {
private String username;
private String password;
public WikibaseCredentials() {
username = null;
password = null;
}
public WikibaseCredentials(String username, String password) {
this.username = username;
this.password = password;
}
public String getUsername() {
return username;
}
public String getPassword() {
return password;
}
public boolean isNonNull() {
return username != null && password != null && !"null".equals(username) && !"null".equals(password);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("class");
writer.value(this.getClass().getName());
writer.key("username");
writer.value(username);
writer.key("password");
writer.value(password);
writer.endObject();
}
public static WikibaseCredentials load(JSONObject obj)
throws JSONException {
return new WikibaseCredentials(obj.getString("username"), obj.getString("password"));
}
}

106
extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java Normal file
View File

@ -0,0 +1,106 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import java.math.BigDecimal;
import java.util.Locale;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.updates.scheduler.QuickStatementsUpdateScheduler;
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
/**
* Prints a Wikibase value as a string as required by QuickStatements. Format
* documentation: https://www.wikidata.org/wiki/Help:QuickStatements
*
* Any new entity id will be assumed to be the last one created, represented
* with "LAST". It is fine to do this assumption because we are working on edit
* batches previously scheduled by {@link QuickStatementsUpdateScheduler}.
*
* @author Antonin Delpeuch
*
*/
public class QSValuePrinter implements ValueVisitor<String> {
@Override
public String visit(DatatypeIdValue value) {
// impossible case (this is actually a bug in WDTK, DatatypeIdValue should not subclass Value)
throw new IllegalArgumentException();
}
@Override
public String visit(EntityIdValue value) {
if (ReconEntityIdValue.class.isInstance(value) && ((ReconEntityIdValue) value).isNew()) {
return "LAST";
}
return value.getId();
}
@Override
public String visit(GlobeCoordinatesValue value) {
return String.format(Locale.US, "@%f/%f", value.getLatitude(), value.getLongitude());
}
@Override
public String visit(MonolingualTextValue value) {
return String.format("%s:\"%s\"", value.getLanguageCode(), value.getText());
}
@Override
public String visit(QuantityValue value) {
String unitPrefix = "http://www.wikidata.org/entity/Q";
String unitIri = value.getUnit();
String unitRepresentation = "", boundsRepresentation = "";
if (!unitIri.isEmpty()) {
if (!unitIri.startsWith(unitPrefix)) return null; // QuickStatements only accepts Qids as units
unitRepresentation = "U" + unitIri.substring(unitPrefix.length());
}
if (value.getLowerBound() != null) {
// bounds are always null at the same time so we know they are both not null
BigDecimal lowerBound = value.getLowerBound();
BigDecimal upperBound = value.getUpperBound();
boundsRepresentation = String.format(Locale.US, "[%s,%s]", lowerBound.toString(), upperBound.toString());
}
return String.format(Locale.US, "%s%s%s", value.getNumericValue().toString(), boundsRepresentation,
unitRepresentation);
}
@Override
public String visit(StringValue value) {
return "\"" + value.getString() + "\"";
}
@Override
public String visit(TimeValue value) {
return String.format("+%04d-%02d-%02dT%02d:%02d:%02dZ/%d", value.getYear(), value.getMonth(), value.getDay(),
value.getHour(), value.getMinute(), value.getSecond(), value.getPrecision());
}
}

191
extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java Normal file
View File

@ -0,0 +1,191 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.exporters;
import java.io.IOException;
import java.io.Writer;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.ImpossibleSchedulingException;
import org.openrefine.wikidata.updates.scheduler.QuickStatementsUpdateScheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
import com.google.refine.browsing.Engine;
import com.google.refine.exporters.WriterExporter;
import com.google.refine.model.Project;
public class QuickStatementsExporter implements WriterExporter {
final static Logger logger = LoggerFactory.getLogger("QuickStatementsExporter");
public static final String impossibleSchedulingErrorMessage = "This edit batch cannot be performed with QuickStatements due to the structure of its new items.";
public static final String noSchemaErrorMessage = "No schema was provided. You need to align your project with Wikidata first.";
public QuickStatementsExporter() {
}
@Override
public String getContentType() {
return "text/plain";
}
@Override
public void export(Project project, Properties options, Engine engine, Writer writer)
throws IOException {
WikibaseSchema schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema");
if (schema == null) {
writer.write(noSchemaErrorMessage);
} else {
translateSchema(project, engine, schema, writer);
}
}
/**
* Exports a project and a schema to a QuickStatements file
*
* @param project
* the project to translate
* @param engine
* the engine used for evaluation of the edits
* @param schema
* the WikibaseSchema used for translation of tabular data to edits
* @param writer
* the writer to which the QS should be written
* @throws IOException
*/
public void translateSchema(Project project, Engine engine, WikibaseSchema schema, Writer writer)
throws IOException {
List<ItemUpdate> items = schema.evaluate(project, engine);
translateItemList(items, writer);
}
public void translateItemList(List<ItemUpdate> updates, Writer writer)
throws IOException {
QuickStatementsUpdateScheduler scheduler = new QuickStatementsUpdateScheduler();
try {
List<ItemUpdate> scheduled = scheduler.schedule(updates);
for (ItemUpdate item : scheduled) {
translateItem(item, writer);
}
} catch (ImpossibleSchedulingException e) {
writer.write(impossibleSchedulingErrorMessage);
}
}
protected void translateNameDescr(String qid, Set<MonolingualTextValue> values, String prefix, ItemIdValue id,
Writer writer)
throws IOException {
for (MonolingualTextValue value : values) {
writer.write(qid + "\t");
writer.write(prefix);
writer.write(value.getLanguageCode());
writer.write("\t\"");
writer.write(value.getText());
writer.write("\"\n");
}
}
protected void translateItem(ItemUpdate item, Writer writer)
throws IOException {
String qid = item.getItemId().getId();
if (item.isNew()) {
writer.write("CREATE\n");
qid = "LAST";
item = item.normalizeLabelsAndAliases();
}
translateNameDescr(qid, item.getLabels(), "L", item.getItemId(), writer);
translateNameDescr(qid, item.getDescriptions(), "D", item.getItemId(), writer);
translateNameDescr(qid, item.getAliases(), "A", item.getItemId(), writer);
for (Statement s : item.getAddedStatements()) {
translateStatement(qid, s, s.getClaim().getMainSnak().getPropertyId().getId(), true, writer);
}
for (Statement s : item.getDeletedStatements()) {
translateStatement(qid, s, s.getClaim().getMainSnak().getPropertyId().getId(), false, writer);
}
}
protected void translateStatement(String qid, Statement statement, String pid, boolean add, Writer writer)
throws IOException {
Claim claim = statement.getClaim();
Value val = claim.getValue();
ValueVisitor<String> vv = new QSValuePrinter();
String targetValue = val.accept(vv);
if (targetValue != null) {
if (!add) {
writer.write("- ");
}
writer.write(qid + "\t" + pid + "\t" + targetValue);
for (SnakGroup q : claim.getQualifiers()) {
translateSnakGroup(q, false, writer);
}
for (Reference r : statement.getReferences()) {
for (SnakGroup g : r.getSnakGroups()) {
translateSnakGroup(g, true, writer);
}
break; // QS only supports one reference
}
writer.write("\n");
}
}
protected void translateSnakGroup(SnakGroup sg, boolean reference, Writer writer)
throws IOException {
for (Snak s : sg.getSnaks()) {
translateSnak(s, reference, writer);
}
}
protected void translateSnak(Snak s, boolean reference, Writer writer)
throws IOException {
String pid = s.getPropertyId().getId();
if (reference) {
pid = pid.replace('P', 'S');
}
Value val = s.getValue();
ValueVisitor<String> vv = new QSValuePrinter();
String valStr = val.accept(vv);
if (valStr != null) {
writer.write("\t" + pid + "\t" + valStr);
}
}
}

237
extensions/wikidata/src/org/openrefine/wikidata/operations/PerformWikibaseEditsOperation.java Normal file
View File

@ -0,0 +1,237 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.List;
import java.util.Properties;
import java.util.Random;
import org.apache.commons.lang.Validate;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.editing.ConnectionManager;
import org.openrefine.wikidata.editing.EditBatchProcessor;
import org.openrefine.wikidata.editing.NewItemLibrary;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.util.WebResourceFetcherImpl;
import org.wikidata.wdtk.wikibaseapi.ApiConnection;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.browsing.Engine;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.process.LongRunningProcess;
import com.google.refine.process.Process;
import com.google.refine.util.Pool;
public class PerformWikibaseEditsOperation extends EngineDependentOperation {
static final Logger logger = LoggerFactory.getLogger(PerformWikibaseEditsOperation.class);
private String summary;
public PerformWikibaseEditsOperation(JSONObject engineConfig, String summary) {
super(engineConfig);
Validate.notNull(summary, "An edit summary must be provided.");
Validate.notEmpty(summary, "An edit summary must be provided.");
this.summary = summary;
}
static public AbstractOperation reconstruct(Project project, JSONObject obj)
throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
String summary = null;
if (obj.has("summary")) {
summary = obj.getString("summary");
}
return new PerformWikibaseEditsOperation(engineConfig, summary);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op");
writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description");
writer.value("Perform Wikibase edits");
writer.key("summary");
writer.value(summary);
writer.key("engineConfig");
writer.value(getEngineConfig());
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Peform edits on Wikidata";
}
@Override
public Process createProcess(Project project, Properties options)
throws Exception {
return new PerformEditsProcess(project, createEngine(project), getBriefDescription(project), summary);
}
static public class PerformWikibaseEditsChange implements Change {
private NewItemLibrary newItemLibrary;
public PerformWikibaseEditsChange(NewItemLibrary library) {
newItemLibrary = library;
}
@Override
public void apply(Project project) {
// we don't re-run changes on Wikidata
newItemLibrary.updateReconciledCells(project, false);
}
@Override
public void revert(Project project) {
// this does not do anything on Wikibase side -
// (we don't revert changes on Wikidata either)
newItemLibrary.updateReconciledCells(project, true);
}
@Override
public void save(Writer writer, Properties options)
throws IOException {
if (newItemLibrary != null) {
writer.write("newItems=");
ObjectMapper mapper = new ObjectMapper();
writer.write(mapper.writeValueAsString(newItemLibrary) + "\n");
}
writer.write("/ec/\n"); // end of change
}
static public Change load(LineNumberReader reader, Pool pool)
throws Exception {
NewItemLibrary library = new NewItemLibrary();
String line = null;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("newItems".equals(field)) {
ObjectMapper mapper = new ObjectMapper();
library = mapper.readValue(value, NewItemLibrary.class);
}
}
return new PerformWikibaseEditsChange(library);
}
}
public class PerformEditsProcess extends LongRunningProcess implements Runnable {
protected Project _project;
protected Engine _engine;
protected WikibaseSchema _schema;
protected String _summary;
protected final long _historyEntryID;
protected PerformEditsProcess(Project project, Engine engine, String description, String summary) {
super(description);
this._project = project;
this._engine = engine;
this._schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema");
this._summary = summary;
this._historyEntryID = HistoryEntry.allocateID();
}
@Override
public void run() {
WebResourceFetcherImpl.setUserAgent("OpenRefine Wikidata extension");
ConnectionManager manager = ConnectionManager.getInstance();
if (!manager.isLoggedIn()) {
return;
}
ApiConnection connection = manager.getConnection();
WikibaseDataFetcher wbdf = new WikibaseDataFetcher(connection, _schema.getBaseIri());
WikibaseDataEditor wbde = new WikibaseDataEditor(connection, _schema.getBaseIri());
// Generate batch token
long token = (new Random()).nextLong();
String summary = _summary + String.format(" ([[:toollabs:editgroups/b/OR/%s|details]])",
(Long.toHexString(token).substring(0, 7)));
// Evaluate the schema
List<ItemUpdate> itemDocuments = _schema.evaluate(_project, _engine);
// Prepare the edits
NewItemLibrary newItemLibrary = new NewItemLibrary();
EditBatchProcessor processor = new EditBatchProcessor(wbdf, wbde, itemDocuments, newItemLibrary, summary,
50);
// Perform edits
logger.info("Performing edits");
while (processor.remainingEdits() > 0) {
try {
processor.performEdit();
} catch (InterruptedException e) {
_canceled = true;
}
_progress = processor.progress();
if (_canceled) {
break;
}
}
_progress = 100;
if (!_canceled) {
Change change = new PerformWikibaseEditsChange(newItemLibrary);
HistoryEntry historyEntry = new HistoryEntry(_historyEntryID, _project, _description,
PerformWikibaseEditsOperation.this, change);
_project.history.addEntry(historyEntry);
_project.processManager.onDoneProcess(this);
}
}
@Override
protected Runnable getRunnable() {
return this;
}
}
}

161
extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java Normal file
View File

@ -0,0 +1,161 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.schema.WikibaseSchema;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
public class SaveWikibaseSchemaOperation extends AbstractOperation {
final public String operationDescription = "Save Wikibase schema";
final protected WikibaseSchema _schema;
public SaveWikibaseSchemaOperation(WikibaseSchema schema) {
this._schema = schema;
}
static public AbstractOperation reconstruct(Project project, JSONObject obj)
throws Exception {
return new SaveWikibaseSchemaOperation(WikibaseSchema.reconstruct(obj.getJSONObject("schema")));
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op");
writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description");
writer.value(operationDescription);
writer.key("schema");
_schema.write(writer, options);
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Save Wikibase schema skelton";
}
@Override
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID)
throws Exception {
String description = operationDescription;
Change change = new WikibaseSchemaChange(_schema);
return new HistoryEntry(historyEntryID, project, description, SaveWikibaseSchemaOperation.this, change);
}
static public class WikibaseSchemaChange implements Change {
final protected WikibaseSchema _newSchema;
protected WikibaseSchema _oldSchema = null;
public final static String overlayModelKey = "wikibaseSchema";
public WikibaseSchemaChange(WikibaseSchema newSchema) {
_newSchema = newSchema;
}
public void apply(Project project) {
synchronized (project) {
_oldSchema = (WikibaseSchema) project.overlayModels.get(overlayModelKey);
project.overlayModels.put(overlayModelKey, _newSchema);
}
}
public void revert(Project project) {
synchronized (project) {
if (_oldSchema == null) {
project.overlayModels.remove(overlayModelKey);
} else {
project.overlayModels.put(overlayModelKey, _oldSchema);
}
}
}
public void save(Writer writer, Properties options)
throws IOException {
writer.write("newSchema=");
writeWikibaseSchema(_newSchema, writer);
writer.write('\n');
writer.write("oldSchema=");
writeWikibaseSchema(_oldSchema, writer);
writer.write('\n');
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool)
throws Exception {
WikibaseSchema oldSchema = null;
WikibaseSchema newSchema = null;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("oldSchema".equals(field) && value.length() > 0) {
oldSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
} else if ("newSchema".equals(field) && value.length() > 0) {
newSchema = WikibaseSchema.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
}
}
WikibaseSchemaChange change = new WikibaseSchemaChange(newSchema);
change._oldSchema = oldSchema;
return change;
}
static protected void writeWikibaseSchema(WikibaseSchema s, Writer writer)
throws IOException {
if (s != null) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
s.write(jsonWriter, new Properties());
} catch (JSONException e) {
e.printStackTrace();
}
}
}
}
}

137
extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java Normal file
View File

@ -0,0 +1,137 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.Set;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* An object that fetches constraints about properties.
*
* @author Antonin Delpeuch
*
*/
public interface ConstraintFetcher {
/**
* Retrieves the regular expression for formatting a property, or null if there
* is no such constraint
*
* @param pid
* @return the expression of a regular expression which should be compatible
* with java.util.regex
*/
String getFormatRegex(PropertyIdValue pid);
/**
* Retrieves the property that is the inverse of a given property
*
* @param pid:
* the property to retrieve the inverse for
* @return the pid of the inverse property
*/
PropertyIdValue getInversePid(PropertyIdValue pid);
/**
* Is this property supposed to be symmetric (its own inverse)?
*/
boolean isSymmetric(PropertyIdValue pid);
/**
* Can this property be used as values?
*/
boolean allowedAsValue(PropertyIdValue pid);
/**
* Can this property be used as qualifiers?
*/
boolean allowedAsQualifier(PropertyIdValue pid);
/**
* Can this property be used in a reference?
*/
boolean allowedAsReference(PropertyIdValue pid);
/**
* Get the list of allowed qualifiers (as property ids) for this property (null
* if any)
*/
Set<PropertyIdValue> allowedQualifiers(PropertyIdValue pid);
/**
* Get the list of mandatory qualifiers (as property ids) for this property
* (null if any)
*/
Set<PropertyIdValue> mandatoryQualifiers(PropertyIdValue pid);
/**
* Get the set of allowed values for this property (null if no such constraint).
* This set may contain null if one of the allowed values in novalue or somevalue.
*/
Set<Value> allowedValues(PropertyIdValue pid);
/**
* Get the set of disallowed values for this property (null if no such constraint).
* This set may contain null if one of the allowed values in novalue or somevalue.
*/
Set<Value> disallowedValues(PropertyIdValue pid);
/**
* Is this property expected to have at most one value per item?
*/
boolean hasSingleValue(PropertyIdValue pid);
/**
* Is this property expected to have a single best value only?
*/
boolean hasSingleBestValue(PropertyIdValue pid);
/**
* Is this property expected to have distinct values?
*/
boolean hasDistinctValues(PropertyIdValue pid);
/**
* Can statements using this property have uncertainty bounds?
*/
boolean boundsAllowed(PropertyIdValue pid);
/**
* Is this property expected to have integer values only?
*/
boolean integerValued(PropertyIdValue pid);
/**
* Returns the allowed units for this property. If empty, no unit is allowed. If null, any unit is allowed.
*/
Set<ItemIdValue> allowedUnits(PropertyIdValue pid);
/**
* Can this property be used on items?
*/
boolean usableOnItems(PropertyIdValue pid);
}

130
extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java Normal file
View File

@ -0,0 +1,130 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.openrefine.wikidata.qa.scrutinizers.DistinctValuesScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.EditScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.EntityTypeScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.FormatScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.InverseConstraintScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.NewItemScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.NoEditsMadeScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.QualifierCompatibilityScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.QuantityScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.RestrictedPositionScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.RestrictedValuesScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.SelfReferentialScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.SingleValueScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.UnsourcedScrutinizer;
import org.openrefine.wikidata.qa.scrutinizers.WhitespaceScrutinizer;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
/**
* Runs a collection of edit scrutinizers on an edit batch.
*
* @author Antonin Delpeuch
*
*/
public class EditInspector {
private Map<String, EditScrutinizer> scrutinizers;
private QAWarningStore warningStore;
private ConstraintFetcher fetcher;
public EditInspector(QAWarningStore warningStore) {
this.scrutinizers = new HashMap<>();
this.fetcher = new WikidataConstraintFetcher();
this.warningStore = warningStore;
// Register all known scrutinizers here
register(new NewItemScrutinizer());
register(new FormatScrutinizer());
register(new InverseConstraintScrutinizer());
register(new SelfReferentialScrutinizer());
register(new UnsourcedScrutinizer());
register(new RestrictedPositionScrutinizer());
register(new QualifierCompatibilityScrutinizer());
register(new SingleValueScrutinizer());
register(new DistinctValuesScrutinizer());
register(new NoEditsMadeScrutinizer());
register(new WhitespaceScrutinizer());
register(new QuantityScrutinizer());
register(new RestrictedValuesScrutinizer());
register(new EntityTypeScrutinizer());
}
/**
* Adds a new scrutinizer to the inspector
*
* @param scrutinizer
*/
public void register(EditScrutinizer scrutinizer) {
String key = scrutinizer.getClass().getName();
scrutinizers.put(key, scrutinizer);
scrutinizer.setStore(warningStore);
scrutinizer.setFetcher(fetcher);
}
/**
* Inspect a batch of edits with the registered scrutinizers
*
* @param editBatch
*/
public void inspect(List<ItemUpdate> editBatch) {
// First, schedule them with some scheduler,
// so that all newly created entities appear in the batch
WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler();
editBatch = scheduler.schedule(editBatch);
Map<EntityIdValue, ItemUpdate> updates = ItemUpdate.groupBySubject(editBatch);
List<ItemUpdate> mergedUpdates = updates.values().stream().collect(Collectors.toList());
for (EditScrutinizer scrutinizer : scrutinizers.values()) {
scrutinizer.batchIsBeginning();
}
for(ItemUpdate update : mergedUpdates) {
if(!update.isNull()) {
for (EditScrutinizer scrutinizer : scrutinizers.values()) {
scrutinizer.scrutinize(update);
}
}
}
for(EditScrutinizer scrutinizer : scrutinizers.values()) {
scrutinizer.batchIsFinished();
}
if (warningStore.getNbWarnings() == 0) {
warningStore.addWarning(new QAWarning("no-issue-detected", null, QAWarning.Severity.INFO, 0));
}
}
}

167
extensions/wikidata/src/org/openrefine/wikidata/qa/QAWarning.java Normal file
View File

@ -0,0 +1,167 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.utils.JacksonJsonizable;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A class to represent a QA warning emitted by the Wikidata schema This could
* probably be reused at a broader scale, for instance for Data Package
* validation.
*
* @author Antonin Delpeuch
*
*/
public class QAWarning extends JacksonJsonizable implements Comparable<QAWarning> {
public enum Severity {
INFO, // We just report something to the user but it is probably fine
WARNING, // Edits that look wrong but in some cases they are actually fine
IMPORTANT, // There is almost surely something wrong about the edit but in rare cases we
// might want to allow it
CRITICAL, // We should never edit if there is a critical issue
}
/// The type of QA warning emitted
private final String type;
// The key for aggregation of other QA warnings together - this specializes the
// id
private final String bucketId;
// The severity of the issue
private final Severity severity;
// The number of times this issue was found
private final int count;
// Other details about the warning, that can be displayed to the user
private final Map<String, Object> properties;
public QAWarning(String type, String bucketId, Severity severity, int count) {
Validate.notNull(type);
this.type = type;
this.bucketId = bucketId;
Validate.notNull(severity);
this.severity = severity;
this.count = count;
this.properties = new HashMap<>();
}
/**
* @return the full key for aggregation of QA warnings
*/
@JsonIgnore
public String getAggregationId() {
if (this.bucketId != null) {
return this.type + "_" + this.bucketId;
} else {
return this.type;
}
}
/**
* Aggregates another QA warning of the same aggregation id.
*
* @param other
*/
public QAWarning aggregate(QAWarning other) {
assert other.getAggregationId().equals(getAggregationId());
int newCount = count + other.getCount();
Severity newSeverity = severity;
if (other.getSeverity().compareTo(severity) > 0) {
newSeverity = other.getSeverity();
}
QAWarning merged = new QAWarning(getType(), getBucketId(), newSeverity, newCount);
for (Entry<String, Object> entry : properties.entrySet()) {
merged.setProperty(entry.getKey(), entry.getValue());
}
for (Entry<String, Object> entry : other.getProperties().entrySet()) {
merged.setProperty(entry.getKey(), entry.getValue());
}
return merged;
}
/**
* Sets a property of the QA warning, to be used by the front-end for display.
*
* @param key:
* the name of the property
* @param value
* should be Jackson-serializable
*/
public void setProperty(String key, Object value) {
this.properties.put(key, value);
}
@JsonProperty("type")
public String getType() {
return type;
}
@JsonProperty("bucketId")
public String getBucketId() {
return bucketId;
}
@JsonProperty("severity")
public Severity getSeverity() {
return severity;
}
@JsonProperty("count")
public int getCount() {
return count;
}
@JsonProperty("properties")
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public Map<String, Object> getProperties() {
return properties;
}
/**
* Warnings are sorted by decreasing severity.
*/
@Override
public int compareTo(QAWarning other) {
return -severity.compareTo(other.getSeverity());
}
@Override
public boolean equals(Object other) {
if (other == null || !QAWarning.class.isInstance(other)) {
return false;
}
QAWarning otherWarning = (QAWarning) other;
return type.equals(otherWarning.getType()) && bucketId.equals(otherWarning.getBucketId())
&& severity.equals(otherWarning.getSeverity()) && count == otherWarning.getCount()
&& properties.equals(otherWarning.getProperties());
}
}

99
extensions/wikidata/src/org/openrefine/wikidata/qa/QAWarningStore.java Normal file
View File

@ -0,0 +1,99 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* A store for QA warnings which aggregates them by type.
*
* @author Antonin Delpeuch
*/
public class QAWarningStore {
@JsonIgnore
private Map<String, QAWarning> map;
@JsonIgnore
private QAWarning.Severity maxSeverity;
@JsonIgnore
private int totalWarnings;
public QAWarningStore() {
this.map = new HashMap<>();
this.maxSeverity = QAWarning.Severity.INFO;
}
/**
* Stores a warning, aggregating it with any existing
*
* @param warning
*/
public void addWarning(QAWarning warning) {
String aggregationKey = warning.getAggregationId();
QAWarning.Severity severity = warning.getSeverity();
if (severity.compareTo(maxSeverity) > 0) {
maxSeverity = severity;
}
totalWarnings += warning.getCount();
if (map.containsKey(aggregationKey)) {
QAWarning existing = map.get(aggregationKey);
map.put(aggregationKey, existing.aggregate(warning));
} else {
map.put(aggregationKey, warning);
}
}
/**
* Returns the list of aggregated warnings, ordered by decreasing severity
*/
@JsonProperty("warnings")
public List<QAWarning> getWarnings() {
List<QAWarning> result = new ArrayList<>(map.values());
Collections.sort(result);
return result;
}
/**
* Returns the maximum severity of the stored warnings (INFO if empty)
*/
@JsonProperty("max_severity")
public QAWarning.Severity getMaxSeverity() {
return maxSeverity;
}
/**
* Returns the total number of warnings
*/
@JsonProperty("nb_warnings")
public int getNbWarnings() {
return totalWarnings;
}
}

335
extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java Normal file
View File

@ -0,0 +1,335 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.openrefine.wikidata.utils.EntityCache;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* This class provides an abstraction over the way constraint definitions are
* stored in Wikidata.
*
* @author Antonin Delpeuch
*
*/
public class WikidataConstraintFetcher implements ConstraintFetcher {
public static String WIKIDATA_CONSTRAINT_PID = "P2302";
public static String FORMAT_CONSTRAINT_QID = "Q21502404";
public static String FORMAT_REGEX_PID = "P1793";
public static String INVERSE_CONSTRAINT_QID = "Q21510855";
public static String INVERSE_PROPERTY_PID = "P2306";
public static String SYMMETRIC_CONSTRAINT_QID = "Q21510862";
public static String SCOPE_CONSTRAINT_QID = "Q53869507";
public static String SCOPE_CONSTRAINT_PID = "P5314";
public static String SCOPE_CONSTRAINT_VALUE_QID = "Q54828448";
public static String SCOPE_CONSTRAINT_QUALIFIER_QID = "Q54828449";
public static String SCOPE_CONSTRAINT_REFERENCE_QID = "Q54828450";
public static String USED_ONLY_AS_QUALIFIER_CONSTRAINT_QID = "Q21510863";
public static String USED_ONLY_AS_REFERENCE_CONSTRAINT_QID = "Q21528959";
public static String ALLOWED_QUALIFIERS_CONSTRAINT_QID = "Q21510851";
public static String ALLOWED_QUALIFIERS_CONSTRAINT_PID = "P2306";
public static String MANDATORY_QUALIFIERS_CONSTRAINT_QID = "Q21510856";
public static String MANDATORY_QUALIFIERS_CONSTRAINT_PID = "P2306";
public static String ALLOWED_VALUES_CONSTRAINT_QID = "Q21510859";
public static String ALLOWED_VALUES_CONSTRAINT_PID = "P2305";
public static String DISALLOWED_VALUES_CONSTRAINT_QID = "Q52558054";
public static String DISALLOWED_VALUES_CONSTRAINT_PID = "P2305";
public static String SINGLE_VALUE_CONSTRAINT_QID = "Q19474404";
public static String SINGLE_BEST_VALUE_CONSTRAINT_QID = "Q52060874";
public static String DISTINCT_VALUES_CONSTRAINT_QID = "Q21502410";
public static String NO_BOUNDS_CONSTRAINT_QID = "Q51723761";
public static String INTEGER_VALUED_CONSTRAINT_QID = "Q52848401";
public static String ALLOWED_UNITS_CONSTRAINT_QID = "Q21514353";
public static String ALLOWED_UNITS_CONSTRAINT_PID = "P2305";
public static String ALLOWED_ENTITY_TYPES_QID = "Q52004125";
public static String ALLOWED_ITEM_TYPE_QID = "Q29934200";
public static String ALLOWED_ENTITY_TYPES_PID = "P2305";
// The following constraints still need to be implemented:
public static String TYPE_CONSTRAINT_QID = "Q21503250";
@Override
public String getFormatRegex(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, FORMAT_CONSTRAINT_QID);
if (specs != null) {
List<Value> regexes = findValues(specs, FORMAT_REGEX_PID);
if (!regexes.isEmpty()) {
return ((StringValue) regexes.get(0)).getString();
}
}
return null;
}
@Override
public PropertyIdValue getInversePid(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, INVERSE_CONSTRAINT_QID);
if (specs != null) {
List<Value> inverses = findValues(specs, INVERSE_PROPERTY_PID);
if (!inverses.isEmpty()) {
return (PropertyIdValue) inverses.get(0);
}
}
return null;
}
@Override
public boolean allowedAsValue(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, SCOPE_CONSTRAINT_QID);
if (specs != null) {
ItemIdValue target = Datamodel.makeWikidataItemIdValue(SCOPE_CONSTRAINT_VALUE_QID);
return findValues(specs, SCOPE_CONSTRAINT_PID).contains(target);
}
return true;
}
@Override
public boolean allowedAsQualifier(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, SCOPE_CONSTRAINT_QID);
if (specs != null) {
ItemIdValue target = Datamodel.makeWikidataItemIdValue(SCOPE_CONSTRAINT_QUALIFIER_QID);
return findValues(specs, SCOPE_CONSTRAINT_PID).contains(target);
}
return true;
}
@Override
public boolean allowedAsReference(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, SCOPE_CONSTRAINT_QID);
if (specs != null) {
ItemIdValue target = Datamodel.makeWikidataItemIdValue(SCOPE_CONSTRAINT_REFERENCE_QID);
return findValues(specs, SCOPE_CONSTRAINT_PID).contains(target);
}
return true;
}
@Override
public Set<PropertyIdValue> allowedQualifiers(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, ALLOWED_QUALIFIERS_CONSTRAINT_QID);
if (specs != null) {
List<Value> properties = findValues(specs, ALLOWED_QUALIFIERS_CONSTRAINT_PID);
return properties.stream()
.filter(e -> e != null)
.map(e -> (PropertyIdValue) e)
.collect(Collectors.toSet());
}
return null;
}
@Override
public Set<PropertyIdValue> mandatoryQualifiers(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, MANDATORY_QUALIFIERS_CONSTRAINT_QID);
if (specs != null) {
List<Value> properties = findValues(specs, MANDATORY_QUALIFIERS_CONSTRAINT_PID);
return properties.stream()
.filter(e -> e != null)
.map(e -> (PropertyIdValue) e)
.collect(Collectors.toSet());
}
return null;
}
@Override
public boolean hasSingleValue(PropertyIdValue pid) {
return getSingleConstraint(pid, SINGLE_VALUE_CONSTRAINT_QID) != null;
}
@Override
public boolean hasSingleBestValue(PropertyIdValue pid) {
return getSingleConstraint(pid, SINGLE_BEST_VALUE_CONSTRAINT_QID) != null;
}
@Override
public boolean hasDistinctValues(PropertyIdValue pid) {
return getSingleConstraint(pid, DISTINCT_VALUES_CONSTRAINT_QID) != null;
}
@Override
public boolean isSymmetric(PropertyIdValue pid) {
return getSingleConstraint(pid, SYMMETRIC_CONSTRAINT_QID) != null;
}
@Override
public Set<Value> allowedValues(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, ALLOWED_VALUES_CONSTRAINT_QID);
if (specs != null) {
List<Value> properties = findValues(specs, ALLOWED_VALUES_CONSTRAINT_PID);
return properties.stream().collect(Collectors.toSet());
}
return null;
}
@Override
public Set<Value> disallowedValues(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, DISALLOWED_VALUES_CONSTRAINT_QID);
if (specs != null) {
List<Value> properties = findValues(specs, DISALLOWED_VALUES_CONSTRAINT_PID);
return properties.stream().collect(Collectors.toSet());
}
return null;
}
@Override
public boolean boundsAllowed(PropertyIdValue pid) {
return getSingleConstraint(pid, NO_BOUNDS_CONSTRAINT_QID) == null;
}
@Override
public boolean integerValued(PropertyIdValue pid) {
return getSingleConstraint(pid, INTEGER_VALUED_CONSTRAINT_QID) != null;
}
@Override
public Set<ItemIdValue> allowedUnits(PropertyIdValue pid) {
List<SnakGroup> specs = getSingleConstraint(pid, ALLOWED_UNITS_CONSTRAINT_QID);
if (specs != null) {
List<Value> properties = findValues(specs, ALLOWED_UNITS_CONSTRAINT_PID);
return properties.stream().map(e -> e == null ? null : (ItemIdValue) e).collect(Collectors.toSet());
}
return null;
}
@Override
public boolean usableOnItems(PropertyIdValue pid) {
List<SnakGroup> constraint = getSingleConstraint(pid, ALLOWED_ENTITY_TYPES_QID);
if (constraint != null) {
return findValues(constraint, ALLOWED_ENTITY_TYPES_PID).contains(
Datamodel.makeWikidataItemIdValue(ALLOWED_ITEM_TYPE_QID));
}
return true;
}
/**
* Returns a single constraint for a particular type and a property, or null if
* there is no such constraint
*
* @param pid:
* the property to retrieve the constraints for
* @param qid:
* the type of the constraints
* @return the list of qualifiers for the constraint, or null if it does not
* exist
*/
protected List<SnakGroup> getSingleConstraint(PropertyIdValue pid, String qid) {
Statement statement = getConstraintsByType(pid, qid).findFirst().orElse(null);
if (statement != null) {
return statement.getClaim().getQualifiers();
}
return null;
}
/**
* Gets the list of constraints of a particular type for a property
*
* @param pid:
* the property to retrieve the constraints for
* @param qid:
* the type of the constraints
* @return the stream of matching constraint statements
*/
protected Stream<Statement> getConstraintsByType(PropertyIdValue pid, String qid) {
Stream<Statement> allConstraints = getConstraintStatements(pid).stream()
.filter(s -> ((EntityIdValue) s.getValue()).getId().equals(qid));
return allConstraints;
}
/**
* Gets all the constraint statements for a given property
*
* @param pid
* : the id of the property to retrieve the constraints for
* @return the list of constraint statements
*/
protected List<Statement> getConstraintStatements(PropertyIdValue pid) {
PropertyDocument doc = (PropertyDocument) EntityCache.getEntityDocument(pid);
StatementGroup group = doc.findStatementGroup(WIKIDATA_CONSTRAINT_PID);
if (group != null) {
return group.getStatements().stream()
.filter(s -> s.getValue() != null && s.getValue() instanceof EntityIdValue)
.collect(Collectors.toList());
} else {
return new ArrayList<Statement>();
}
}
/**
* Returns the values of a given property in qualifiers
*
* @param groups:
* the qualifiers
* @param pid:
* the property to filter on
* @return
*/
protected List<Value> findValues(List<SnakGroup> groups, String pid) {
List<Value> results = new ArrayList<>();
for (SnakGroup group : groups) {
if (group.getProperty().getId().equals(pid)) {
for (Snak snak : group.getSnaks())
results.add(snak.getValue());
}
}
return results;
}
}

75
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DistinctValuesScrutinizer.java Normal file
View File

@ -0,0 +1,75 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.Map;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that checks for properties using the same value on different
* items.
*
* @author Antonin Delpeuch
*
*/
public class DistinctValuesScrutinizer extends StatementScrutinizer {
public final static String type = "identical-values-for-distinct-valued-property";
private Map<PropertyIdValue, Map<Value, EntityIdValue>> _seenValues;
public DistinctValuesScrutinizer() {
_seenValues = new HashMap<>();
}
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId();
if (_fetcher.hasDistinctValues(pid)) {
Value mainSnakValue = statement.getClaim().getMainSnak().getValue();
Map<Value, EntityIdValue> seen = _seenValues.get(pid);
if (seen == null) {
seen = new HashMap<Value, EntityIdValue>();
_seenValues.put(pid, seen);
}
if (seen.containsKey(mainSnakValue)) {
EntityIdValue otherId = seen.get(mainSnakValue);
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("item1_entity", entityId);
issue.setProperty("item2_entity", otherId);
addIssue(issue);
} else {
seen.put(mainSnakValue, entityId);
}
}
}
}

128
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java Normal file
View File

@ -0,0 +1,128 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.ConstraintFetcher;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.qa.QAWarning.Severity;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.updates.ItemUpdate;
/**
* Inspects an edit batch and emits warnings.
*
* @author Antonin Delpeuch
*/
public abstract class EditScrutinizer {
protected QAWarningStore _store;
protected ConstraintFetcher _fetcher;
public EditScrutinizer() {
_fetcher = null;
_store = null;
}
public void setStore(QAWarningStore store) {
_store = store;
}
public void setFetcher(ConstraintFetcher fetcher) {
_fetcher = fetcher;
}
/**
* Called before an edit batch is scrutinized.
*/
public void batchIsBeginning() {
}
/**
* Reads the candidate edits and emits warnings in the store
*
* @param edit:
* the list of ItemUpdates to scrutinize
*/
public abstract void scrutinize(ItemUpdate edit);
/**
* Method called once the edit batch has been read entirely
*/
public void batchIsFinished() {
}
/**
* Emits an issue that will be reported to the user,
* after merging with other issues of the same kind.
*
* @param warning
* the issue to report
*/
protected void addIssue(QAWarning warning) {
_store.addWarning(warning);
}
protected void addIssue(String type, String aggregationId, Severity severity, int count) {
addIssue(new QAWarning(type, aggregationId, severity, count));
}
/**
* Helper to be used by subclasses to emit simple INFO warnings
*
* @param warning
*/
protected void info(String type) {
addIssue(type, null, QAWarning.Severity.INFO, 1);
}
/**
* Helper to be used by subclasses to emit simple warnings
*
* @param warning
*/
protected void warning(String type) {
addIssue(type, null, QAWarning.Severity.WARNING, 1);
}
/**
* Helper to be used by subclasses to emit simple important warnings
*
* @param warning
*/
protected void important(String type) {
addIssue(type, null, QAWarning.Severity.IMPORTANT, 1);
}
/**
* Helper to be used by subclasses to emit simple critical warnings
*
* @param warning
*/
protected void critical(String type) {
addIssue(type, null, QAWarning.Severity.CRITICAL, 1);
}
}

22
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EntityTypeScrutinizer.java Normal file
View File

@ -0,0 +1,22 @@
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
public class EntityTypeScrutinizer extends SnakScrutinizer {
public final static String type = "invalid-entity-type";
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
PropertyIdValue pid = snak.getPropertyId();
if(!_fetcher.usableOnItems(pid)) {
QAWarning issue = new QAWarning(type, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", entityId);
addIssue(issue);
}
}
}

100
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/FormatScrutinizer.java Normal file
View File

@ -0,0 +1,100 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
/**
* A scrutinizer that detects incorrect formats in text values (mostly
* identifiers).
*
* @author Antonin Delpeuch
*
*/
public class FormatScrutinizer extends SnakScrutinizer {
public static final String type = "add-statements-with-invalid-format";
private Map<PropertyIdValue, Pattern> _patterns;
public FormatScrutinizer() {
_patterns = new HashMap<>();
}
/**
* Loads the regex for a property and compiles it to a pattern (this is cached
* upstream, plus we are doing it only once per property and batch).
*
* @param pid
* the id of the property to fetch the constraints for
* @return
*/
protected Pattern getPattern(PropertyIdValue pid) {
if (_patterns.containsKey(pid)) {
return _patterns.get(pid);
} else {
String regex = _fetcher.getFormatRegex(pid);
Pattern pattern = null;
if (regex != null) {
pattern = Pattern.compile(regex);
}
_patterns.put(pid, pattern);
return pattern;
}
}
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
if (StringValue.class.isInstance(snak.getValue())) {
String value = ((StringValue) snak.getValue()).getString();
PropertyIdValue pid = snak.getPropertyId();
Pattern pattern = getPattern(pid);
if (pattern == null) {
return;
}
if (!pattern.matcher(value).matches()) {
if (added) {
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("regex", pattern.toString());
issue.setProperty("example_value", value);
issue.setProperty("example_item_entity", entityId);
addIssue(issue);
} else {
info("remove-statements-with-invalid-format");
}
}
}
}
}

125
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java Normal file
View File

@ -0,0 +1,125 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that checks for missing inverse statements in edit batches.
*
* @author Antonin Delpeuch
*
*/
public class InverseConstraintScrutinizer extends StatementScrutinizer {
public static final String type = "missing-inverse-statements";
private Map<PropertyIdValue, PropertyIdValue> _inverse;
private Map<PropertyIdValue, Map<EntityIdValue, Set<EntityIdValue>>> _statements;
public InverseConstraintScrutinizer() {
_inverse = new HashMap<>();
_statements = new HashMap<>();
}
protected PropertyIdValue getInverseConstraint(PropertyIdValue pid) {
if (_inverse.containsKey(pid)) {
return _inverse.get(pid);
} else {
PropertyIdValue inversePid = _fetcher.getInversePid(pid);
if (inversePid == null && _fetcher.isSymmetric(pid)) {
inversePid = pid;
}
_inverse.put(pid, inversePid);
_statements.put(pid, new HashMap<EntityIdValue, Set<EntityIdValue>>());
// We are doing this check because we do not have any guarantee that
// the inverse constraints are consistent on Wikidata.
if (inversePid != null && !_inverse.containsKey(inversePid)) {
_inverse.put(inversePid, pid);
_statements.put(inversePid, new HashMap<EntityIdValue, Set<EntityIdValue>>());
}
return inversePid;
}
}
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
if (!added) {
return; // TODO support for deleted statements
}
Value mainSnakValue = statement.getClaim().getMainSnak().getValue();
if (ItemIdValue.class.isInstance(mainSnakValue)) {
PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId();
PropertyIdValue inversePid = getInverseConstraint(pid);
if (inversePid != null) {
EntityIdValue targetEntityId = (EntityIdValue) mainSnakValue;
Set<EntityIdValue> currentValues = _statements.get(pid).get(entityId);
if (currentValues == null) {
currentValues = new HashSet<EntityIdValue>();
_statements.get(pid).put(entityId, currentValues);
}
currentValues.add(targetEntityId);
}
}
}
@Override
public void batchIsFinished() {
// For each pair of inverse properties (in each direction)
for (Entry<PropertyIdValue, PropertyIdValue> propertyPair : _inverse.entrySet()) {
// Get the statements made for the first
PropertyIdValue ourProperty = propertyPair.getKey();
for (Entry<EntityIdValue, Set<EntityIdValue>> itemLinks : _statements.get(ourProperty).entrySet()) {
// For each outgoing link
for (EntityIdValue idValue : itemLinks.getValue()) {
// Check that they are in the statements made for the second
PropertyIdValue missingProperty = propertyPair.getValue();
Set<EntityIdValue> reciprocalLinks = _statements.get(missingProperty).get(idValue);
if (reciprocalLinks == null || !reciprocalLinks.contains(itemLinks.getKey())) {
QAWarning issue = new QAWarning(type, ourProperty.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("added_property_entity", ourProperty);
issue.setProperty("inverse_property_entity", missingProperty);
issue.setProperty("source_entity", itemLinks.getKey());
issue.setProperty("target_entity", idValue);
addIssue(issue);
}
}
}
}
}
}

83
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java Normal file
View File

@ -0,0 +1,83 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
/**
* A scrutinizer that inspects new items.
*
* @author Antonin Delpeuch
*/
public class NewItemScrutinizer extends EditScrutinizer {
public static final String noLabelType = "new-item-without-labels-or-aliases";
public static final String noDescType = "new-item-without-descriptions";
public static final String deletedStatementsType = "new-item-with-deleted-statements";
public static final String noTypeType = "new-item-without-P31-or-P279";
public static final String newItemType = "new-item-created";
@Override
public void scrutinize(ItemUpdate update) {
if (update.isNew()) {
info(newItemType);
if (update.getLabels().isEmpty() && update.getAliases().isEmpty()) {
QAWarning issue = new QAWarning(noLabelType, null, QAWarning.Severity.CRITICAL, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
if (update.getDescriptions().isEmpty()) {
QAWarning issue = new QAWarning(noDescType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
if (!update.getDeletedStatements().isEmpty()) {
QAWarning issue = new QAWarning(deletedStatementsType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
// Try to find a "instance of" or "subclass of" claim
boolean typeFound = false;
for (StatementGroup group : update.getAddedStatementGroups()) {
String pid = group.getProperty().getId();
if ("P31".equals(pid) || "P279".equals(pid)) {
typeFound = true;
break;
}
}
if (!typeFound) {
QAWarning issue = new QAWarning(noTypeType, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
}
}
}
}

51
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java Normal file
View File

@ -0,0 +1,51 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.updates.ItemUpdate;
public class NoEditsMadeScrutinizer extends EditScrutinizer {
public static final String type = "no-edit-generated";
private boolean nonNullUpdateSeen = false;
@Override
public void batchIsBeginning() {
nonNullUpdateSeen = false;
}
@Override
public void scrutinize(ItemUpdate edit) {
nonNullUpdateSeen = true;
}
@Override
public void batchIsFinished() {
if(!nonNullUpdateSeen) {
info(type);
}
}
}

110
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QualifierCompatibilityScrutinizer.java Normal file
View File

@ -0,0 +1,110 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer that checks the compatibility of the qualifiers and the
* property of a statement, and looks for mandatory qualifiers.
*
* @author Antonin Delpeuch
*/
public class QualifierCompatibilityScrutinizer extends StatementScrutinizer {
public static final String missingMandatoryQualifiersType = "missing-mandatory-qualifiers";
public static final String disallowedQualifiersType = "disallowed-qualifiers";
private Map<PropertyIdValue, Set<PropertyIdValue>> _allowedQualifiers;
private Map<PropertyIdValue, Set<PropertyIdValue>> _mandatoryQualifiers;
public QualifierCompatibilityScrutinizer() {
_allowedQualifiers = new HashMap<>();
_mandatoryQualifiers = new HashMap<>();
}
protected boolean qualifierIsAllowed(PropertyIdValue statementProperty, PropertyIdValue qualifierProperty) {
Set<PropertyIdValue> allowed = null;
if (_allowedQualifiers.containsKey(statementProperty)) {
allowed = _allowedQualifiers.get(statementProperty);
} else {
allowed = _fetcher.allowedQualifiers(statementProperty);
_allowedQualifiers.put(statementProperty, allowed);
}
return allowed == null || allowed.contains(qualifierProperty);
}
protected Set<PropertyIdValue> mandatoryQualifiers(PropertyIdValue statementProperty) {
Set<PropertyIdValue> mandatory = null;
if (_mandatoryQualifiers.containsKey(statementProperty)) {
mandatory = _mandatoryQualifiers.get(statementProperty);
} else {
mandatory = _fetcher.mandatoryQualifiers(statementProperty);
if (mandatory == null) {
mandatory = new HashSet<>();
}
_mandatoryQualifiers.put(statementProperty, mandatory);
}
return mandatory;
}
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
PropertyIdValue statementProperty = statement.getClaim().getMainSnak().getPropertyId();
Set<PropertyIdValue> qualifiers = statement.getClaim().getQualifiers().stream().map(e -> e.getProperty())
.collect(Collectors.toSet());
Set<PropertyIdValue> missingQualifiers = mandatoryQualifiers(statementProperty).stream()
.filter(p -> !qualifiers.contains(p)).collect(Collectors.toSet());
Set<PropertyIdValue> disallowedQualifiers = qualifiers.stream()
.filter(p -> !qualifierIsAllowed(statementProperty, p)).collect(Collectors.toSet());
for (PropertyIdValue missing : missingQualifiers) {
QAWarning issue = new QAWarning(missingMandatoryQualifiersType,
statementProperty.getId() + "-" + missing.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("statement_property_entity", statementProperty);
issue.setProperty("missing_property_entity", missing);
issue.setProperty("example_item_entity", entityId);
addIssue(issue);
}
for (PropertyIdValue disallowed : disallowedQualifiers) {
QAWarning issue = new QAWarning(disallowedQualifiersType,
statementProperty.getId() + "-" + disallowed.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("statement_property_entity", statementProperty);
issue.setProperty("disallowed_property_entity", disallowed);
issue.setProperty("example_item_entity", entityId);
addIssue(issue);
}
}
}

70
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java Normal file
View File

@ -0,0 +1,70 @@
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.Set;
import java.util.stream.Collectors;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
/**
* Scrutinizer checking for units and bounds in quantities.
*
* @author Antonin Delpeuch
*
*/
public class QuantityScrutinizer extends SnakScrutinizer {
public static final String boundsDisallowedType = "bounds-disallowed";
public static final String integerConstraintType = "values-should-be-integers";
public static final String invalidUnitType = "invalid-unit";
public static final String noUnitProvidedType = "no-unit-provided";
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
if (QuantityValue.class.isInstance(snak.getValue()) && added) {
PropertyIdValue pid = snak.getPropertyId();
QuantityValue value = (QuantityValue)snak.getValue();
if(!_fetcher.boundsAllowed(pid) && (value.getUpperBound() != null || value.getLowerBound() != null)) {
QAWarning issue = new QAWarning(boundsDisallowedType, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("example_value", value.getNumericValue().toString());
issue.setProperty("example_item_entity", entityId);
addIssue(issue);
}
if(_fetcher.integerValued(pid) && value.getNumericValue().scale() > 0) {
QAWarning issue = new QAWarning(integerConstraintType, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("example_value", value.getNumericValue().toString());
issue.setProperty("example_item_entity", entityId);
addIssue(issue);
}
Set<ItemIdValue> allowedUnits = _fetcher.allowedUnits(pid);
String currentUnit = null;
if (value.getUnit() != null && !value.getUnit().equals("")) {
currentUnit = value.getUnit();
}
if(allowedUnits != null &&
!allowedUnits.stream().map(u -> u != null ? u.getIri() : null)
.collect(Collectors.toSet()).contains(currentUnit)) {
String issueType = currentUnit == null ? noUnitProvidedType : invalidUnitType;
QAWarning issue = new QAWarning(issueType, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("example_value", value.getNumericValue().toString());
issue.setProperty("example_item_entity", entityId);
if (currentUnit != null) {
issue.setProperty("unit_entity",
// this is a hack but it will not be needed anymore in the upcoming version of Wikidata-Toolkit
Datamodel.makeWikidataItemIdValue(currentUnit.substring(currentUnit.indexOf("Q"))));
}
addIssue(issue);
}
}
}
}

85
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedPositionScrutinizer.java Normal file
View File

@ -0,0 +1,85 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.Iterator;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
public class RestrictedPositionScrutinizer extends StatementScrutinizer {
protected enum SnakPosition {
MAINSNAK, QUALIFIER, REFERENCE
}
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
// Skip the main snak
scrutinize(statement.getClaim().getMainSnak(), entityId, SnakPosition.MAINSNAK, added);
// Qualifiers
scrutinizeSnakSet(statement.getClaim().getAllQualifiers(), entityId, SnakPosition.QUALIFIER, added);
// References
for (Reference ref : statement.getReferences()) {
scrutinizeSnakSet(ref.getAllSnaks(), entityId, SnakPosition.REFERENCE, added);
}
}
protected void scrutinizeSnakSet(Iterator<Snak> snaks, EntityIdValue entityId, SnakPosition position,
boolean added) {
while (snaks.hasNext()) {
Snak snak = snaks.next();
scrutinize(snak, entityId, position, added);
}
}
public void scrutinize(Snak snak, EntityIdValue entityId, SnakPosition position, boolean added) {
if (!positionAllowed(snak.getPropertyId(), position)) {
String positionStr = position.toString().toLowerCase();
QAWarning issue = new QAWarning("property-found-in-" + positionStr,
snak.getPropertyId().getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", snak.getPropertyId());
addIssue(issue);
}
}
public boolean positionAllowed(PropertyIdValue pid, SnakPosition position) {
if(position.equals(SnakPosition.MAINSNAK)) {
return _fetcher.allowedAsValue(pid);
} else if(position.equals(SnakPosition.QUALIFIER)) {
return _fetcher.allowedAsQualifier(pid);
} else if(position.equals(SnakPosition.REFERENCE)) {
return _fetcher.allowedAsReference(pid);
}
return true;
}
}

32
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizer.java Normal file
View File

@ -0,0 +1,32 @@
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.Set;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Value;
public class RestrictedValuesScrutinizer extends SnakScrutinizer {
public static String type = "forbidden-value";
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
PropertyIdValue pid = snak.getPropertyId();
Value value = snak.getValue();
Set<Value> allowedValues = _fetcher.allowedValues(pid);
Set<Value> disallowedValues = _fetcher.disallowedValues(pid);
if((allowedValues != null && !allowedValues.contains(value)) ||
(disallowedValues != null && disallowedValues.contains(value))) {
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("example_value_entity", value);
issue.setProperty("example_subject_entity", entityId);
addIssue(issue);
}
}
}

50
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SelfReferentialScrutinizer.java Normal file
View File

@ -0,0 +1,50 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
/**
* A scrutinizer that checks for self-referential statements. These statements
* are flagged by Wikibase as suspicious.
*
* @author Antonin Delpeuch
*
*/
public class SelfReferentialScrutinizer extends SnakScrutinizer {
public static final String type = "self-referential-statements";
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
if (entityId.equals(snak.getValue())) {
QAWarning issue = new QAWarning(type, null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_entity", entityId);
addIssue(issue);
}
}
}

66
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java Normal file
View File

@ -0,0 +1,66 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashSet;
import java.util.Set;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* For now this scrutinizer only checks for uniqueness at the item level (it
* ignores qualifiers and references).
*
* Given that all ranks are currently set to Normal, this also checks for
* single best values.
*
* @author Antonin Delpeuch
*
*/
public class SingleValueScrutinizer extends EditScrutinizer {
public static final String type = "single-valued-property-added-more-than-once";
@Override
public void scrutinize(ItemUpdate update) {
Set<PropertyIdValue> seenSingleProperties = new HashSet<>();
for (Statement statement : update.getAddedStatements()) {
PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId();
if (seenSingleProperties.contains(pid)) {
QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.WARNING, 1);
issue.setProperty("property_entity", pid);
issue.setProperty("example_entity", update.getItemId());
addIssue(issue);
} else if (_fetcher.hasSingleValue(pid) || _fetcher.hasSingleBestValue(pid)) {
seenSingleProperties.add(pid);
}
}
}
}

74
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SnakScrutinizer.java Normal file
View File

@ -0,0 +1,74 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.Iterator;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer that inspects snaks individually, no matter whether they appear
* as main snaks, qualifiers or references.
*
* @author Antonin Delpeuch
*
*/
public abstract class SnakScrutinizer extends StatementScrutinizer {
/**
* This is the method that subclasses should override to implement their checks.
*
* @param snak:
* the snak to inspect
* @param entityId:
* the item on which it is going to (dis)appear
* @param added:
* whether this snak is going to be added or deleted
*/
public abstract void scrutinize(Snak snak, EntityIdValue entityId, boolean added);
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
// Main snak
scrutinize(statement.getClaim().getMainSnak(), entityId, added);
// Qualifiers
scrutinizeSnakSet(statement.getClaim().getAllQualifiers(), entityId, added);
// References
for (Reference ref : statement.getReferences()) {
scrutinizeSnakSet(ref.getAllSnaks(), entityId, added);
}
}
protected void scrutinizeSnakSet(Iterator<Snak> snaks, EntityIdValue entityId, boolean added) {
while (snaks.hasNext()) {
Snak snak = snaks.next();
scrutinize(snak, entityId, added);
}
}
}

55
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java Normal file
View File

@ -0,0 +1,55 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
public abstract class StatementScrutinizer extends EditScrutinizer {
@Override
public void scrutinize(ItemUpdate update) {
EntityIdValue currentEntityId = update.getItemId();
for (Statement statement : update.getAddedStatements()) {
scrutinize(statement, currentEntityId, true);
}
for (Statement statement : update.getDeletedStatements()) {
scrutinize(statement, currentEntityId, false);
}
}
/**
* The method that should be overridden by subclasses, implementing the checks
* on one statement
*
* @param statement:
* the statement to scrutinize
* @param entityId:
* the id of the entity on which this statement is made or removed
* @param added:
* whether this statement was added or deleted
*/
public abstract void scrutinize(Statement statement, EntityIdValue entityId, boolean added);
}

46
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/UnsourcedScrutinizer.java Normal file
View File

@ -0,0 +1,46 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* A scrutinizer checking for unsourced statements
*
* @author Antonin Delpeuch
*
*/
public class UnsourcedScrutinizer extends StatementScrutinizer {
public static final String type = "unsourced-statements";
@Override
public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
if (statement.getReferences().isEmpty() && added) {
warning(type);
}
}
}

62
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java Normal file
View File

@ -0,0 +1,62 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* A scrutinizer that inspects the values of snaks and terms
*
* @author Antonin Delpeuch
*
*/
public abstract class ValueScrutinizer extends SnakScrutinizer {
@Override
public void scrutinize(ItemUpdate update) {
super.scrutinize(update);
for (MonolingualTextValue label : update.getLabels()) {
scrutinize(label);
}
for (MonolingualTextValue alias : update.getAliases()) {
scrutinize(alias);
}
for (MonolingualTextValue description : update.getDescriptions()) {
scrutinize(description);
}
}
public abstract void scrutinize(Value value);
@Override
public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) {
scrutinize(snak.getValue());
}
}

85
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/WhitespaceScrutinizer.java Normal file
View File

@ -0,0 +1,85 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import org.openrefine.wikidata.qa.QAWarning;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
/**
* Scrutinizes strings for trailing / leading whitespace, and others
*
* @author Antonin Delpeuch
*
*/
public class WhitespaceScrutinizer extends ValueScrutinizer {
private Map<String, Pattern> _issuesMap;
public static final String leadingWhitespaceType = "leading-whitespace";
public static final String trailingWhitespaceType = "trailing-whitespace";
public static final String duplicateWhitespaceType = "duplicate-whitespace";
public static final String nonPrintableCharsType = "non-printable-characters";
public WhitespaceScrutinizer() {
_issuesMap = new HashMap<>();
_issuesMap.put(leadingWhitespaceType, Pattern.compile("^\\s"));
_issuesMap.put(trailingWhitespaceType, Pattern.compile("\\s$"));
_issuesMap.put(duplicateWhitespaceType, Pattern.compile("\\s\\s"));
// https://stackoverflow.com/questions/14565934/regular-expression-to-remove-all-non-printable-characters
_issuesMap.put(nonPrintableCharsType, Pattern.compile("[\\x00\\x03\\x08\\x0B\\x0C\\x0E-\\x1F]"));
}
@Override
public void scrutinize(Value value) {
String str = null;
if (MonolingualTextValue.class.isInstance(value)) {
str = ((MonolingualTextValue) value).getText();
} else if (StringValue.class.isInstance(value)) {
str = ((StringValue) value).getString();
}
if (str != null) {
for (Entry<String, Pattern> entry : _issuesMap.entrySet()) {
if (entry.getValue().matcher(str).find()) {
emitWarning(entry.getKey(), str);
}
}
}
}
private void emitWarning(String type, String example) {
QAWarning warning = new QAWarning(type, null, QAWarning.Severity.WARNING, 1);
warning.setProperty("example_string", example);
addIssue(warning);
}
}

107
extensions/wikidata/src/org/openrefine/wikidata/schema/ExpressionContext.java Normal file
View File

@ -0,0 +1,107 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import org.apache.commons.lang.Validate;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.qa.QAWarningStore;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ColumnModel;
import com.google.refine.model.Row;
/**
* A class holding all the necessary information about the context in which a
* schema expression is evaluated.
*
* @author Antonin Delpeuch
*
*/
public class ExpressionContext {
private String baseIRI;
private int rowId;
private Row row;
private ColumnModel columnModel;
private QAWarningStore warningStore;
/**
* Builds an expression context to evaluate a schema on a row
*
* @param baseIRI
* the siteIRI of the schema
* @param rowId
* the id of the row currently visited
* @param row
* the row itself
* @param columnModel
* lets us access cells by column name
* @param warningStore
* where to store the issues encountered when evaluating (can be set
* to null if these issues should be ignored)
*/
public ExpressionContext(String baseIRI, int rowId, Row row, ColumnModel columnModel, QAWarningStore warningStore) {
Validate.notNull(baseIRI);
this.baseIRI = baseIRI;
this.rowId = rowId;
Validate.notNull(row);
this.row = row;
Validate.notNull(columnModel);
this.columnModel = columnModel;
this.warningStore = warningStore;
}
public String getBaseIRI() {
return baseIRI;
}
/**
* Retrieves a cell in the current row, by column name. If the column does not
* exist, null is returned.
*
* @param name
* the name of the column to retrieve the cell from
* @return the cell
*/
public Cell getCellByName(String name) {
Column column = columnModel.getColumnByName(name);
if (column != null) {
int idx = column.getCellIndex();
return row.getCell(idx);
} else {
return null;
}
}
public int getRowId() {
return rowId;
}
public void addWarning(QAWarning warning) {
if (warningStore != null) {
warningStore.addWarning(warning);
}
}
}

158
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java Normal file
View File

@ -0,0 +1,158 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
import java.util.Map.Entry;
import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableMap;
/**
* A constant for a time value, accepting a number of formats which determine
* the precision of the parsed value.
*
* @author Antonin Delpeuch
*
*/
public class WbDateConstant implements WbExpression<TimeValue> {
/**
* Map of formats accepted by the parser. Each format is associated to the time
* precision it induces (an integer according to Wikibase's data model).
*/
public static Map<SimpleDateFormat, Integer> acceptedFormats = ImmutableMap.<SimpleDateFormat, Integer> builder()
.put(new SimpleDateFormat("yyyy"), 9)
.put(new SimpleDateFormat("yyyy-MM"), 10)
.put(new SimpleDateFormat("yyyy-MM-dd"), 11)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"), 13)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"), 13)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), 14).build();
private TimeValue parsed;
private String origDatestamp;
/**
* Constructor. Used for deserialization from JSON. The object will be
* constructed even if the time cannot be parsed (it will evaluate to null) in
* {@link evaluate}.
*
* @param origDatestamp
* the date value as a string
*/
@JsonCreator
public WbDateConstant(@JsonProperty("value") String origDatestamp) {
Validate.notNull(origDatestamp);
this.setOrigDatestamp(origDatestamp);
}
@Override
public TimeValue evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException {
return parsed;
}
/**
* Parses a timestamp into a Wikibase {@link TimeValue}. The precision is
* automatically inferred from the format.
*
* @param datestamp
* the time to parse
* @return
* @throws ParseException
* if the time cannot be parsed
*/
public static TimeValue parse(String datestamp)
throws ParseException {
Date date = null;
int precision = 9; // default precision (will be overridden)
for (Entry<SimpleDateFormat, Integer> entry : acceptedFormats.entrySet()) {
ParsePosition position = new ParsePosition(0);
String trimmedDatestamp = datestamp.trim();
date = entry.getKey().parse(trimmedDatestamp, position);
// Ignore parses which failed or do not consume all the input
if (date != null && position.getIndex() == trimmedDatestamp.length()) {
precision = entry.getValue();
break;
}
}
if (date == null) {
throw new ParseException("Invalid date.", 0);
} else {
Calendar calendar = Calendar.getInstance();
calendar = Calendar.getInstance();
calendar.setTime(date);
return Datamodel.makeTimeValue(calendar.get(Calendar.YEAR), (byte) (calendar.get(Calendar.MONTH) + 1),
(byte) calendar.get(Calendar.DAY_OF_MONTH), (byte) calendar.get(Calendar.HOUR_OF_DAY),
(byte) calendar.get(Calendar.MINUTE), (byte) calendar.get(Calendar.SECOND), (byte) precision, 0, 0,
0, TimeValue.CM_GREGORIAN_PRO);
}
}
/**
* @return the original datestamp
*/
@JsonProperty("value")
public String getOrigDatestamp() {
return origDatestamp;
}
private void setOrigDatestamp(String origDatestamp) {
this.origDatestamp = origDatestamp;
try {
this.parsed = parse(origDatestamp);
} catch (ParseException e) {
throw new IllegalArgumentException("Invalid datestamp provided: " + origDatestamp);
}
}
@Override
public boolean equals(Object other) {
if (other == null || !WbDateConstant.class.isInstance(other)) {
return false;
}
WbDateConstant otherConstant = (WbDateConstant) other;
return origDatestamp.equals(otherConstant.getOrigDatestamp());
}
@Override
public int hashCode() {
return origDatestamp.hashCode();
}
}

74
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java Normal file
View File

@ -0,0 +1,74 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.text.ParseException;
import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.refine.model.Cell;
/**
* An expression that represents a time value, extracted from a string. A number
* of formats are recognized, see {@link WbDateConstant} for details.
*
* @author Antonin Delpeuch
*
*/
public class WbDateVariable extends WbVariableExpr<TimeValue> {
@JsonCreator
public WbDateVariable() {
}
public WbDateVariable(String columnName) {
setColumnName(columnName);
}
@Override
public TimeValue fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException {
try {
// TODO accept parsed dates (without converting them to strings)
return WbDateConstant.parse(cell.value.toString());
} catch (ParseException e) {
if(!cell.value.toString().isEmpty()) {
QAWarning issue = new QAWarning("ignored-date", null, QAWarning.Severity.WARNING, 1);
issue.setProperty("example_value", cell.value.toString());
ctxt.addWarning(issue);
}
throw new SkipSchemaExpressionException();
}
}
@Override
public boolean equals(Object other) {
return equalAsVariables(other, WbDateVariable.class);
}
}

Some files were not shown because too many files have changed in this diff Show More