diff --git a/MUOR/MUOR/configs/muor.conf b/MUOR/MUOR/configs/muor.conf index 4b1d858d9..7b08b6c82 100644 --- a/MUOR/MUOR/configs/muor.conf +++ b/MUOR/MUOR/configs/muor.conf @@ -44,6 +44,10 @@ server { proxy_pass http://django; } + location /session-refresh/ { + proxy_pass http://django; + } + location / { proxy_pass http://django; diff --git a/MUOR/MUOR/configs/muor_before_location_block.conf b/MUOR/MUOR/configs/muor_before_location_block.conf index 5acda5d3c..74c2f184f 100644 --- a/MUOR/MUOR/configs/muor_before_location_block.conf +++ b/MUOR/MUOR/configs/muor_before_location_block.conf @@ -44,6 +44,10 @@ server { proxy_pass http://django; } + location /session-refresh/ { + proxy_pass http://django; + } + location / { proxy_pass http://django; diff --git a/MUOR/templates/session_refresh.html b/MUOR/templates/session_refresh.html index 2e9d4488f..f9cb2ab4a 100644 --- a/MUOR/templates/session_refresh.html +++ b/MUOR/templates/session_refresh.html @@ -1,7 +1,7 @@ - + session_refresh diff --git a/OpenRefine/.gitattributes b/OpenRefine/.gitattributes deleted file mode 100644 index 7057ad030..000000000 --- a/OpenRefine/.gitattributes +++ /dev/null @@ -1,22 +0,0 @@ -# Auto detect text files and perform LF normalization -* text=off - -# Custom for Visual Studio -*.cs diff=csharp -*.sln merge=union -*.csproj merge=union -*.vbproj merge=union -*.fsproj merge=union -*.dbproj merge=union - -# Standard to msysgit -*.doc diff=astextplain -*.DOC diff=astextplain -*.docx diff=astextplain -*.DOCX diff=astextplain -*.dot diff=astextplain -*.DOT diff=astextplain -*.pdf diff=astextplain -*.PDF diff=astextplain -*.rtf diff=astextplain -*.RTF diff=astextplain diff --git a/OpenRefine/.gitignore b/OpenRefine/.gitignore deleted file mode 100644 index 3cf04d797..000000000 --- a/OpenRefine/.gitignore +++ /dev/null @@ -1,72 +0,0 @@ -.idea/ -*.log -logs -*~ -\#*# -.*.swp -*.DS_Store -*.class -*.patch -.com.apple.timemachine.supported -.import-temp/ -build/ -dist/ -server/classes/ -main/webapp/WEB-INF/classes/ -main/tests/server/classes/ -main/test-output/ -appengine/classes/ -tools/ -broker/appengine/module/MOD-INF/classes/ -broker/core/module/MOD-INF/classes/ -broker/core/WEB-INF/lib/ -broker/core/data/ -broker/core/test-output/ -tmp/ -/test-output -test-out/ -/bin -open-refine.log -.vscode -.metadata # Eclipse plugins specific - -# Locally stored "Eclipse launch configurations" -*.launch - -.idea -*.iml - -main/target/ -main/webapp/WEB-INF/lib/ -server/target/ -extensions/*/target/ -extensions/*/module/MOD-INF/classes/ -extensions/*/module/MOD-INF/lib/ -extensions/target -target/ -**/test-output/* -*.versionsBackup - -# Ignore classpath as it is automatically generated by Maven integrations in IDEs -*.classpath -*.project -*.settings - -# Ignore Apache Maven default download path -apache-maven-*/ -apache-maven-*-bin.tar.gz - -# Ignore refine-dev.ini -refine-dev.ini - -# Java annotation processor (APT) -.factorypath - -# Code Recommenders -.recommenders/ - -# STS (Spring Tool Suite) -.springBeans - -# Ignore Node modules that might inadvertently install at current path instead of user or configured env var NODE_PATH -node_modules/ diff --git a/OpenRefine/CODE_OF_CONDUCT.md b/OpenRefine/CODE_OF_CONDUCT.md deleted file mode 100644 index d7ffc7432..000000000 --- a/OpenRefine/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,82 +0,0 @@ -# OpenRefine Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as OpenRefine -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, sex characteristics, gender identity and expression, -level of experience, education, socio-economic status, nationality, personal -appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies within all project spaces, and it also applies when -an individual is representing the project or its community in public spaces. -Examples of representing a project or community include using an official -project e-mail address, posting via an official social media account, or acting -as an appointed representative at an online or offline event. Representation of -a project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the Code of Conduct Committee at code-of-conduct@openrefine.org. -The committee consists of: -* Lozanna Rossenova -* Chris Erdmann -* Jessica Hardwicke - -All complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The code of conduct is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq - diff --git a/OpenRefine/CONTRIBUTING.md b/OpenRefine/CONTRIBUTING.md deleted file mode 100644 index f463341eb..000000000 --- a/OpenRefine/CONTRIBUTING.md +++ /dev/null @@ -1,62 +0,0 @@ -This document presents how you can contribute to the OpenRefine project. Please also review our [Governance model](https://github.com/OpenRefine/OpenRefine/blob/master/GOVERNANCE.md) - -## Documentation, Questions or Problem - -Our issue list is only for reporting specific bugs and requesting specific features. If you just don't know how to do something using OpenRefine, or want to discuss some ideas, please -- try the [documentation wiki](https://github.com/OpenRefine/OpenRefine/wiki/Documentation-For-Users) -- ask on the [OpenRefine mailing list](https://groups.google.com/d/forum/openrefine). - -If you really want to file a bug or request a feature, go to this [issue list](https://github.com/OpenRefine/OpenRefine/issues). Please use the search function first to make sure a similar issue doesn't already exist. - -## Promote OpenRefine - -You don't need to be a coder to contribute to OpenRefine. Did you write a tutorial or article about OpenRefine on your blog or site? Are you organizing a workshop or presentation for OpenRefine in your city? Let us know via our [user discussion list](https://groups.google.com/d/forum/openrefine) or Twitter account ([@OpenRefine](http://twitter.com/OpenRefine)). We will share the news via our monthly update and via our Twitter handle. - -## Contributing translations - -You can help us [translate OpenRefine](https://github.com/OpenRefine/OpenRefine/wiki/Translate-OpenRefine) in as many languages as possible [via Weblate](https://hosted.weblate.org/engage/openrefine/?utm_source=widget). - -## Contributing code - -You can contribute code in three different ways: -- Fix minor bugs - you can check the issues flagged as [help wanted](https://github.com/OpenRefine/OpenRefine/labels/help%20wanted) or [good first issue](https://github.com/OpenRefine/OpenRefine/labels/good%20first%20issue) or [good second issue](https://github.com/OpenRefine/OpenRefine/labels/good%20second%20issue) -- Develop an OpenRefine extension -- Start your own distribution or fork - -All developers including new distributions and plugin developers are invited to leverage the following OpenRefine project management areas to avoid splitting the community in different communication channels. -- the [wiki](https://github.com/OpenRefine/OpenRefine/wiki) for shared documentation between both user docs and [documentation for developer](https://github.com/OpenRefine/OpenRefine/wiki/Documentation-For-Developers) -- the [developer mailing list](https://groups.google.com/forum/?fromgroups#!forum/openrefine-dev) for technical questions, new feature development and anything code related. We invite you to share you idea first via the developer mailing list. Someone may be able to point out to existing development saving you hours of research and development. -- [OpenRefine github issue tracker](https://github.com/OpenRefine/OpenRefine/issues) for requesting new features and bug reports. -- [Gitter Chat](https://gitter.im/OpenRefine/OpenRefine) - -### How to submit PR's (pull requests), patches, and bug fixes - -- Read [Your first pull request](https://github.com/OpenRefine/OpenRefine/wiki/Your-first-pull-request) -- Avoid merging master in your branch because it makes code review a lot harder. -- If you want to keep your branch up to date with our master, it would be nicer if you could just rebase your branch instead. That would keep the history a lot cleaner. -- Please avoid adding unrelated changes in the PR. Do a separate PR and rebase once they get merged can work really well. -- It is important that pull requests are used systematically, even by those who have the rights to merge them. - -If you make trivial changes, you can send them directly via a pull request. **Please make your changes in a new git branch and send your patch**, including appropriate test cases. - -We want to keep the quality of the trunk at a very high level, since this is ultimately where the Stable Releases are built from after bugs are fixed. Please take the time to test your changes (including travis-ci) before sending a pull request. - -OpenRefine is volunteer supported. Pull Requests are reviewed and merged by volunteers. All Pull Requests will be answered, however it may take some time to get back to you. Thank you in advance for your patience. - -If you don't know where to start and are looking for a bug to fix, please see our [issue list](https://github.com/OpenRefine/OpenRefine/issues). - -### New functionalities via extensions - -OpenRefine support a plugin architecture to extend its functionality. You can find more information on how to write extension on [our wiki](https://github.com/OpenRefine/OpenRefine/wiki/Write-An-Extension). Giuliano Tortoreto wrote a separate documentation detailing how to build an extension for OpenRefine. A [LaTeX](https://github.com/OpenRefine/OpenRefineExtensionDoc) and [PDF version](https://github.com/OpenRefine/OpenRefineExtensionDoc/blob/master/main.pdf) are available. - -If you want to list your extension on the download page, please edit [this file](https://github.com/OpenRefine/openrefine.github.com/blob/master/download.md). - -### New distributions - -OpenRefine is already available in many different distributions (see the [download page](http://openrefine.org/download.html)). New distributions often package OpenRefine for a specific usage or port it. We are fine with new forks ([see discussion](https://groups.google.com/forum/#!msg/openrefine/pasNnMDJ3p8/LrZz_GiFCwAJ)) but we invite you to engage with the community to share your roadmap and progress. - -Github offers a powerful system to work between different repositories and we encourage you to leverage it: -- You can cross reference issues and pull requests between Github repository using `user/repository#number` ([see more here](https://github.com/blog/967-github-secrets#cross-repository-issue-references)) -- If you want to merge a Pull Request that is pending for review to your own repository check the pull request locally ([see more here](https://help.github.com/articles/checking-out-pull-requests-locally/)). - -Don't forget to contribute to the upstream ([main OpenRefine repository](https://github.com/openrefine/openrefine.git)) so your changes from your distribution can be reviewed and merged and to keep other developers aware of your progress. If you want to list your distribution on the download page, please edit [this file](https://github.com/OpenRefine/openrefine.github.com/blob/master/download.md). diff --git a/OpenRefine/Dockerfile b/OpenRefine/Dockerfile deleted file mode 100644 index d95fe8661..000000000 --- a/OpenRefine/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM maven:3-jdk-11 as build - -WORKDIR /usr/src/app -COPY . OpenRefine -WORKDIR /usr/src/app/OpenRefine - -RUN ./refine clean -RUN ./refine build - -FROM openjdk:11 - -WORKDIR /usr/app -COPY --from=build /usr/src/app/OpenRefine . - -VOLUME /data -EXPOSE 3333 - -ENTRYPOINT ["/usr/app/refine"] -CMD ["-i", "0.0.0.0", "-d", "/data", "-m", "3G"] diff --git a/OpenRefine/GOVERNANCE.md b/OpenRefine/GOVERNANCE.md deleted file mode 100644 index d8d3a0b5f..000000000 --- a/OpenRefine/GOVERNANCE.md +++ /dev/null @@ -1,128 +0,0 @@ -# OpenRefine Governance Model - -## Summary / Overview -OpenRefine is a free, open-source, powerful tool for working with messy data. OpenRefine has a plugin architecture and is distributed under the [new BSD license](http://opensource.org/licenses/BSD-3-Clause) allowing modification, distribution and name changes. - -## Roles and responsibilities -OpenRefine development is based on user consensus and open discussion between users. Decision making must be done in a transparent, open fashion (ie. using discussion list and issue list). No decisions about the project’s direction, bug fixes or features may be done in private without community involvement and participation. Discussions must begin at the earliest possible point on a topic; the community’s participation is vital during the entire decision-making process. - -All project participants abide by the [Code of Conduct](https://github.com/OpenRefine/OpenRefine/blob/master/CODE_OF_CONDUCT.md). - -Anyone with an interest in the project can join the community, contribute to the project design, and participate in the decision making process. This document describes how that participation takes place. - -### Users -Users are community members who have a need for the project. Through their usage, they give the project a purpose. Users are encouraged to participate in the project life by providing feedback on how their needs are satisfied. - -Users can help the project by: - -- Advertising and advocating for the project -- Informing developers of strengths and weaknesses of the tool through the [user discussion list](https://groups.google.com/forum/?fromgroups#!forum/openrefine) or the [issue list](https://github.com/OpenRefine/OpenRefine/issues?state=open) -- Providing moral support (a ‘thank you’ goes a long way) -- Writing tutorials - -How to become an OpenRefine user? [Download OpenRefine](http://openrefine.org/download.html) and start refining! - -### Contributors -Contributors are users getting involved in the project more closely. Contributions can take many forms: - -- Supporting new users via the [user discussion list](https://groups.google.com/forum/?fromgroups#!forum/openrefine) -- Submitting patches to fix bugs or add features via pull requests -- Maintaining and improving the [website](https://openrefine.org/) -- Writing and maintaining the [documentation](https://github.com/OpenRefine/OpenRefine/tree/master/docs) - -How to become an OpenRefine contributor? You will find more details in our [contributing guideline](https://github.com/OpenRefine/OpenRefine/blob/master/CONTRIBUTING.md) - -### Committers -If you make regular contribution to OpenRefine, you will most likely become a Committer. - -Committers have earned enough trust from the community to review and merge pull requests. - -Therefore Committers: -- Help contributors via the [developer discussion list](https://groups.google.com/forum/?fromgroups#!forum/openrefine-dev). -- Triage issues, pull requests and [projects](https://github.com/OpenRefine/OpenRefine/projects) -- Have direct access to the code base to create new branches -- Organize the wiki -- Review and merge pull requests submitted by contributors -- Are part of the OpenRefine organization and have the OpenRefine badge on their GitHub profile - -#### How to become a Committer? - -Be a contributor and be nominated as a Committer. Current Committer selects and elects new Committer. You may nominate yourself. Nomination should be sent to the [developer discussion list](https://groups.google.com/forum/?fromgroups#!forum/openrefine-dev) - -#### Current list of Committers -The list is available here: https://github.com/orgs/OpenRefine/people. - -### Release Manager -The Release Manager is responsible to -* Coordinate with the community to select which issue and pull request are part of the release -* Prepare and coordinate publishing the release - -The current Release Manager is Antonin Delpeuch - -### Steering Committee -The steering committee oversees the general direction of the project and initates links with other organizations and projects. - -* Advise the Project’s staff on processes, strategy, and operations; -* Participate in decision making and/or review of roadmaps, as time allows; -* Participate to some Steering Committee meetings, when time allows; -* Help the Project build connections and partnerships by helping project leadership to steward relationships with funders and partners, making strategic introductions for project leadership, reviewing documents when needed, as time allows; -* Act as an advocate for the Project in events and support the project’s communication online, as time allows. - -#### How to be part of the Steering Committee -Steering Committee are invited by the Advisory Committee - -#### Current list of Steering Committee members -* Chris Erdmann (University of North Carolina, RENCI) -* Alicia Fagerving (Wikimedia Sverige) -* Sandra Fauconnier (Wikimedia Foundation) -* Rufus Pollock (Datopian, Open Knowledge Foundation) -* Simon Rogers (Google News Initiative) -* Lozana Rossenova (Rhizome) -* Juliane Schneider (Harvard Catalyst, Clinical and Translational Science Center) -* Wesley Sherperd (Unifyd Insights) -* Fabio Tacchelli (Siren Solutions) - -### Advisory Committee -The Advisory Committee runs the adminstrative aspect of the project on a day to day basis with the support of Code for Science and Society (CS&S). Its member are bound by the fiscal sponsorship agreement with Code for Science and Society. They meet once per month with Code for Science and Society. - -It is composed of at least three members. No more than 49 percent of the members of the Advisory Committee may be paid by the project. - -Advisory Committee members: -* Represent OpenRefine with Code for Science and Society -* Provide guidance and oversight of the Project’s staff and operations; -* Approve budgets, and contracts -* Vote to terminate contracts when necessary. -* Participate in at least one call per quarter; -* Help advocate for the Project; -* Help the Project build connections and partnerships; -* Can be part of the Admin team for the project on GitHub - -#### Conflict of Interest -In the event of any conflict of interest (a Committee Member, their family member, or someone with whom the Committee Member has a close academic or employment relationship is involved in a decision), the Committee Member must immediately notify other Committee Members. The Committee Member will be asked to recuse themselves from ongoing conversations, and decision process regarding the Transaction. - -#### Current list of Advisory Committee members -* Martin Magdinier -* Antonin Delpeuch. - -## Code For Science and Society - -Since January 2020, OpenRefine is a member project of the Code For Science and Society (CS&S). - -CS&S provides administrative and fiscal infrastructure to receive and manage funds under CS&S Fiscal Sponsorship Agreement. CS&S is a USA based organization qualified as exempt from federal income tax under Section 501(c)(3) of the Internal Revenue Code (IRC) and classified as a public charity under IRC Sections 509(a)(1) and 170(b)(1)(A)(vi). - -### Manage funds - -While the vast majority of OpenRefine contributors are volunteers, we recognize the need to attract and retain contributors to help: -- perform time critical maintenance tasks for the project (for example release management, security update, pull request review and comments) -- address issues that will not be naturally done by the community (for example documentation or working on large or long term projects) - -CS&S currently manages the current funding sources: -- **Grants**: Funds are allocated based on the grants requirements. Grants help to secure resources to achieve long term goal or supports software maintenance, growth, development, and community engagement. -- **Google Summer of Code**: Google Summer of Code mentors are eligible for USD 500 compensation for their work. -- **Donations made through GitHub Sponsors or made directly to CS&S on behalf of OpenRefine**: Decisions on usage of funds is made by the Advisory committee with guidance from the community. - -Combined with CS&S, the Steering and Advisory committees have experience applying and managing grants. If you are interested in applying to a grant to improve OpenRefine, please share your idea early on the Developer mailing list. It will be a pleasure helping you through your grant application. New grants must be approved by the Advisory Committee. that CS&S retains a 15% handling fees on any donation to finance their operations (20% for government funding). - -### Other assets - -CS&S owns and manages the domain openrefine.org diff --git a/OpenRefine/IDEs/eclipse/README.txt b/OpenRefine/IDEs/eclipse/README.txt deleted file mode 100644 index b92ff7e97..000000000 --- a/OpenRefine/IDEs/eclipse/README.txt +++ /dev/null @@ -1,29 +0,0 @@ - - OpenRefine for Eclipse - ------------------------- - - - This file contains Eclipse-specific help files that can get simplify your life - developing OpenRefine with Eclipse (http://www.eclipse.org/). - - - - Code Style Format Configurations (Refine.style.xml) - ------------------------------------------------------ - - This is the code formatting configurations that all OpenRefine developers should follow. - - To import, open the Eclipse preferences, then follow to "Java > Code Style > Formatter" - and click the "Import" button and load the file. - - - - - o - - - - Thank you for your interest. - - - The OpenRefine Development Team - http://github.com/OpenRefine/OpenRefine - \ No newline at end of file diff --git a/OpenRefine/IDEs/eclipse/Refine-codetemplates.xml b/OpenRefine/IDEs/eclipse/Refine-codetemplates.xml deleted file mode 100644 index 1c75c7acf..000000000 --- a/OpenRefine/IDEs/eclipse/Refine-codetemplates.xml +++ /dev/null @@ -1,62 +0,0 @@ - \ No newline at end of file diff --git a/OpenRefine/IDEs/eclipse/Refine.style.xml b/OpenRefine/IDEs/eclipse/Refine.style.xml deleted file mode 100644 index 63c8a5f8e..000000000 --- a/OpenRefine/IDEs/eclipse/Refine.style.xml +++ /dev/null @@ -1,390 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/OpenRefine/LICENSE.txt b/OpenRefine/LICENSE.txt deleted file mode 100644 index 58393644c..000000000 --- a/OpenRefine/LICENSE.txt +++ /dev/null @@ -1,11 +0,0 @@ -Copyright 2010, Google Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/OpenRefine/README.md b/OpenRefine/README.md deleted file mode 100644 index 228946dd9..000000000 --- a/OpenRefine/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# OpenRefine - -[![DOI](https://zenodo.org/badge/6220644.svg)](https://zenodo.org/badge/latestdoi/6220644) -[![Join the chat at https://gitter.im/OpenRefine/OpenRefine](https://badges.gitter.im/OpenRefine/OpenRefine.svg)](https://gitter.im/OpenRefine/OpenRefine) ![Java CI](https://github.com/OpenRefine/OpenRefine/workflows/Java%20CI/badge.svg) [![Coverage Status](https://coveralls.io/repos/github/OpenRefine/OpenRefine/badge.svg?branch=master)](https://coveralls.io/github/OpenRefine/OpenRefine?branch=master) [![Translation progress](https://hosted.weblate.org/widgets/openrefine/-/svg-badge.svg)](https://hosted.weblate.org/engage/openrefine/?utm_source=widget) [![Total alerts](https://img.shields.io/lgtm/alerts/g/OpenRefine/OpenRefine.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/OpenRefine/OpenRefine/alerts/) - -OpenRefine is a Java-based power tool that allows you to load data, understand it, -clean it up, reconcile it, and augment it with data coming from -the web. All from a web browser and the comfort and privacy of your own computer. - -[](http://openrefine.org) - -## Download - -* [OpenRefine Releases](https://github.com/OpenRefine/OpenRefine/releases) - -## Snapshot releases - -Latest development version, packaged for: -* [Linux](https://oss.sonatype.org/service/local/artifact/maven/content?r=snapshots&g=org.openrefine&a=openrefine&v=3.6-SNAPSHOT&c=linux&p=tar.gz) -* [MacOS](https://oss.sonatype.org/service/local/artifact/maven/content?r=snapshots&g=org.openrefine&a=openrefine&v=3.6-SNAPSHOT&c=mac&p=dmg) -* [Windows](https://oss.sonatype.org/service/local/artifact/maven/content?r=snapshots&g=org.openrefine&a=openrefine&v=3.6-SNAPSHOT&c=win&p=zip) - -## Run from source - -If you have cloned this repository to your computer, you can run OpenRefine with: - -* `./refine` on Mac OS and Linux -* `refine.bat` on Windows - -This requires [JDK 8](https://jdk.java.net) and [Apache Maven](https://maven.apache.org/). - -## Documentation and Videos - -* [User Manual](https://docs.openrefine.org) -* [FAQ](https://github.com/OpenRefine/OpenRefine/wiki/FAQ) -* [Official Website and tutorial videos](http://openrefine.org) - -## Contributing to the project - -* [Developers Guide & Architecture](https://github.com/OpenRefine/OpenRefine/wiki/Documentation-For-Developers) -* [Contributing Guide](https://github.com/OpenRefine/OpenRefine/blob/master/CONTRIBUTING.md) -* [Project Governance](https://github.com/OpenRefine/OpenRefine/blob/master/GOVERNANCE.md) - -## Contact us - -* [Mailing List](https://groups.google.com/forum/#!forum/openrefine) -* [Twitter](http://www.twitter.com/openrefine) -* [Gitter](https://gitter.im/OpenRefine/OpenRefine) -* [Matrix (bridged from Gitter)](https://matrix.to/#/#OpenRefine_OpenRefine:gitter.im) - -## Licensing and legal issues - -OpenRefine is open source software and is licensed under the BSD license -located in the [LICENSE.txt](LICENSE.txt). See the folder `licenses` for information on open source -libraries that OpenRefine depends on. - -## Credits - -This software was created by Metaweb Technologies, Inc. and originally written -and conceived by David Huynh . Metaweb Technologies, Inc. -was acquired by Google, Inc. in July 2010 and the product was renamed Google Refine. -In October 2012, it was renamed OpenRefine as it transitioned to a -community-supported product. - -See [CONTRIBUTING.md](./CONTRIBUTING.md) for instructions on how to contribute yourself. diff --git a/OpenRefine/SECURITY.md b/OpenRefine/SECURITY.md deleted file mode 100644 index 5fc1181b9..000000000 --- a/OpenRefine/SECURITY.md +++ /dev/null @@ -1,18 +0,0 @@ -# OpenRefine Security Policy - -## Supported Versions - -| Version | Supported | -| ------- | ------------------ | -| 3.5.x | :white_check_mark: | -| <= 3.4 | :x: | - -## Reporting a Vulnerability - -You can privately report a vulnerability to us by sending a report to this private mailing list [mailto:openrefine-coredev@googlegroups.com](mailto:openrefine-coredev@googlegroups.com) - -Our core team will try their best to fix any valid vulnerability that is reported to them. - -Keep in mind that OpenRefine is designed to run locally on a users PC, while also making network calls across the internet only upon a users choice or command. - -As such, certain vulnerabilities might not apply to OpenRefine's design. diff --git a/OpenRefine/appveyor.yml b/OpenRefine/appveyor.yml deleted file mode 100644 index 81ed3c167..000000000 --- a/OpenRefine/appveyor.yml +++ /dev/null @@ -1,63 +0,0 @@ -version: 1.0.{build} - -services: -- mysql -- postgresql96 - -init: -- cmd: java -version 2>&1 | find "version" - -clone_depth: 5 -skip_branch_with_pr: true -skip_commits: - files: - - docs/ - -environment: - JAVA_HOME: C:\Program Files\Java\jdk11 - MAVEN_HOME: C:\Program Files (x86)\Apache\Maven - -matrix: - fast_finish: true - -build: off - -# scripts to run before test -before_test: -- cmd: echo Running scripts before build... -- cmd: |- - PATH=C:\Program Files\PostgreSQL\9.6\bin\;C:\Program Files\MySQL\MySQL Server 5.7\bin\;%PATH% - SET MYSQL_PWD=Password12! - mysql -u root --password=Password12! -e "create database test_db;" - mysql -u root test_db --password=Password12! < extensions\database\tests\conf\test-mysql.sql - echo "localhost:*:test_db:postgres:Password12!" > C:\Program Files\PostgreSQL\9.6\pgpass.conf - echo "localhost:*:test_db:postgres:Password12!" > pgpass.conf - echo "localhost:*:test_db:postgres:Password12!" > %userprofile%\pgpass.conf - SET PGPASSFILE=C:\Program Files\PostgreSQL\9.6\pgpass.conf - SET PGPASSWORD=Password12! - SET PGUSER=postgres - createdb test_db - psql -U postgres test_db < extensions\database\tests\conf\test-pgsql.sql - - copy extensions\database\tests\conf\appveyor_tests.xml extensions\database\tests\conf\tests.xml - copy packaging\test_pom.xml packaging\pom.xml -- cmd: |- - mvn process-resources - mvn package -DskipTests=true -Dmaven.javadoc.skip=true -B -V - -test_script: -- cmd: echo Running test_script... -- cmd: >- - echo PATH is: - - path - - refine test - -on_failure: -- cmd: |- - dir - -cache: - - C:\Users\appveyor\.m2 - diff --git a/OpenRefine/benchmark/pom.xml b/OpenRefine/benchmark/pom.xml deleted file mode 100644 index 56cf7a872..000000000 --- a/OpenRefine/benchmark/pom.xml +++ /dev/null @@ -1,159 +0,0 @@ - - - 4.0.0 - - openrefine - org.openrefine - 3.6-SNAPSHOT - - - benchmark - jar - - OpenRefine Java JMH benchmarks - - - - ${project.groupId} - main - ${project.version} - provided - - - javax.servlet - javax.servlet-api - ${servlet-api.version} - provided - - - org.openjdk.jmh - jmh-core - ${jmh.version} - - - org.openjdk.jmh - jmh-generator-annprocess - ${jmh.version} - provided - - - org.testng - testng - 7.4.0 - test - - - - - UTF-8 - 1.34 - ${java.minversion} - openrefine-benchmarks - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.9.0 - - ${javac.target} - ${javac.target} - ${javac.target} - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.4 - - - package - - shade - - - ${uberjar.name} - - - org.openjdk.jmh.Main - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - ${project.groupId}:main:*:* - *:* - - - - - - - - - - - maven-clean-plugin - 2.5 - - - maven-install-plugin - 2.5.2 - - - maven-jar-plugin - ${maven-jar-plugin.version} - - - maven-javadoc-plugin - 3.3.1 - - - maven-resources-plugin - 3.2.0 - - - maven-site-plugin - 3.10.0 - - - maven-source-plugin - 3.2.1 - - - maven-surefire-plugin - 2.17 - - true - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - - - diff --git a/OpenRefine/benchmark/src/main/java/org/openrefine/benchmark/ToNumberBenchmark.java b/OpenRefine/benchmark/src/main/java/org/openrefine/benchmark/ToNumberBenchmark.java deleted file mode 100644 index ee55a2e79..000000000 --- a/OpenRefine/benchmark/src/main/java/org/openrefine/benchmark/ToNumberBenchmark.java +++ /dev/null @@ -1,94 +0,0 @@ -/******************************************************************************* - * Copyright (C) 2020, OpenRefine contributors - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ - -package org.openrefine.benchmark; - -import java.util.Properties; -import java.util.Random; -import java.util.concurrent.TimeUnit; - -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; - -import com.google.refine.expr.functions.ToNumber; - -public class ToNumberBenchmark { - - static Properties bindings = new Properties(); - - @State(Scope.Benchmark) - public static class ExecutionPlan { - - @Param({ "1000", "10000" }) - public int iterations; - - public ToNumber f; - String[] args = new String[1]; - String testData; - String testDataInt; - Random rnd = new Random(); - - @Setup(Level.Invocation) - public void setUp() { - f = new ToNumber(); - testData = Double.toString(rnd.nextDouble() * 10000); - testDataInt = testData.replace(".", ""); - } - } - - @Benchmark - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - @Warmup(iterations = 3, time = 200, timeUnit = TimeUnit.MILLISECONDS) - @Measurement(iterations = 5, time = 200, timeUnit = TimeUnit.MILLISECONDS) - public void toDoubleNew(ExecutionPlan plan, Blackhole blackhole) { - plan.args[0] = plan.testData; - blackhole.consume(plan.f.call(bindings, plan.args)); - } - - @Benchmark - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - @Warmup(iterations = 3, time = 200, timeUnit = TimeUnit.MILLISECONDS) - @Measurement(iterations = 5, time = 200, timeUnit = TimeUnit.MILLISECONDS) - @Fork(1) - public void toLongNew(ExecutionPlan plan, Blackhole blackhole) { - plan.args[0] = plan.testDataInt; - blackhole.consume(plan.f.call(bindings, plan.args)); - } -} diff --git a/OpenRefine/conf/openrefine.l4j.ini b/OpenRefine/conf/openrefine.l4j.ini deleted file mode 100644 index e4b7cff25..000000000 --- a/OpenRefine/conf/openrefine.l4j.ini +++ /dev/null @@ -1,13 +0,0 @@ -# Launch4j runtime config - -# initial memory heap size --Xms256M - -# max memory memory heap size --Xmx1024M - -# Use system defined HTTP proxies --Djava.net.useSystemProxies=true - -#-XX:+UseLargePages -#-Dsomevar="%SOMEVAR%" diff --git a/OpenRefine/conf/pmd.rules.xml b/OpenRefine/conf/pmd.rules.xml deleted file mode 100644 index 2dfe998cf..000000000 --- a/OpenRefine/conf/pmd.rules.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - OpenRefine PMD Ruleset - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/OpenRefine/docs/.gitignore b/OpenRefine/docs/.gitignore deleted file mode 100644 index c32cbeb55..000000000 --- a/OpenRefine/docs/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -node_modules - -lib/core/metadata.js -lib/core/MetadataBlog.js - -translated_docs -build/ -package-lock.json -i18n/* - -# generated files -.docusaurus -.cache-loader - -# misc -.DS_Store -.env.local -.env.development.local -.env.test.local -.env.production.local - -npm-debug.log* -yarn-debug.log* -yarn-error.log* diff --git a/OpenRefine/docs/.node-version b/OpenRefine/docs/.node-version deleted file mode 100644 index 8351c1939..000000000 --- a/OpenRefine/docs/.node-version +++ /dev/null @@ -1 +0,0 @@ -14 diff --git a/OpenRefine/docs/README.md b/OpenRefine/docs/README.md deleted file mode 100644 index 777db3d07..000000000 --- a/OpenRefine/docs/README.md +++ /dev/null @@ -1,106 +0,0 @@ -How to build these docs -======================= - -We use [Docusaurus 2](https://v2.docusaurus.io/) for our docs, a modern static website generator. - -### Requirements - -Assuming you have [Node.js (LTS recommended)](https://nodejs.org/en/download/) installed (which includes npm), you can install Docusaurus with: - -You will need to install [Yarn](https://yarnpkg.com/getting-started/install) before you can build the site. - -```sh -npm install -g yarn -``` - -### Installation - -Once you have installed yarn, navigate to docs directory & set-up the dependencies. - -```sh -cd docs -yarn -``` - -### Local Development - -```sh -yarn start -``` - -This command starts a local development server and opens up a browser window. Usually at the URL -Most changes are reflected live without having to restart the server. - -If you get an error starting yarn mentioning `update.latest` such as - -```sh ->yarn start - -if (notifier.update && semver.gt(this.update.latest, this.update.current)) { -TypeError: Cannot read property 'latest' of undefined - at Object. (E:\GitHubRepos\OpenRefine\docs\node_modules\@docusaurus\core\bin\docusaurus.js:49:46) -error Command failed with exit code 1. -``` - -then it is likely that you will need to first run - -```sh -yarn upgrade -``` - -which will update all dependencies and store them in the `yarn.lock` file to also be committed. - -### Next version of OpenRefine docs - -If you wish to work on the next version of docs for OpenRefine (`master` branch) then you will need to: - -1. Git checkout our `master` branch -2. Edit files under `docs/docs/` -3. Preview changes with the URL kept pointing to which will automatically - show changes live with yarn after you save a file. - -### Build - -```sh -yarn build -``` - -This command generates static content into the `build` directory and can be served using any static contents hosting service. -### (Optional) Test Build Locally -You can locally test ([with parameters](https://docusaurus.io/docs/cli#docusaurus-serve-sitedir)) the static content in the `build` directory (in case you don't have access to a hosting service) by using: - -```sh -yarn serve -``` - -or to build and then serve locally with one command: - -```sh -yarn serve --build -``` - -### Deployment - -```sh -GIT_USER= USE_SSH=true yarn deploy -``` - -If you are using GitHub pages for hosting, this command is a convenient way to build the website -and push to the `gh-pages` branch. - -### Translations - -It is now possible to translate the [OpenRefine docs](https://docs.openrefine.org/) via the -Crowdin platform: - - - -Unfortunately, unlike Weblate, we need to manually invite anyone who -wants to contribute translations. Feel free to request an invite by emailing us at openrefine-dev@googlegroups.com -We can also add languages, depending on interest. - -Your translations will not be immediately published on , it will take a few days -(at the next commit on the master branch). -The translated pages will first appear under (the documentation for the development version). -When we publish a version, the translations for that version, we will take a snapshot of the translations during that time. -We will trial this process for 3.5. diff --git a/OpenRefine/docs/crowdin.yml b/OpenRefine/docs/crowdin.yml deleted file mode 100644 index 24042688e..000000000 --- a/OpenRefine/docs/crowdin.yml +++ /dev/null @@ -1,16 +0,0 @@ -project_id: '455186' -api_token_env: 'CROWDIN_PERSONAL_TOKEN' -preserve_hierarchy: true -files: [ - # Markdown files for the user manual - { - source: '/docs/manual/**/*', - translation: '/i18n/%two_letters_code%/docusaurus-plugin-content-docs/current/**/%original_file_name%', - }, - # Docs home page - { - source: '/docs/index.md', - translation: '/i18n/%two_letters_code%/docusaurus-plugin-content-docs/current/%original_file_name%', - }, - - ] diff --git a/OpenRefine/docs/docs/index.md b/OpenRefine/docs/docs/index.md deleted file mode 100644 index 473cb8c7a..000000000 --- a/OpenRefine/docs/docs/index.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -slug: / -id: index -title: OpenRefine user manual -sidebar_label: Introduction ---- - - -This manual is designed to comprehensively walk through every aspect of setting up and using OpenRefine 3.4.1, including every interface function and feature. - - - -This user manual starts with instructions for [installing or upgrading OpenRefine on Windows, Mac, and Linux computers](manual/installing). It then walks you through [the interface and how to run OpenRefine](manual/running#jvm-preferences) from a program or command line, with or without setting custom preferences and modifications. - -The manual then teaches you how to [start a project](manual/starting) by importing an existing dataset. We work through how to [view and learn about your data](manual/exploring) using facets, filters, and sorting. - -Then we launch into [transforming that data permanently](manual/transforming) through common and custom transformations, clustering, pulling data from the web, [reconciling](manual/reconciling), and [writing expressions](manual/expressions). - -Finally we discuss what to do with your improved dataset, whether [exporting](manual/exporting) it to a file or [uploading statements to Wikidata or another Wikibase instance](manual/wikibase/overview). - -If you're stuck on any aspect and can't find an answer in the manual, try the [Troubleshooting page](manual/troubleshooting) for links to various places to find help. - -If you are new and want to learn how to use OpenRefine using an example dataset, you may wish to start with a user-contributed tutorial from our [recommendations list](https://github.com/OpenRefine/OpenRefine/wiki/External-Resources). diff --git a/OpenRefine/docs/docs/manual/cellediting.md b/OpenRefine/docs/docs/manual/cellediting.md deleted file mode 100644 index 103c8ed45..000000000 --- a/OpenRefine/docs/docs/manual/cellediting.md +++ /dev/null @@ -1,165 +0,0 @@ ---- -id: cellediting -title: Cell editing -sidebar_label: Cell editing ---- -## Overview {#overview} - -OpenRefine offers a number of features to edit and improve the contents of cells automatically and efficiently. - -One way of doing this is editing through a [text facet](facets#text-facet). Once you have created a facet on a column, hover over the displayed results in the sidebar. Click on the small “edit” button that appears to the right of the facet, and type in a new value. This will apply to all the cells in the facet. - -You can apply a text facet on numbers, boolean values, and dates, but if you edit a value it will be converted into the text [data type](exploring#data-types) (regardless of whether you edit a date into another correctly-formatted date, or a “true” value into “false”, etc.). - -## Transform {#transform} - -Select Edit cellsTransform... to open up an expressions window. From here, you can apply [expressions](expressions) to your data. The simplest examples are GREL functions such as [`toUppercase()`](grelfunctions#touppercases) or [`toLowercase()`](grelfunctions#tolowercases), used in expressions as `toUppercase(value)` or `toLowercase(value)`. When used on a column operation, `value` is the information in each cell in the selected column. - -Use the preview to ensure your data is being transformed correctly. - -You can also switch to the Undo / Redo tab inside the expressions window to reuse expressions you’ve already attempted in this project, whether they have been undone or not. - -OpenRefine offers you some frequently-used transformations in the next menu option, Common transforms. For more custom transforms, read up on [expressions](expressions). - -## Common transforms {#common-transforms} - -### Trim leading and trailing whitespace {#trim-leading-and-trailing-whitespace} - -Often cell contents that should be identical, and look identical, are different because of space or line-break characters that are invisible to users. This function will get rid of any characters that sit before or after visible text characters. - -### Collapse consecutive whitespace {#collapse-consecutive-whitespace} - -You may find that some text cells contain what look like spaces but are actually tabs, or contain multiple spaces in a row. This function will remove all space characters that sit in sequence and replace them with a single space. - -### Unescape HTML {#unescape-html} - -Your data may come from an HTML-formatted source that expresses some characters through references (such as “&nbsp;” for a space, or “%u0107” for a ć) instead of the actual Unicode characters. You can use the “unescape HTML entities” transform to look for these codes and replace them with the characters they represent. For other formatting that needs to be escaped, try a custom transformation with [`escape()`](grelfunctions#escapes-s-mode). - -### Replace smart quotes with ASCII {#replace-smart-quotes-with-ascii} - -Smart quotes (or curly quotes) recognize whether they come at the beginning or end of a string, and will generate an “open” quote (“) and a “close” quote (”). These characters are not ASCII-compliant (though they are UTF8-compliant) so you can use this tranform to replace them with a straight double quote character (") instead. - -### Case transforms {#case-transforms} - -You can transform an entire column of text into UPPERCASE, lowercase, or Title Case using these three options. This can be useful if you are planning to do textual analysis and wish to avoid case-sensitivity (which some functions are) causing problems in your analysis. Consider also using a [custom facet](facets#custom-text-facet) to temporarily modify cases instead of this permanent operation if appropriate. - -### Data-type transforms {#data-type-transforms} - -As detailed in [Data types](exploring#data-types), OpenRefine recognizes different data types: string, number, boolean, and date. When you use these transforms, OpenRefine will check to see if the given values can be converted, then both transform the data in the cells (such as “3” as a text string to “3” as a number) and convert the data type on each successfully transformed cell. Cells that cannot be transformed will output the original value and maintain their original data type. - -:::caution -Be aware that dates may require manual intervention to transform successfully: see the section on [Dates](exploring#dates) for more information. -::: - -Because these common transforms do not offer the ability to output an error instead of the original cell contents, be careful to look for unconverted and untransformed values. You will see a yellow alert at the top of screen that will tell you how many cells were converted - if this number does not match your current row set, you will need to look for and manually correct the remaining cells. Also consider faceting by data type, with the GREL function [`type()`](grelfunctions#typeo). - -You can also convert cells into null values or empty strings. This can be useful if you wish to, for example, erase duplicates that you have identified and are analyzing as a subset. - -## Fill down and blank down {#fill-down-and-blank-down} - -Fill down and blank down are two functions most frequently used when encountering data organized into [records](exploring#row-types-rows-vs-records) - that is, multiple rows associated with one specific entity. - -If you receive information in rows mode and want to convert it to records mode, the easiest way is to sort your first column by the value that you want to use as a unique records key, [make that sorting permanent](transforming#edit-rows), then blank down all the duplicates in that column. OpenRefine will retain the first unique value and erase the rest. Then you can switch from “Show as rows” to “Show as records” and OpenRefine will associate rows to each other based on the remaining values in the first column. - -Be careful that your data is sorted properly before you begin blanking down - not just the first column but other columns you may want to have in a certain order. For example, you may have multiple identical entries in the first column, one with a value in the second column and one with an empty cell in the second column. In this case you want the row with the second-column value to come first, so that you can clean up empty rows later, once you blank down. - -If, conversely, you’ve received data with empty cells because it was already in something akin to records mode, you can fill down information to the rest of the rows. This will duplicate whatever value exists in the topmost cell with a value: if the first row in the record is blank, it will take information from the next cell, or the cell after that, until it finds a value. The blank cells above this will remain blank. - -## Split multi-valued cells {#split-multi-valued-cells} - -Splitting cells with more than one value in them is a common way to get your data from single rows into [multi-row records](exploring#rows-vs-records). Survey data, for example, frequently allows respondents to “Select all that apply,” or an inventory list might have items filed under more than one category. - -You can split a column based on any character or series of characters you input, such as a semi-colon (;) or a slash (/). The default is a comma. Splitting based on a separator will remove the separator characters, so you may wish to include a space with your separator (; ) if it exists in your data. - -You can use [expressions](expressions) to design the point at which a cell should split itself into two or more rows. This can be used to identify special characters or create more advanced evaluations. You can split on a line-break by entering `\n` and checking the “[regular expression](expressions#regular-expressions)” checkbox. - -Regular expressions can be useful if the split is not straightforward: say, if a capital letter (`[A-Z]`) indicates the beginning of a new string, or if you need to _not_ always split on a character that appears in both the strings and as a separator. Remember that this will remove all the matching characters. - -You can also split based on the lengths of the strings you expect to find. This can be useful if you have predictable data in the cells: for example, a 10-digit phone number, followed by a space, followed by another 10-digit phone number. Any characters past the explicit length you’ve specified will be discarded: if you split by “11, 10” any characters that may come after the 21st character will disappear. If some cells only have one phone number, you will end up with blank rows. - -If you have data that should be split into multiple columns instead of multiple rows, see [Split into several columns](columnediting#split-into-several-columns). - -## Join multi-valued cells {#join-multi-valued-cells} - -Joining will reverse the “split multi-valued cells” operation, or join up information from multiple rows into one row. All the strings will be compressed into the topmost cell in the record, in the order they appear. A window will appear where you can set the separator; the default is a comma and a space (, ). This separator is optional. We suggest the separator | as a sufficiently rare character. - -## Cluster and edit {#cluster-and-edit} - -Creating a facet on a column is a great way to look for inconsistencies in your data; clustering is a great way to fix those inconsistencies. Clustering uses a variety of comparison methods to find text entries that are similar but not exact, then shares those results with you so that you can merge the cells that should match. Where editing a single cell or text facet at a time can be time-consuming and difficult, clustering is quick and streamlined. - -Clustering always requires the user to approve each suggested edit - it will display values it thinks are variations on the same thing, and you can select which version to keep and apply across all the matching cells (or type in your own version). - -OpenRefine will do a number of cleanup operations behind the scenes in order to do its analysis, but only the merges you approve will modify your data. Understanding those different behind-the-scenes cleanups can help you choose which clustering method will be more accurate and effective. - -You can start the process in two ways: using the dropdown menu on your column, select Edit cellsCluster and edit…; or create a text facet and then press the “Cluster” button that appears in the facet box. - -![A screenshot of the Clustering window.](/img/cluster.png) - -The clustering pop-up window will take a small amount of time to analyze your column, and then make some suggestions based on the clustering method currently active. - -For each cluster identified, you can pick one of the existing values to apply to all cells, or manually type in a new value in the text box. And, of course, you can choose not to cluster them at all. OpenRefine will keep analyzing every time you make a change, with Merge selected & re-cluster, and you can work through all the methods this way. - -You can also export the currently identified clusters as a JSON file, or close the window with or without applying your changes. You can also use the histograms on the right to narrow down to, for example, clusters with lots of matching rows, or clusters of long or short values. - -### Clustering methods {#clustering-methods} - -You don’t need to understand the details behind each clustering method to apply them successfully to your data. The order in which these methods are presented in the interface and on this page is the order we recommend - starting with the most strict rules and moving to the most lax, which require more human supervision to apply correctly. - -The clustering pop-up window offers you a variety of clustering methods: - -* key collision - * fingerprint - * ngram-fingerprint - * metaphone3 - * cologne-phonetic - * Daitch-Mokotoff - * Beider-Morse -* nearest neighbor - * levenshtein - * ppm - -#### Key collision {#key-collision} - -**Key collisions** are very fast and can process millions of cells in seconds: - -**Fingerprinting** is the least likely to produce false positives, so it’s a good place to start. It does the same kind of data-cleaning behind the scenes that you might think to do manually: fix whitespace into single spaces, put all uppercase letters into lowercase, discard punctuation, remove diacritics (e.g. accents) from characters, split up all strings (words) and sort them alphabetically (so “Zhenyi, Wang” becomes “wang zhenyi”). - -**N-gram fingerprinting** allows you to set the _n_ value to whatever number you’d like, and will create n-grams of _n_ size (after doing some cleaning), alphabetize them, then join them back together into a fingerprint. For example, a 1-gram fingerprint will simply organize all the letters in the cell into alphabetical order - by creating segments one character in length. A 2-gram fingerprint will find all the two-character segments, remove duplicates, alphabetize them, and join them back together (for example, “banana” generates “ba an na an na,” which becomes “anbana”). - -This can help match cells that have typos, or incorrect spaces (such as matching “lookout” and “look out,” which fingerprinting itself won’t identify because it separates words). The higher the _n_ value, the fewer clusters will be identified. With 1-grams, keep an eye out for mismatched values that are near-anagrams of each other (such as “Wellington” and “Elgin Town”). - -##### Phonetic clustering {#phonetic-clustering} - -The next four methods are phonetic algorithms: they identify letters that sound the same when pronounced out loud, and assess text values based on that (such as knowing that a word with an “S” might be a mistype of a word with a “Z”). They are great for spotting mistakes made by not knowing the spelling of a word or name after hearing it spoken aloud. - -**Metaphone3 fingerprinting** is an English-language phonetic algorithm. For example, “Reuben Gevorkiantz” and “Ruben Gevorkyants” share the same phonetic fingerprint in English. - -**Cologne fingerprinting** is another phonetic algorithm, but for German pronunciation. - -**Daitch-Mokotoff** is a phonetic algorithm for Slavic and Yiddish words, especially names. **Baider-Morse** is a version of Daitch-Mokotoff that is slightly more strict. - -Regardless of the language of your data, applying each of them might find different potential matches: for example, Metaphone clusters “Cornwall” and “Corn Hill” and “Green Hill,” while Cologne clusters “Greenvale” and “Granville” and “Cornwall” and “Green Wall.” - -#### Nearest neighbor {#nearest-neighbor} - -**Nearest neighbor** clustering methods are slower than key collision methods. They allow the user to set a radius - a threshold for matching or not matching. OpenRefine uses a “blocking” method first, which sorts values based on whether they have a certain amount of similarity (the default is “6” for a six-character string of identical characters) and then runs the nearest-neighbor operations on those sorted groups. - -We recommend setting the block number to at least 3, and then increasing it if you need to be more strict (for example, if every value with “river” is being matched, you should increase it to 6 or more). Note that bigger block values will take much longer to process, while smaller blocks may miss matches. Increasing the radius will make the matches more lax, as bigger differences will be clustered. - -**Levenshtein distance** counts the number of edits required to make one value perfectly match another. As in the key collision methods above, it will do things like change uppercase to lowercase, fix whitespace, change special characters, etc. Each character that gets changed counts as 1 “distance.” “New York” and “newyork” have an edit distance value of 3 (“N” to “n”; “Y” to “y”; remove the space). It can do relatively advanced edits, such as understand the distance between “M. Makeba” and “Miriam Makeba” (5), but it may create false positives if these distances are greater than other, simpler transformations (such as the one-character distance to “B. Makeba,” another person entirely). - -**PPM (Prediction by Partial Matching)** uses compression to see whether two values are similar or different. In practice, this method is very lax even for small radius values and tends to generate many false positives, but because it operates at a sub-character level it is capable of finding substructures that are not easily identifiable by distances that work at the character level. So it should be used as a “last resort” clustering method. It is also more effective on longer strings than on shorter ones. - -For more of the theory behind clustering, see [Clustering In Depth](https://github.com/OpenRefine/OpenRefine/wiki/Clustering-In-Depth). - -## Replace {#replace} - -OpenRefine provides a find/replace function for you to edit your data. Selecting Edit cellsReplace will bring up a simple window where you can input a string to search and a string to replace it with. You can set case-sensitivity, and set it to only select whole words, defined by a string with spaces or punctuation around it (to prevent, for example, “house” selecting the “house” part of “doghouse”). You can use [regular expressions](expressions#regular-expressions) in this field. You may wish to preview the results of this operation by testing it with a [Text filter](facets#text-filter) first. - -You can also perform a sort of find/replace operation by editing one cell, and selecting “apply to all identical cells.” - -## Edit one cell at a time {#edit-one-cell-at-a-time} - -You can edit individual cells by hovering your mouse over that cell. You should see a tiny blue link labeled “edit.” Click it to edit the cell. That pops up a window with a bigger text field for you to edit. You can change the [data type](exploring#data-types) of that cell, and you can apply these changes to all identical cells (in the same column), using this pop-up window. - -You will likely want to avoid doing this except in rare cases - the more efficient means of improving your data will be through automated and bulk operations. \ No newline at end of file diff --git a/OpenRefine/docs/docs/manual/columnediting.md b/OpenRefine/docs/docs/manual/columnediting.md deleted file mode 100644 index 754a61c62..000000000 --- a/OpenRefine/docs/docs/manual/columnediting.md +++ /dev/null @@ -1,124 +0,0 @@ ---- -id: columnediting -title: Column editing -sidebar_label: Column editing ---- - -## Overview {#overview} - -Column editing contains some of the most powerful data-improvement methods in OpenRefine. The operations in the Edit column menu involve using one column of data to add entirely new columns and fields to your dataset. - -## Splitting or joining {#splitting-or-joining} - -Many users find that they frequently need to make their data more granular: for example, splitting a “Firstname Lastname” column into two columns, one for first names and one for last names. The reverse is also often true: you may have several columns of category values that you want to join into one “category” column. -. -### Split into several columns {#split-into-several-columns} - -![A screenshot of the settings window for splitting columns.](/img/columnsplit.png) - -You can find this operation at Edit columnSplit into several columns.... Splitting one column into several columns requires you to identify the character, string lengths, or evaluating expression you want to split on. Just like [splitting multi-valued cells into rows](cellediting#split-multi-valued-cells), splitting cells into multiple columns will remove the separator character or string you indicate. Splitting by lengths will discard any information that comes after the specified total length. - -You can also specify a maximum number of new columns to be made: separator characters after this limit will be ignored, and the remaining characters will end up in the last column. - -New columns will be named after the original column, with a number: “Location 1,” “Location 2,” etc. You can choose to remove the original column with this operation, and you can have [data types](exploring#data-types) identified where possible. This function will work best with converting strings to numbers, and may not work with [dates](exploring#dates). - -### Join columns {#join-columns} - -![A screenshot of the settings window for joining columns.](/img/columnjoin.png) - -You can join columns by selecting Edit columnJoin columns.... All the columns currently in your dataset will appear in the pop-up window. You can select or un-select all the columns you want to join, and drag columns to put them in the order you want to join them in. You will define a separator character (optional) and define a string to insert into empty cells (nulls). - -The joined data will appear in the column you originally selected, or you can create a new column for this content and specify a name. You can delete all the columns that were used in this join operation. - -## Add column based on this column {#add-column-based-on-this-column} - -Selecting Edit columnAdd column based on this column... will open up an [expressions](expressions) window where you can transform the data from this column (using `value`), or write a more complex expression that takes information from any number of columns or from external sources. - -Expressions used in this operation will rely on your knowledge of variables. You can learn more in the [Expressions section on variables](expressions#variables). - -The simplest way to use this operation is simply leave the default `value` in the expression field, to create an exact copy of your column. For a column of [reconciled data](reconciling), you can use the variable `cell` instead, to copy both the original string and the existing reconciliation data. This will include matched values, candidates, and new items. - -One useful expression is to create a column based on concatenating (merging) two other columns. Select either of the source columns, choose Edit columnAdd column based on this column..., name your new column, and use the following format in the expression window: - -``` -cells["Column 1"].value + cells["Column 2"].value -``` - -If your column names do not contain spaces, you can use the following format instead: - -``` -cells.Column1.value + cells.Column2.value -``` - -If you are in records mode instead of rows mode, you can concatenate using the following format: - -``` -row.record.cells.Column1.value + row.record.cells.Column2.value -``` - -You may wish to add separators or spaces, or modify your input during this operation with more advanced expressions. - -## Add column by fetching URLs {#add-column-by-fetching-urls} - -Through the Add column by fetching URLs function, OpenRefine supports the ability to fetch HTML or data from web pages or services. In this operation you will be building URL strings based on your column of data, by using `value` to insert a relevant substring. Your chosen column needs to contains parts of paths to valid HTML pages or files online. - -If you have a column of URLs and want to fetch the information that they point to, you can simply run the expression as `value`. If your column has, for example, unique identifiers for Wikidata entities (numerical values starting with Q), you can download the JSON-formatted metadata about each entity with - -``` -"https://www.wikidata.org/wiki/Special:EntityData/" + value + ".json" -``` - -or whatever metadata format you prefer. Information about the format options in Wikidata can be found [here](https://www.wikidata.org/wiki/Wikidata:Data_access). The service you are fetching data from may have similar documentation on its provided options. - -![A screenshot of the settings window for fetching URLs.](/img/fetchingURLs.png) - -This service is more useful when getting metadata files instead of HTML, but you may wish to work with a page’s entire HTML contents and then parse out information from that. - -:::caution -Be aware that the fetching process can take quite some time and that servers may not want to fulfill hundreds or thousands of page requests in seconds. Fetching allows you to set a “throttle delay” which determines the amount of time between requests. The default is 5 seconds per row in your dataset (5000 milliseconds). We recommend leaving this at 1000 or greater. -::: - -Note the following: -* Before pressing “OK,” copy and paste a URL or two from the preview and test them in another browser tab to make sure they work. -* In some situations you may need to set [HTTP request headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers). To set these, click the small “Show” button next to “HTTP headers to be used when fetching URLs” in the settings window. The authorization credentials get logged in your operation history in plain text, which may be a security concern for you. You can set the following request headers: - * [User-Agent](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent) - * [Accept](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept) - * [Authorization](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization) - -### Common errors {#common-errors} - -When OpenRefine attempts to fetch information from a web service, it can fail in a variety of ways. The following information is meant to help troubleshoot and fix problems encountered when using this function. - -First, make sure that your fetching operation is storing errors (check “store error”). Then run the fetch and look at the error messages. - -**“HTTP error 403 : Forbidden”** can be simply down to you not having access to the URL you are trying to use. If you can access the same URL with your browser, the remote site may be blocking OpenRefine because it doesn't recognize its request as valid. Changing the [User-Agent](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent) request header may help. If you believe you should have access to a site but are “forbidden,” you may wish to contract the administrators. - -**“HTTP error 404 : Not Found”** indicates that the information you are requesting does not exist, perhaps due to a problem with your cell values if it only happening in certain rows. - -**“HTTP error 500 : Internal Server Error”** indicates the remote server is having a problem filling your request. You may wish to simply wait and try again later, or double-check the URLs. - -**“error: javax.net.ssl.SSLHandshakeException: Received fatal alert: handshake_failure”** can occur when you are trying to retrieve information over HTTPS but the remote site is using an encryption not supported by the Java virtual machine being used by OpenRefine. - -You can check which encryption methods are supported by your OpenRefine/Java installation by using a service such as **How's my SSL**. Add the URL `https://www.howsmyssl.com/a/check` to an OpenRefine cell and run “Add column by fetching URLs” on it, which will provide a description of the SSL client being used. - -You can try installing additional encryption supports by installing the [Java Cryptography Extension](https://www.oracle.com/java/technologies/javase-jce8-downloads.html). -Note that for Mac users and for Windows users with the OpenRefine installation with bundled JRE, these updated cipher suites need to be dropped into the Java install within the OpenRefine application: - -* On Mac, it will look something like `/Applications/OpenRefine.app/Contents/PlugIns/jdk1.8.0_60.jdk/Contents/Home/jre/lib/security`. -* On Windows: `\server\target\jre\lib\security`. - -**“javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed”** can appear when the remote site is using an HTTPS certificate not trusted by your local Java installation. You will need to make sure that the certificate, or (more likely) the root certificate, is trusted. - -The list of trusted certificates is stored in an encrypted file called `cacerts` in your local Java installation. This can be read and updated by a tool called “keytool.” You can find directions on how to add a security certificate to the list of trusted certificates for a Java installation [here](http://magicmonster.com/kb/prg/java/ssl/pkix_path_building_failed.html) and [here](http://javarevisited.blogspot.co.uk/2012/03/add-list-certficates-java-keystore.html). - -Note that for Mac users and for Windows users with the OpenRefine installation with bundled JRE, the `cacerts` file within the OpenRefine application needs to be updated. - -* On Mac, it will look something like `/Applications/OpenRefine.app/Contents/PlugIns/jdk1.8.0_60.jdk/Contents/Home/jre/lib/security/cacerts`. -* On Windows: `\server\target\jre\lib\security\`. - -## Renaming, removing, and moving {#renaming-removing-and-moving} - -Every column's Edit column dropdown contains options to move it (to the beginning, end, left, or right), rename it, and delete it. -These operations can be undone, but a removed column cannot be restored later if you keep modifying your data. If you wish to temporarily hide a column, go to [View](sortview#view)Collapse this column instead. - -Be cautious about moving columns in [records mode](cellediting#rows-vs-records): if you change the first column in your dataset (the key column), your records may change in unintended ways. diff --git a/OpenRefine/docs/docs/manual/exploring.md b/OpenRefine/docs/docs/manual/exploring.md deleted file mode 100644 index 23bafc1a7..000000000 --- a/OpenRefine/docs/docs/manual/exploring.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -id: exploring -title: Exploring data -sidebar_label: Overview ---- - -## Overview {#overview} - -OpenRefine offers lots of features to help you learn about your dataset, even if you don’t change a single character. In this section we cover different ways for sorting through, filtering, and viewing your data. - -Unlike spreadsheets, OpenRefine doesn’t store formulas and display the output of those calculations; it only shows the value inside each cell. It doesn’t support cell colors or text formatting. - -## Data types {#data-types} - -Each piece of information (each cell) in OpenRefine is assigned a data type. Some file formats, when imported, can set data types that are recognized by OpenRefine. Cells without an associated data type on import will be considered a “string” at first, but you can have OpenRefine convert cell contents into other data types later. This is set at the cell level, not at the column level. - -You can see data types in action when you preview a new project: check the box next to Attempt to parse cell text into numbers, and cells will be converted to the “number” data type based on their contents. You’ll see numbers change from black text to green if they are recognized. - -The data type will determine what you can do with the value. For example, if you want to add two values together, they must both be recognized as the number type. - -You can check data types at any time by: -* clicking “edit” on a single cell (where you can also edit the type) -* creating a Custom Text Facet on a column, and inserting `type(value)` into the Expression field. This will generate the data type in the preview, and you can facet by data type if you press OK. - -The data types supported are: -* string (one or more text characters) -* number (one or more characters of numbers only) -* boolean (values of “true” or “false”) -* [date](#dates) (ISO-8601-compliant extended format with time in UTC: YYYY-MM-DDTHH:MM:SSZ) - -OpenRefine recognizes two further data types as a result of its own processes: -* error -* null - -An “error” data type is created when the cell is storing an error generated during a transformation in OpenRefine. - -A “null” data type is a special type that means “this cell has no value.” It’s distinct from cells that have values such as “0” or “false”, or cells that look empty but have whitespace in them, or cells that contain empty strings. When you use `type(value)`, it will show you that the cell’s value is “null” and its type is “undefined.” You can opt to [show “null” values](sortview#showhide-null), by going to AllViewShow/Hide ‘null’ values in cells. - -Changing a cell's data type is not the same operation as transforming its contents. For example, using a column-wide transform such as TransformCommon transformsTo date may not convert all values successfully, but going to an individual cell, clicking “edit”, and changing the data type can successfully convert text to a date. These operations use different underlying code. Learn more about date formatting and transformations in the next section. - -To transform data from one type to another, see [Transforming data](cellediting#data-type-transforms) for information on using common tranforms, and see [Expressions](expressions) for information on using [toString()](grelfunctions#tostringo-string-format-optional), [toDate()](grelfunctions#todateo-b-monthfirst-s-format1-s-format2-), and other functions. - - -### Dates {#dates} - -A “date” type is created when a column is [transformed into dates](transforming#to-date), when an expression is used to [convert cells to dates](grelfunctions#todateo-b-monthfirst-s-format1-s-format2-) or when individual cells are set to have the data type “date”. - -Date-formatted data in OpenRefine relies on a number of conversion tools and standards. For something to be considered a date in OpenRefine, it will be converted into the ISO-8601-compliant extended format with time in UTC: YYYY-MM-DDTHH:MM:SSZ. - -When you run Edit cellsCommon transformsTo date, the following column of strings on the left will transform into the values on the right: - -|Input|→|Output| -|---|---|---| -|23/12/2019|→|2019-12-23T00:00:00Z| -|14-10-2015|→|2015-10-14T00:00:00Z| -|2012 02 16|→|2012-02-16T00:00:00Z| -|August 2nd 1964|→|1964-08-02T00:00:00Z| -|today|→|today| -|never|→|never| - -OpenRefine uses a variety of tools to recognize, convert, and format [dates](exploring#dates) and so some of the values above can be reformatted using other methods. In this case, clicking the “today” cell and editing its data type manually will convert “today” into a value such as “2020-08-14T00:00:00Z”. Attempting the same data-type change on “never” will give you an error message and refuse to proceed. - -You can do more precise conversion and formatting using expressions and arguments based on the state of your data: see the GREL functions reference section on [Date functions](grelfunctions#date-functions) for more help. - -You can convert dates into a more human-readable format when you [export your data using the custom tabular exporter](exporting#custom-tabular-exporter). You are given the option to keep your dates in the ISO 8601 format, to output short, medium, long, or full locale formats, or to specify a custom format. This means that you can format your dates into, for example, MM/DD/YY (the US short standard) with or without including the time, after working with ISO-8601-formatted dates in your project. - -The following table shows some example [date and time formatting styles for the U.S. and French locales](https://docs.oracle.com/javase/tutorial/i18n/format/dateFormat.html): - -|Style |U.S. Locale |French Locale| -|---|---|---| -|Default |Jun 30, 2009 7:03:47 AM |30 juin 2009 07:03:47| -|Short |6/30/09 7:03 AM |30/06/09 07:03| -|Medium |Jun 30, 2009 7:03:47 AM |30 juin 2009 07:03:47| -|Long |June 30, 2009 7:03:47 AM PDT |30 juin 2009 07:03:47 PDT| -|Full |Tuesday, June 30, 2009 7:03:47 AM PDT |mardi 30 juin 2009 07 h 03 PDT| - -## Rows vs. records {#rows-vs-records} - -A row is a simple way to organize data: a series of cells, one cell per column. Sometimes there are multiple pieces of information in one cell, such as when a survey respondent can select more than one response. - -In cases where there is more than one value for a single column in one or more rows, you may wish to use OpenRefine’s records mode: this defines a single record as potentially containing more than one row. From there you can transform cells into multiple rows, each cell containing one value you’d like to work with. - -Generally, when you import some data, OpenRefine reads that data in row mode. From the project screen, you can convert the project into records mode. OpenRefine remembers this action and will present you with records mode each time you open the project from then on. - -OpenRefine understands records based on the content of the first column, what we call the “key column.” Splitting a row into a multi-row record will base all association on the first column in your dataset. - -If you have more than one column to split out into multiple rows, OpenRefine will keep your data associated with its original record, and associate subgroups based on the top-most row in each group. - -You can imagine the structure as a tree with many branches, all leading back to the same trunk. - -For example, your key column may be a film or television show, with multiple cast members identified by name, associated to that work. You may have one or more roles listed for each person. The roles are linked to the actors, which are linked to the title. - -|Work|Actor|Role| -|---|---|---| -|The Wizard of Oz|Judy Garland|Dorothy Gale| -||Ray Bolger|"Hunk"| -|||The Scarecrow| -||Jack Haley|"Hickory"| -|||The Tin Man| -||Bert Lahr|"Zeke"| -|||The Cowardly Lion| -||Frank Morgan|Professor Marvel| -|||The Gatekeeper| -|||The Carriage Driver| -|||The Guard| -|||The Wizard of Oz| -||Margaret Hamilton|Miss Almira Gulch| -|||The Wicked Witch of the West| - -Once you are in records mode, you can still move some columns around, but if you move a column to the beginning, you may find your data becomes misaligned. The new key column will sort into records based on empty cells, and values in the old key column will be assigned to the last row in the old record (the key value sitting above those values). - -OpenRefine assigns a unique key behind the scenes, so your records don’t need a unique identifier in the key column. You can keep track of which rows are assigned to each record by the record number that appears under the All column. - -To [split multi-valued cells](transforming#split-multi-valued-cells) and apply other operations that take advantage of records mode, see [Transforming data](transforming). - -Be careful when in records mode that you do not accidentally delete rows based on being blank in one column where there is a value in another. diff --git a/OpenRefine/docs/docs/manual/exporting.md b/OpenRefine/docs/docs/manual/exporting.md deleted file mode 100644 index 21b1516a9..000000000 --- a/OpenRefine/docs/docs/manual/exporting.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -id: exporting -title: Exporting your work -sidebar_label: Exporting ---- - -## Overview {#overview} - -Once your dataset is ready, you will need to get it out of OpenRefine and into the system of your choice. OpenRefine outputs a number of file formats, can upload your data directly into Google Sheets, and can create or update statements on Wikidata. - -You can also [export your full project data](#export-a-project) so that it can be opened by someone else using OpenRefine (or yourself, on another computer). - -## Export data {#export-data} - -![A screenshot of the Export dropdown.](/img/export-menu.png) - -Many of the options only export data in the current view - that is, with current filters and facets applied. Some will give you the choice to export your entire dataset or just the currently-viewed rows. - -To export data from a project, click the Export dropdown button in the top right corner and pick the format you want. Your options are: - -* Tab-separated value (TSV) or Comma-separated value (CSV) -* HTML-formatted table -* Excel spreadsheet (XLS or XLSX) -* Open Document Format (ODF) spreadsheet (ODS) -* Upload to Google Sheets (requires [Google account authorization](starting#google-sheet-from-drive)) -* [Custom tabular exporter](#custom-tabular-exporter) -* [SQL statement exporter](#sql-statement-exporter) -* [Templating exporter](#templating-exporter), which generates JSON by default - -You can also export reconciled data to Wikidata, or export your Wikidata schema for future use with other OpenRefine projects: - -* [Upload edits to Wikidata](wikibase/uploading#uploading-with-openrefine) -* [Export to QuickStatements](wikibase/uploading#uploading-with-quickstatements) (version 1) -* [Export Wikidata schema](wikibase/overview#import-and-export-schema) - -### Custom tabular exporter {#custom-tabular-exporter} - -![A screenshot of the custom tabular content tab.](/img/custom-tabular-exporter.png) - -With the custom tabular exporter, you can choose which of your data to export, the separator you wish to use, and whether you'd like to download the result to your computer or upload it into a Google Sheet. - -On the Content tab, you can drag and drop the columns appearing in the column list to reorder the output. The options for reconciled and date data are applied to each column individually. - -This exporter is especially useful with reconciled data, as you can choose whether you wish to output the cells' original values, the matched values, or the matched IDs. Ouputting “match entity's name”, “matched entity's ID”, or “cell's content” will output, respectively, the contents of `cell.recon.match.name`, `cell.recon.match.id`, and `cell.value`. - -“Output nothing for unmatched cells” will export empty cells for both newly-created matches and cells with no chosen matches. “Link to matched entity's page” will produce hyperlinked text in an HTML table output, but have no effect in other formats. - -At this time, the date-formatting options in this window do not work. You can [keep track of this issue on Github](https://github.com/OpenRefine/OpenRefine/issues/3368). -In the future, you will be able to choose how to [output date-formatted cells](exploring#dates). You can create a custom date output by using [formatting according to the SimpleDateFormat parsing key found here](grelfunctions#todateo-b-monthfirst-s-format1-s-format2-). - -![A screenshot of the custom tabular file download tab.](/img/custom-tabular-exporter2.png) - -On the Download tab, you can generate a preview of how the first ten rows of your dataset will output. If you do not choose one of the file formats on the right, the Download button will generate a text file. On the Upload tab, you can create a new Google Sheet. - -With the Option Code tab, you can copy JSON of your current custom settings to reuse on another export, or you can paste in existing JSON settings to apply to the current project. - -### SQL exporter {#sql-exporter} - -The SQL exporter creates a SQL statement containing the data you’ve exported, which you can use to overwrite or add to an existing database. Choosing ExportSQL exporter will bring up a window with two tabs: one to define what data to output, and another to modify other aspects of the SQL statement, with options to preview and download the statement. - -![A screenshot of the SQL statement content window.](/img/sql-exporter.png) - -The Content tab allows you to craft your dataset into an SQL table. From here, you can choose which columns to export, the data type to export for each (or choose "VARCHAR"), and the maximum character length for each field (if applicable based on the data type). You can set a default value for empty cells after unchecking “Allow null” in one or more columns. - -With this output tool, you can choose whether to output only currently visible rows, or all the rows in your dataset, as well as whether to include empty rows. The option to “Trim column names” will remove their whitespace characters. - -![A screenshot of the SQL statement download window.](/img/sql-exporter2.png) - -The Download tab allows you to finalize your complete SQL statement. - -Include schema means that you will start your statement with the creation of a table. Without that, you will only have an INSERT statement. - -Include content means including the INSERT statement with data from your project. Without that, you will only create empty columns. - -You can include DROP and IF EXISTS if you require them, and set a name for the table to which the statement will refer. - -You can then preview your statement, which will open up a new browser tab/window showing a statement with the first ten rows of your data (if included), or you can save a `.sql` file to your computer. - -### Templating exporter {#templating-exporter} - -If you pick Templating… from the Export dropdown menu, you can “roll your own” exporter. This is useful for formats that we don't support natively yet, or won't support. The Templating exporter generates JSON by default. - -![A screenshot of the Templating exporter generating JSON by default.](/img/templating-exporter.png) - -The Templating Export window allows you to set your own separators, prefix, and suffix to create a complete dataset in the language of your choice. In the Row template section, you can choose which columns to generate from each row by calling them with [variables](expressions#variables). - -This can be used to: -* output [reconciliation data](expressions#reconciliation), such as `cells["ColumnName"].recon.match.name` -* create multiple columns of output from different [member fields](expressions#variables) of a single project column -* employ [expressions](expressions) to modify data for output: for example, `cells["ColumnName"].value.toUppercase()`. - -Anything that appears inside doubled curly braces ({{ }}) is treated as a GREL expression; anything outside is generated as straight text. You can use Jython or Clojure by declaring it at the start: -``` -{{jython:return cells["ColumnName"].value}} -``` - -:::caution -Note that some syntax is different in this tool than elsewhere in OpenRefine: a forward slash must be escaped with a backslash, while other characters do not need escaping. You cannot, at this time, include a closing curly brace (}) anywhere in your expression, or it will cause it to malfunction. -::: - -You can include [regular expressions](expressions#regular-expressions) as usual (inside forward slashes, with any GREL function that accepts them). For example, you could output a version of your cells with punctuation removed, using an expression such as -``` -{{jsonize(cells["ColumnName"].value.replaceChars("/[.!?$&,/]/",""))}} -``` - -You could also simply output a plain-text document inserting data from your project into sentences: for example, "In `{{cells["Year"].value}}` we received `{{cells["RequestCount"].value}}` requests." - -You can use the shorthand `${ColumnName}` (no need for quotes) to insert column values directly. You cannot use this inside an expression, because of the closing curly brace. - -If your projects is in records mode, the Row separator field will insert a separator between records, rather than individual rows. Rows inside a single record will be directly appended to one another as per the content in the Row Template field. - -Once you have created your template, you may wish to save the text you produced in each field, in order to reuse it in the future. Once you click Export OpenRefine will output a simple `.txt` file, and your template will be discarded. - -We have recipes on using the Templating exporter to [produce several different formats](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#12-templating-exporter). - -## Export a project {#export-a-project} - -You can share a project in progress with another computer, a colleague, or with someone who wants to check your history. This can be useful for showing that your data cleanup didn’t distort or manipulate the information in any way. Once you have exported a project, another OpenRefine installation can [import it as a new project](starting#import-a-project). - -You can either save it locally or upload it to Google Drive (which requires you to authorize a Google account). - -:::caution -OpenRefine project archives contain confidential data from previous steps, which will still be accessible to anyone who has the archive. If you are hoping to keep your original dataset hidden for privacy reasons, such as using OpenRefine to anonymize information, do not share your project archive. -::: - -To save your project archive locally: from the Export dropdown, select OpenRefine project archive to file. OpenRefine exports your full project with all of its history. It does not export any current views or applied facets. Existing reconciliation information will be preserved, but the importing computer will need to add the same reconciliation services to keep working with that data. - -OpenRefine exports files in `.tar.gz` format. You can rename the file when you save it; otherwise it will bear the project name. - -To save your project archive to Google Drive: from the Export dropdown, select OpenRefine project archive to Google Drive.... OpenRefine will not share the link with you, only confirm that the file was uploaded. - -## Export operations {#export-operations} - -You can [save and re-apply the history of any project](running#reusing-operations) (all the operations shown in the Undo/Redo tab). This creates JSON that you can save for later reuse on another OpenRefine project. diff --git a/OpenRefine/docs/docs/manual/expressions.md b/OpenRefine/docs/docs/manual/expressions.md deleted file mode 100644 index d10420378..000000000 --- a/OpenRefine/docs/docs/manual/expressions.md +++ /dev/null @@ -1,208 +0,0 @@ ---- -id: expressions -title: Expressions -sidebar_label: Overview ---- - -## Overview {#overview} - -You can use expressions in multiple places in OpenRefine to extend data cleanup and transformation. Expressions are available with the following functions: -* Facet: - - * Custom text facet... - * Custom numeric facet… - * Customized facets (click “change” after they have been created to bring up an expressions window) -* Edit cells: - - * Transform… - * Split multi-valued cells… - * Join multi-valued cells… -* Edit column: - - * Split - * Join - * Add column based on this column - * Add column by fetching URLs. - -In the expressions editor window you have the opportunity to select a supported language. The default is [GREL (General Refine Expression Language)](grel); OpenRefine also comes with support for [Clojure](jythonclojure#clojure) and [Jython](jythonclojure#jython). Extensions may offer support for more expressions languages. - -These languages have some syntax differences but support many of the same [variables](#variables). For example, the GREL expression `value.split(" ")[1]` would be written in Jython as `return value.split(" ")[1]`. - -This page is a general reference for available functions, variables, and syntax. For examples that use these expressions for common data tasks, look at the [Recipes section on the wiki](https://github.com/OpenRefine/OpenRefine/wiki/Documentation-For-Users#recipes-and-worked-examples). - -## Expressions {#expressions} - -There are significant differences between OpenRefine's expressions and the spreadsheet formulas you may be used to using for data manipulation. OpenRefine does not store formulas in cells and display output dynamically: OpenRefine’s transformations are one-time operations that can change column contents or generate new columns. These are applied using variables such as `value` or `cell` to perform the same modification to each cell in a column. - -Take the following example: - -|ID|Friend|Age| -|---|---|---| -|1.|John Smith|28| -|2.|Jane Doe|33| - -Were you to apply a transformation to the “friend” column with the expression - -``` - value.split(" ")[1] -``` - -OpenRefine would work through each row, splitting the “friend” values based on a space character. The `value` for row 1 is “John Smith” so the output would be “Smith” (as "[1]" selects the second part of the created output); the `value` for row 2 is “Jane Doe” so the output would be “Doe”. Using variables, a single expression yields different results for different rows. The old information would be discarded; you couldn't get "John" and "Jane" back unless you undid the operation in the [History](running#history-undoredo) tab. - -For another example, if you were to create a new column based on your data using the expression `row.starred`, it would generate a column of true and false values based on whether your rows were starred at that moment. If you were to then star more rows and unstar some rows, that data would not dynamically update - you would need to run the operation again to have current true/false values. - -Note that an expression is typically based on one particular column in the data - the column whose drop-down menu is first selected. Many variables are created to stand for things about the cell in that “base column” of the current row on which the expression is evaluated. There are also variables about rows, which you can use to access cells in other columns. - -## The expressions editor {#the-expressions-editor} - -When you select a function that accepts expressions, you will see a window overlay the screen with what we call the expressions editor. - -![The expressions editor window with a simple expression: value + 10.](/img/expression-editor.png) - -The expressions editor offers you a field for entering your formula and shows you a preview of its transformation on your first few rows of cells. - -There is a dropdown menu from which you can choose an expression language. The default at first is GREL; if you begin working with another language, that selection will persist across OpenRefine. Jython and Clojure are also offered with the installation package, and you may be able to add more language support with third-party extensions and customizations. - -There are also tabs for: -* History, which shows you formulas you’ve recently used from across all your projects -* Starred, which shows you formulas from your History that you’ve starred for reuse -* Help, a quick reference to GREL functions. - -Starring formulas you’ve used in the past can be helpful for repetitive tasks you’re performing in batches. - -You can also choose how formula errors are handled: replicate the original cell value, output an error message into the cell, or ouput a blank cell. - -## Regular expressions {#regular-expressions} - -OpenRefine offers several fields that support the use of regular expressions (regex), such as in a Text filter or a Replace… operation. GREL and other expressions can also use regular expression markup to extend their functionality. - -If this is your first time working with regex, you may wish to read [this tutorial specific to the Java syntax that OpenRefine supports](https://docs.oracle.com/javase/tutorial/essential/regex/). We also recommend this [testing and learning tool](https://regexr.com/). - -### GREL-supported regex {#grel-supported-regex} - -To write a regular expression inside a GREL expression, wrap it between a pair of forward slashes (/) much like the way you would in Javascript. For example, in - -``` -value.replace(/\s+/, " ") -``` - -the regular expression is `\s+`, and the syntax used in the expression wraps it with forward slashes (`/\s+/`). Though the regular expression syntax in OpenRefine follows that of Java (normally in Java, you would write regex as a string and escape it like "\\s+"), a regular expression within a GREL expression is similar to Javascript. - -Do not use slashes to wrap regular expressions outside of a GREL expression. - -On the [GREL functions](grelfunctions) page, functions that support regex will indicate that with a “p” for “pattern.” The GREL functions that support regex are: -* [contains](grelfunctions#containss-sub-or-p) -* [replace](grelfunctions#find-and-replace) -* [find](grelfunctions#find-and-replace) -* [match](grelfunctions#matchs-p) -* [partition](grelfunctions#partitions-s-or-p-fragment-b-omitfragment-optional) -* [rpartition](grelfunctions#rpartitions-s-or-p-fragment-b-omitfragment-optional) -* [split](grelfunctions#splits-s-or-p-sep-b-preservetokens-optional) -* [smartSplit](grelfunctions#smartsplits-s-or-p-sep-optional) - -### Jython-supported regex {#jython-supported-regex} - -You can also use [regex with Jython expressions](http://www.jython.org/docs/library/re.html), instead of GREL, for example with a Custom Text Facet: - -``` -python import re g = re.search(ur"\u2014 (.*),\s*BWV", value) return g.group(1) -``` - -### Clojure-supported regex {#clojure-supported-regex} - -[Clojure](https://clojure.org/reference/reader) uses the same regex engine as Java, and can be invoked with [re-find](http://clojure.github.io/clojure/clojure.core-api.html#clojure.core/re-find), [re-matches](http://clojure.github.io/clojure/clojure.core-api.html#clojure.core/re-matches), etc. You can use the #"pattern" reader macro as described [in the Clojure documentation](https://clojure.org/reference/other_functions#regex). For example, to get the nth element of a returned sequence, you can use the nth function: - -``` -clojure (nth (re-find #"\u2014 (.*),\s*BWV" value) 1) -``` - -## Variables {#variables} - -Most OpenRefine variables have attributes: aspects of the variables that can be called separately. We call these attributes “member fields” because they belong to certain variables. For example, you can query a record to find out how many rows it contains with `row.record.rowCount`: `rowCount` is a member field specific to the `record` variable, which is a member field of `row`. Member fields can be called using a dot separator, or with square brackets (`row["record"]`). The square bracket syntax is also used for variables that can call columns by name, for example, `cells["Postal Code"]`. - -|Variable |Meaning | -|-|-| -| `value` | The value of the cell in the current column of the current row (can be null) | -| `row` | The current row | -| `row.record` | One or more rows grouped together to form a record | -| `cells` | The cells of the current row, with fields that correspond to the column names (or row.cells) | -| `cell` | The cell in the current column of the current row, containing value and other attributes | -| `cell.recon` | The cell's reconciliation information returned from a reconciliation service or provider | -| `rowIndex` | The index value of the current row (the first row is 0) | -| `columnName` | The name of the current cell's column, as a string | - -### Row {#row} - -The `row` variable itself is best used to access its member fields, which you can do using either a dot operator or square brackets: `row.index` or `row["index"]`. - -|Field |Meaning | -|-|-| -| `row.index` | The index value of the current row (the first row is 0) | -| `row.cells` | The cells of the row, returned as an array | -| `row.columnNames` | An array of the column names of the project. This will report all columns, even those with null cell values in that particular row. Call a column by number with `row.columnNames[3]` | -| `row.starred` | A boolean indicating if the row is starred | -| `row.flagged` | A boolean indicating if the row is flagged | -| `row.record` | The [record](#record) object containing the current row | - -For array objects such as `row.columnNames` you can preview the array using the expressions window, and output it as a string using `toString(row.columnNames)` or with something like: - -``` -forEach(row.columnNames,v,v).join("; ") -``` - -### Cells {#cells} - -The `cells` object is used to call information from the columns in your project. For example, `cells.Foo` returns a [cell](#cell) object representing the cell in the column named “Foo” of the current row. If the column name has spaces, use square brackets, e.g., `cells["Postal Code"]`. To get the corresponding column's value inside the `cells` variable, use `.value` at the end, for example, `cells["Postal Code"].value`. There is no `cells.value` - it can only be used with member fields. - -### Cell {#cell} - -A `cell` object contains all the data of a cell and is stored as a single object. - -You can use `cell` on its own in the expressions editor to copy all the contents of a column to another column, including reconciliation information. Although the preview in the expressions editor will only show a small representation (“[object Cell]”), it will actually copy all the cell's data. Try this with Edit ColumnAdd Column based on this column .... - -|Field |Meaning |Member fields | -|-|-|-| -| `cell` | An object containing the entire contents of the cell | .value, .recon, .errorMessage | -| `cell.value` | The value in the cell, which can be a string, a number, a boolean, null, or an error | | -| `cell.recon` | An object encapsulating reconciliation results for that cell | See the [reconciliation](expressions#reconciliation) section | -| `cell.errorMessage` | Returns the message of an *EvalError* instead of the error object itself (use value to return the error object) | .value | - -### Reconciliation {#reconciliation} - -Several of the fields here provide the data used in [reconciliation facets](reconciling#reconciliation-facets). You must type `cell.recon`; `recon` on its own will not work. - -|Field|Meaning |Member fields | -|-|-|-| -| `cell.recon.judgment` | A string: either “matched”, "new”, "none” | | -| `cell.recon.judgmentAction` | A string: either "single” or “similar” (or “unknown”) | | -| `cell.recon.judgmentHistory` | A number, the epoch timestamp (in milliseconds) of your judgment | | -| `cell.recon.matched` | A boolean, true if judgment is “matched” | | -| `cell.recon.match` | The recon candidate that has been matched against this cell (or null) | .id, .name, .type | -| `cell.recon.best` | The highest scoring recon candidate from the reconciliation service (or null) | .id, .name, .type, .score | -| `cell.recon.features` | An array of reconciliation features to help you assess the accuracy of your matches | .typeMatch, .nameMatch, .nameLevenshtein, .nameWordDistance | -| `cell.recon.features.typeMatch` | A boolean, true if your chosen type is “matched” and false if not (or “(no type)” if unreconciled) | | -| `cell.recon.features.nameMatch` | A boolean, true if the cell and candidate strings are identical and false if not (or “(unreconciled)”) | | -| `cell.recon.features.nameLevenshtein` | A number representing the [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance): larger if the difference is greater between value and candidate | | -| `cell.recon.features.nameWordDistance` | A number based on the [word similarity](reconciling#reconciliation-facets) | | -| `cell.recon.candidates` | An array of the top 3 candidates (default) | .id, .name, .type, .score | - -The `cell.recon.candidates` and `cell.recon.best` objects have a few deeper fields: `id`, `name`, `type`, and `score`. `type` is an array of type identifiers for a list of candidates, or a single string for the best candidate. - -Arrays such as `cell.recon.candidates` and `cell.recon.candidates.type` can be joined into lists and stored as strings with something like: -``` -forEach(cell.recon.candidates,v,v.name).join("; ") -``` - -### Record {#record} - -A `row.record` object encapsulates one or more rows that are grouped together, when your project is in records mode. You must call it as `row.record`; `record` will not return values. - -|Field|Meaning | -|-|-| -| `row.record.index` | The index of the current record (starting at 0) | -| `row.record.cells` | An array of the [cells](#cells) in the given column of the record | -| `row.record.fromRowIndex` | The row index of the first row in the record | -| `row.record.toRowIndex` | The row index of the last row in the record + 1 (i.e. the next record) | -| `row.record.rowCount` | A count of the number of rows in the record | - -For example, you can facet by number of rows in each record by creating a Custom Numeric Facet (or a Custom Text Facet) and entering `row.record.rowCount`. \ No newline at end of file diff --git a/OpenRefine/docs/docs/manual/facets.md b/OpenRefine/docs/docs/manual/facets.md deleted file mode 100644 index 06e54b751..000000000 --- a/OpenRefine/docs/docs/manual/facets.md +++ /dev/null @@ -1,329 +0,0 @@ ---- -id: facets -title: Exploring facets -sidebar_label: Facets ---- - -## Overview {#overview} - -Facets are one of OpenRefine’s strongest features - that’s where the diamond logo comes from! - -Faceting allows you to look for patterns and trends. Facets are essentially aspects or angles of data variance in a given column. For example, if you had survey data where respondents indicated one of five responses from “Strongly agree” to “Strongly disagree,” those five responses make up a text facet, showing how many people selected each option. - -Faceted browsing gives you a big-picture look at your data (do they agree or disagree?) and also allows you to filter down to a specific subset to explore it more (what do people who disagree say in other responses?). - -Typically, you create a facet on a particular column. That facet selection appears on the left, in the Facet/Filter tab, and you can click on a displayed facet to view all the records that match. You can also “exclude” the facet, to view every record that does _not_ match, and you can select more than one facet by clicking “include.” - - -### An example {#an-example} - -You can learn about facets and filtering with the following example. You can copy the following table and paste it using the Clipboard method of starting a project if you would like to try it yourself. - -We collected a list of the [10 most populous cities from Wikidata](https://w.wiki/3Em), using an example query of theirs. We removed the GPS coordinates and added the country. - -| cityLabel | population | countryLabel | -|-|-|-| -| Shanghai | 23390000 | People's Republic of China | -| Beijing | 21710000 | People's Republic of China | -| Lagos | 21324000 | Nigeria | -| Dhaka | 16800000 | Bangladesh | -| Mumbai | 15414288 | India | -| Istanbul | 14657434 | Turkey | -| Tokyo | 13942856 | Japan | -| Tianjin | 13245000 | People's Republic of China | -| Guangzhou | 13080500 | People's Republic of China | -| São Paulo | 12106920 | Brazil | - -If we want to see which countries have the most populous cities, we can create a text facet on the “countryLabel” column and OpenRefine will generate a list of all the different strings used in these cells. - -We will see in the sidebar that the countries identified are displayed, along with the number of matches (the “count”). We can sort this list alphabetically or by the count. If you sort by count at the top of the facet window, you’ll learn which countries hold the most populous cities. - -|Facet|Count| -|---|---| -|People's Republic of China|4| -|Bangladesh|1| -|Brazil|1| -|India|1| -|Japan|1| -|Nigeria|1| -|Turkey|1| - -If we want to learn more about a particular country, we can click on its appearance in the facet sidebar. This narrows our dataset down temporarily to only rows matching that facet. - -You’ll see the “10 rows” indicator change to “4 matching rows (10 total)” if you click on “People’s Republic of China”. In the data grid, you’ll see fewer rows: only the ones matching your current filter. Each row will maintain its original numbering, though - in this case, rows #1, 2, and 8. - -If you want to go back to the original dataset, click Reset All or the small “exclude” text next to the facet. If you want to view the most populous cities in both China and India, click “include” next to each facet. Now you’ll see 5 rows - #1, 2, 5, 8, 9. - -We can also explore our data using the population information. In this case, because population is a number, we can create a numeric facet. This will give us the ability to explore by range rather than by exact matching values. - -With the numeric facet, we are given a scale from the smallest to the largest value in the column. We can drag the range minimum and maximum to narrow the results. In this case, if we narrow down to only cities with more than 20 million in population, we get 3 matching rows out of the original 10. - -When you look back at the text facet display of country names, you should see a smaller list with a reduced count: OpenRefine is now displaying the facets of the 3 matching rows, not the total dataset of 10 rows. - -We can combine these facets - say, by narrowing to only the Chinese cities with populations greater than 20 million - simply by clicking in both. You should see 2 matching rows for both these criteria. - -### Things to know about facets {#things-to-know-about-facets} - -When you have facets applied, you will see “matching rows” in the [project grid header](running#project-grid-header). If you click Export and copy your data out of OpenRefine while facets are active, many of the exporting options will only export the matching rows, not all the rows in your project. - -OpenRefine has several default facets, which you’ll learn about below. The most powerful facets are the ones designed by you - custom facets, written using [expressions](expressions) to transform the data behind the scenes and help you narrow down to precisely what you’re looking for. - -Facets are not saved in the project along with the data. But you can save a link to the current state of the application. Find the [Permalink](running#the-project-bar) next to the project’s name. - -You can modify any facet expression by clicking the “change” button to the right of the column name in the facet sidebar. - -Facet boxes that appear in the sidebar can be resized and rearranged. You can drag and drop the title bar of each box to reorder them, and drag on the bottom bar of text facet boxes. - -## Text facet {#text-facet} - -A text facet can be generated on any column with the “text” data type. Select the column dropdown and go to FacetText facet. The created facet will be sorted alphabetically, and can be sorted by count. - -A text facet is very simple: it takes the total contents of the cells of the column in question and matches them up. It does no guessing about typos or near-matches. - -You can edit any entry that appears in the facet display, by hovering over the facet and clicking the “edit” button that appears. You can then type in a new value manually. This will mass-edit every identical cell in the column. This is a great way to fix typos, whitespace, and other issues that may be affecting the way facets appear. You can also automate the cleanup of facets by using [clustering](transforming#cluster-and-edit): a “Cluster” button is displayed within the facet window. It may be most efficient to cluster cells to one value, and then mass-edit that value to your desired string within the clustering operation window. - -Each text facet shows up to 2,000 choices by default. You can [increase this limit on the Preferences screen](running#preferences) if you need to, which may slow down your browser. If your applied facet has more choices than the current limit, you'll be offered the option to increase the limit, which will permanently edit that preference for you. - -The choices and counts displayed in each facet can be copied as tab-separated values. To do so, click on the "X choices" link near the top left corner of the facet. This can be useful to generate small summary tables of your data. - -![A column of years faceted as text and numbers, and with the count ready to be copied.](/img/yeardata.png) - -## Numeric facet {#numeric-facet} - -![A screenshot of an example numeric facet.](/img/numericfacet.png) - -Whereas a text facet groups unique text values into groups, a numeric facet sorts numbers by their range - smallest to biggest. This displays visually as a histogram, and allows you to set a custom facet within that range. You can drag the minimum and maximum range markers to set a range. OpenRefine snaps to some basic equal-sized divisions - 19 in the example set above. - -You will be offered the option to include blank, non-numeric, and error values in your numeric visualization; these will appear in the visual range as “0” values. - -:::info Numbers as text -You can create a text facet on numeric data, which will treat each entry as a string. This can be useful if you wish, for example, to manually include facets instead of selecting a range, or sort by count, or copy that count. -::: - -:::info Faceting customization -As mentioned in the overview, facets can be modified or customized by GREL [expressions](expressions) in many ways. For example, to facet by clusters of [row](expressions#variables) numbers with `row.index/100` or better visualizing numbers greater than 1000 with `max(row.index, 1000)`. -::: - -## Timeline facet {#timeline-facet} - -![A screenshot of an example timeline facet.](/img/timelinefacet.png) - -Much like a numeric facet, a timeline facet will display as a small histogram with the values sorted: in this case, chronologically. A timeline facet only works on cells formatted as the [“date” data type](exploring#dates). - -The facet appears with a count of blank cells and those with errors, which can help you analyze whether your date cells are correctly converted. - -## Scatterplot facet {#scatterplot-facet} - -A scatterplot is a visual representation of two related sets of numeric data. - -You have the option to generate linear scatterplots (where the X and Y axes show continuous increases) or logarithmic scatterplots (where the X and Y axes show exponential or scaled increases). You can also rotate the plot by 45 degrees in either direction, and you can choose the size of the dot indicating a datapoint. You can make these choices in both the preview and in the facet display. - -A scatterplot facet can be generated on any column. You require two or more number columns to generate scatterplots. Selecting FacetScatterplot facet will create a preview of data plotted from every number-formatted column in your dataset, comparing every column against every other column. Each scatterplot will show in its own square, allowing you to choose which data comparison you would like to analyze further. You can control which columns are on the X and Y axes by rearranging the columns in your dataset. - -![A simple scatterplot of two numeric values.](/img/scatterplot.png) - -When you click on your desired square, that two-column comparison will appear in the facets sidebar. From here, you can drag your mouse to draw a rectangle inside the scatterplot, which will narrow down to just the rows matching the points plotted inside that rectangle (as shown by the rectangle inside the square in the image above). This rectangle can be resized by dragging any of the four edges. To draw a new rectangle, simply click and drag your mouse again. To add more scatterplots to the facet sidebar, re-run this process and select a different square. - -If you have multiple facets applied, plotted points in your scatterplot displays will be greyed out if they are not part of the current matching data subset. If the rectangle you have drawn within a scatterplot display only includes grey dots, you will see no matching rows. - -If you would like to export a scatterplot, OpenRefine will open a new tab with a generated PNG file that you can save. - -## Custom text facet {#custom-text-facet} - -You may want to explore your textual data with modifications that aren't permanent. Creating custom text facets will load your column into memory, transform the data temporarily, and store those transformations inside the facet. - -You can also use custom text facets to analyze numerical data, such as by analyzing a number as a string, or by creating a test that will return “true” and “false” as values. - -Clicking on FacetCustom text facet… will bring up an [expressions](expressions) window where you can enter in a GREL, Jython, or Clojure expression to modify how the facet works. - -A custom text facet operates just like a [text facet](#text-facet) by default. Unlike a text facet, however, you cannot click “edit” on the facets that appear in the sidebar and change the matching cells in your dataset - because what they display is modified, not the original entries. - -For example, you may wish to analyze only the first word in a text field - perhaps the first name in a column of “[First Name] [Last Name]” entries. In this case, you can tell OpenRefine to facet only on the information that comes before the first space: - -``` -value.split(" ")[0] -``` - -In this case, `split()` is creating an array of text strings based on every space in the cells ["Firstname", "Lastname"]. Because arrays number their entries starting with 0, we want the first value, so we ask for `[0]`. (Assuming the first name is one word, not something like “Mary Anne.”) We can do the same splitting and ask for the last name with - -``` -value.split(" ")[1] -``` - -You may want to create a facet that references several columns. For example, let’s say you have two columns, “First Name” and “Last Name”, and you want out how many people have the same initial letter for both names (e.g., Marilyn Monroe, Steven Segal). To do so, create a custom text facet on either column and enter the expression - -``` -cells["First Name"].value[0] == cells["Last Name"].value[0] -``` - -That expression will look for the first letter (the character at index 0) of each entry and compare them. Then it will facet your rows into “true” and “false.” - -You can learn more about text-modification functions on the [Expressions page](expressions). - -## Custom numeric facet {#custom-numeric-facet} - -You may want to explore your numerical data with modifications that aren't permanent. You can also use custom numeric facets to analyze textual data, such as by getting the length of text strings (with `value.length()`), or by analyzing it as though it were formatted as numbers (with `toNumber(value)`). - -If you would like to build your own version of a numeric facet, you can use the Custom Numeric Facet option. Clicking on FacetCustom Numeric Facet… will bring up an [expressions](expressions) window where you can enter in a GREL, Jython, or Clojure expression to modify how the facet works. A custom numeric facet operates just like a [numeric facet](#numeric-facet) by default. - -For example, you may wish to create a numeric facet that rounds your value to the nearest integer, enter - -``` -round(value) -``` - -If you have two columns of numbers and for each row you wish to create a numeric facet only on the larger of the two, enter - -``` -max(cells["Column1"].value, cells["Column2"].value) -``` - -If the numeric values in a column are drawn from a power law distribution, then it's better to group them by their logs: - -``` -value.log() -``` - -If the values are periodic you could take the modulus by the period to understand if there's a pattern: - -``` -mod(value, 7) -``` - -You can learn more about numeric-modification functions on the [Expressions page](expressions). - -## Customized facets {#customized-facets} - -Customized facets have been added to expand the number of default facets users can apply with a single click. They represent some common and useful functions you shouldn’t have to work out using an [expression](expressions). - -All facets that display in the Facet/Filter tab can be edited by clicking on the “change” button to the right of the column title. This brings up the expressions window that will allow you to modify and preview the expression being used. - -### Word facet {#word-facet} - -A Word facet is a simple version of a text facet: it splits up the content of the cells based on spaces, and outputs each character string as a facet: - -``` -value.split(" ") -``` - -This can be useful for exploring the language used in a corpus, looking for common first and last names or titles, or seeing what’s in multi-valued cells you don’t wish to split up. - -Word facet is case-sensitive and only splits by spaces, not by line breaks or other natural divisions. - -### Duplicates facet {#duplicates-facet} - -A Duplicates facet will return only rows that have non-unique values in the column you’ve selected. It will create a facet of “true” and “false” values - true being cells that are not unique, and “false” being cells that are. The actual expression being used is - -``` -facetCount(value, 'value', '[Column]') > 1 -``` - -Duplicates facets are case-sensitive and you may wish to filter out things like leading and trailing whitespace or other hard-to-see issues. You can modify the facet expression, for example, with: - -``` -facetCount(trim(toLowercase(value)), 'trim(toLowercase(value))', 'cityLabel') > 1 -``` - -### Numeric log facet {#numeric-log-facet} - -Logarithmic scales reduce wide-ranging quantities to more compact and manageable ranges. A log transformation can be used to make highly skewed distributions less skewed. If your numerical data is unevenly distributed (say, lots of values in one range, and then a long tail extending off into different magnitudes), a Numeric log facet can represent that range better than a simple numeric facet. It will break these values down into more navigable segments than the buckets of a numeric facet. This facet can make patterns in your data more visible. OpenRefine uses a base-10 log, the “common logarithm.” - -For example, we can look at [this data about the body weight of various mammals](http://wiki.stat.ucla.edu/socr/index.php/SOCR_Data_Brain2BodyWeight): - -|Species|BodyWeight (kg)| -|---|---| -| Newborn_Human | 3.2 | -| Adult_Human | 73 | -| Pithecanthropus_Man | 70 | -| Squirrel | 0.8 | -| Hamster | 0.15 | -| Chimpanzee | 50 | -| Rabbit | 1.4 | -| Dog_(Beagle) | 10 | -| Cat | 4.5 | -| Rat | 0.4 | -| Sperm_Whale | 35000 | -| Turtle | 3 | -| Alligator | 270 | - -Most values will be clustered in the 0-100 range, but 35,000 is many magnitudes above that. A numeric facet will create 36 equal buckets of 1,000 each - containing almost all the cells in the first bucket. A numeric log facet will instead display the data more evenly across the visual range. - -![A screenshot of a numeric facet first and a numeric log facet second.](/img/numericlogfacet.png) - -A 1-bounded numeric log facet can be used if you'd like to exclude all the values below 1 (including zero and negative numbers). - -### Text-length facet {#text-length-facet} - -The Text-length facet returns a numerical value for each cell and plots it on a numeric facet chart. The expression used is - -``` -value.length() -``` - -This can be useful to, for example, look for values that did not successfully split on an earlier split operation, or to validate that data is a certain expected length (such as whether a date in YYYY/MM/DD is eight to ten characters). - -You can also employ a Log of text-length facet that allows you to navigate more easily a wide range of string lengths. This can be useful in the case of web-scraping, where lots of textual data is loaded into single cells and needs to be parsed out. - - -### Unicode character-code facet {#unicode-character-code-facet} - -![A screenshot of the Unicode facet.](/img/unicodefacet.png) - -The Unicode facet identifies and returns [Unicode decimal values](https://en.wikipedia.org/wiki/List_of_Unicode_characters). It generates a list of the Unicode numerical values of each character used in each text cell, which allows you to narrow down and search for special characters, punctuation, and other data formatting issues. - -This facet creates a numerical chart, which offers you the ability to narrow down to a range of numbers. For example, lowercase characters are numbers 97-122, uppercase characters are numbers 65-90, and numerical digits are numbers 48-57. - -### Facet by error {#facet-by-error} - -An error is a data type created by OpenRefine in the process of transforming data. For example, say you had converted a column to the number data type. If one cell had text characters in it, OpenRefine could either output the original text string unchanged or output an error. If you allow errors to be created, you can facet by them later to search for them and fix them. - -![A view of the expressions window with an error converting a string to a number.](/img/error.png) - -To store errors in cells, ensure that you have store error selected for the “On error” option in the expressions window. - -### Facet by null, empty, or blank {#facet-by-null-empty-or-blank} - -Any column can be faceted for [null and/or empty cells](#cell-data-types). These can help you find cells where you want to manually enter content. - -“Blank” means both null values and empty values. All three facets will generate “true” and “false” facets, “true” being blank. - -An empty cell is a cell that is set to contain a string, but doesn’t have any characters in it (a zero-length string). This can be left over from an operation that removed characters, or from manually editing a cell and deleting its contents. - -### Facet by star or flag {#facet-by-star-or-flag} - -Stars and flags offer you the opportunity to mark specific rows for yourself for later focus. Stars and flags persist through closing and opening your project, and thus can provide a different function than using a permalink to persist your facets. Stars and flags can be used in any way you want, although they are designed to help you flag errors and star rows of particular importance. - -You can manually star or flag rows simply by clicking on the icons to the left of each row. - -You can also apply stars or flags to all matching rows by using the All dropdown menu (on the first column) and selecting Edit rowsStar rows or Flag rows. This will create “true” and “false” facets in the Facet/Filter. These operations will modify all matching rows in your current subset. You can unstar or unflag them as well. - -You may wish to create a custom subset of your data through a series of separate faceting activities (rather than successively narrowing down with multiple facets applied). For example, you may wish to: -* apply a facet -* star all the matching rows -* remove that facet -* apply another, unrelated facet -* star all the new matching rows (which will not modify already-starred rows) -* remove that facet -* and then work with all of the cumulative starred rows. - -You can also create a text facet on any column with the expression `row.starred` or `row.flagged`. - -## Text filter {#text-filter} - -Filters allow you to narrow down your data based on whether a given column includes a text string. - -When you choose Text filter a box appears in the Facet/Filter tab that allows you to enter in text. Matching rows will narrow dynamically with every character you enter. You can set the search to be case-sensitive or not, and you can use this box to enter in a regular expression. - -For example, you can enter in “side” as a text filter, and it will return all cells in that column containing “side,” “sideways,” “offside,” etc. - -The text filter field supports [regular expressions](expressions#regular-expressions). For example, you can employ a regular expression to view all properly-formatted emails: - -``` -([a-zA-Z0-9_\-\.\+]+)@([a-zA-Z0-9\-\.]+)\.([a-zA-Z0-9\-]{2,15}) -``` - -You can press “invert” on this facet to then see blank cells or invalid email addresses. - -This filter works differently than facets because it is always active as long as it appears in the sidebar. If you “reset” it, you will delete all the text or expression you have entered. - -You can apply multiple text filters in succession, which will successively narrow your data subset. This can be useful if you apply multiple inverted filters, such as to filter out all rows that respond “yes” or “maybe” and only look at the remaining responses. diff --git a/OpenRefine/docs/docs/manual/grel.md b/OpenRefine/docs/docs/manual/grel.md deleted file mode 100644 index 7ec90b464..000000000 --- a/OpenRefine/docs/docs/manual/grel.md +++ /dev/null @@ -1,154 +0,0 @@ ---- -id: grel -title: General Refine Expression Language -sidebar_label: General Refine Expression Language ---- - -## Basics {#basics} - -GREL (General Refine Expression Language) is designed to resemble Javascript. Formulas use variables and depend on data types to do things like string manipulation or mathematical calculations: - -|Example|Output| -|---|---| -| `value + " (approved)"` | Concatenate two strings; whatever is in the cell gets converted to a string first | -| `value + 2.239` | Add 2.239 to the existing value (if a number); append text "2.239" to the end of the string otherwise | -| `value.trim().length()`     | Trim leading and trailing whitespace of the cell value and then output the length of the result | -| `value.substring(7, 10)` | Output the substring of the value from character index 7, 8, and 9 (excluding character index 10) | -| `value.substring(13)` | Output the substring from index 13 to the end of the string | - -Note that the operator for string concatenation is `+` (not “&” as is used in Excel). - -Evaluating conditions uses symbols such as <, >, *, /, etc. To check whether two objects are equal, use two equal signs (`value=="true"`). - -See the [GREL functions page for a thorough reference](grelfunctions) on each function and its inputs and outputs. Read on below for more about the general nature of GREL expressions. - -## Syntax {#syntax} - -In GREL, functions can use either of these two forms: -* functionName(arg0, arg1, ...) -* arg0.functionName(arg1, ...) - -The second form is a shorthand to make expressions easier to read. It simply pulls the first argument out and appends it to the front of the function, with a dot: - -|Dot notation |Full notation | -|-|-| -| `value.trim().length()` | `length(trim(value))` | -| `value.substring(7, 10)` | `substring(value, 7, 10)` | - -So, in the dot shorthand, the functions occur from left to right in the order of calling, rather than in the reverse order with parentheses. This allows you to string together multiple functions in a readable order. - -The dot notation can also be used to access the member fields of [variables](expressions#variables). For referring to column names that contain spaces (anything not a continuous string), use square brackets instead of dot notation: - -|Example |Description | -|-|-| -| `FirstName.cells` | Access the cell in the column named “FirstName” of the current row | -| `cells["First Name"]` | Access the cell in the column called “First Name” of the current row | - -Square brackets can also be used to get substrings and sub-arrays, and single items from arrays: - -|Example |Description | -|-|-| -| `value[1,3]` | A substring of value, starting from character 1 up to but excluding character 3 | -| `"internationalization"[1,-2]` | Will return “nternationalizati” (negative indexes are counted from the end) | -| `row.columnNames[5]` | Will return the name of the fifth column | - -Any function that outputs an array can use square brackets to select only one part of the array to output as a string (remember that the index of the items in an array starts with 0). - -For example, [partition()](grelfunctions#partitions-s-or-p-fragment-b-omitfragment-optional) would normally output an array of three items: the part before your chosen fragment, the fragment you've identified, and the part after. Selecting only the third part with `"internationalization".partition("nation")[2]` will output “alization” (and so will [-1], indicating the final item in the array). - -## Controls {#controls} - -GREL offers controls to support branching and looping (that is, “if” and “for” functions), but unlike functions, their arguments don't all get evaluated before they get run. A control can decide which part of the code to execute and can affect the environment bindings. Functions, on the other hand, can't do either. Each control decides which of their arguments to evaluate to `value`, and how. - -Please note that the GREL control names are case-sensitive: for example, the isError() control can't be called with iserror(). - -#### if(e, eTrue, eFalse) {#ife-etrue-efalse} - -Expression e is evaluated to a value. If that value is true, then expression eTrue is evaluated and the result is the value of the whole if() expression. Otherwise, expression eFalse is evaluated and that result is the value. - -Examples: - -| Example expression | Result | -| ------------------------------------------------------------------------ | ------------ | -| `if("internationalization".length() > 10, "big string", "small string")` | “big string” | -| `if(mod(37, 2) == 0, "even", "odd")` | “odd” | - -Nested if (switch case) example: - - if(value == 'Place', 'http://www.example.com/Location', - - if(value == 'Person', 'http://www.example.com/Agent', - - if(value == 'Book', 'http://www.example.com/Publication', - - null))) - -#### with(e1, variable v, e2) {#withe1-variable-v-e2} - -Evaluates expression e1 and binds its value to variable v. Then evaluates expression e2 and returns that result. - -| Example expression | Result | -| ------------------------------------------------------------------------------------ | ---------- | -| `with("european union".split(" "), a, a.length())` | 2 | -| `with("european union".split(" "), a, forEach(a, v, v.length()))` | [ 8, 5 ] | -| `with("european union".split(" "), a, forEach(a, v, v.length()).sum() / a.length())` | 6.5 | - -#### filter(e1, v, e test) {#filtere1-v-e-test} - -Evaluates expression e1 to an array. Then for each array element, binds its value to variable v, evaluates expression test - which should return a boolean. If the boolean is true, pushes v onto the result array. - -| Expression | Result | -| ---------------------------------------------- | ------------- | -| `filter([ 3, 4, 8, 7, 9 ], v, mod(v, 2) == 1)` | [ 3, 7, 9 ] | - -#### forEach(e1, v, e2) {#foreache1-v-e2} - -Evaluates expression e1 to an array. Then for each array element, binds its value to variable v, evaluates expression e2, and pushes the result onto the result array. When e1 is a JSON object, `forEach` iterates over its keys. - -| Expression | Result | -| ------------------------------------------ | ------------------- | -| `forEach([ 3, 4, 8, 7, 9 ], v, mod(v, 2))` | [ 1, 0, 0, 1, 1 ] | - -#### forEachIndex(e1, i, v, e2) {#foreachindexe1-i-v-e2} - -Evaluates expression e1 to an array. Then for each array element, binds its index to variable i and its value to variable v, evaluates expression e2, and pushes the result onto the result array. - -| Expression | Result | -| ------------------------------------------------------------------------------- | --------------------------- | -| `forEachIndex([ "anne", "ben", "cindy" ], i, v, (i + 1) + ". " + v).join(", ")` | 1. anne, 2. ben, 3. cindy | - -#### forRange(n from, n to, n step, v, e) {#forrangen-from-n-to-n-step-v-e} - -Iterates over the variable v starting at from, incrementing by the value of step each time while less than to. At each iteration, evaluates expression e, and pushes the result onto the result array. - -#### forNonBlank(e, v, eNonBlank, eBlank) {#fornonblanke-v-enonblank-eblank} - -Evaluates expression e. If it is non-blank, forNonBlank() binds its value to variable v, evaluates expression eNonBlank and returns the result. Otherwise (if e evaluates to blank), forNonBlank() evaluates expression eBlank and returns that result instead. - -Unlike other GREL functions beginning with “for,” forNonBlank() is not iterative. forNonBlank() essentially offers a shorter syntax to achieving the same outcome by using the isNonBlank() function within an “if” statement. - -#### isBlank(e), isNonBlank(e), isNull(e), isNotNull(e), isNumeric(e), isError(e) {#isblanke-isnonblanke-isnulle-isnotnulle-isnumerice-iserrore} - -Evaluates the expression e, and returns a boolean based on the named evaluation. - -Examples: - -| Expression | Result | -| ------------------- | ------- | -| `isBlank("abc")` | false | -| `isNonBlank("abc")` | true | -| `isNull("abc")` | false | -| `isNotNull("abc")` | true | -| `isNumeric(2)` | true | -| `isError(1)` | false | -| `isError("abc")` | false | -| `isError(1 / 0)` | true | - -Remember that these are controls and not functions: you can’t use dot notation (for example, the format `e.isX()` will not work). - -## Constants {#constants} -|Name |Meaning | -|-|-| -| true | The boolean constant true | -| false | The boolean constant false | -| PI | From [Java's Math.PI](https://docs.oracle.com/javase/8/docs/api/java/lang/Math.html#PI), the value of pi (that is, 3.1415...) | diff --git a/OpenRefine/docs/docs/manual/grelfunctions.md b/OpenRefine/docs/docs/manual/grelfunctions.md deleted file mode 100644 index 5e4893851..000000000 --- a/OpenRefine/docs/docs/manual/grelfunctions.md +++ /dev/null @@ -1,535 +0,0 @@ ---- -id: grelfunctions -title: GREL functions -sidebar_label: GREL functions ---- - -## Reading this reference {#reading-this-reference} - -For the reference below, the function is given in full-length notation and the in-text examples are written in dot notation. Shorthands are used to indicate the kind of [data type](exploring#data-types) used in each function: s for string, b for boolean, n for number, d for date, a for array, p for a regex pattern, and o for object (meaning any data type), as well as “null” and “error” data types. - -If a function can take more than one kind of data as input or can output more than one kind of data, that is indicated with more than one letter (as with “s or a”) or with o for object, meaning it can take any type of data (string, boolean, date, number, etc.). - -We also use shorthands for substring (“sub”) and separator string (“sep”). -Optional arguments will say “(optional)”. - -In places where OpenRefine will accept a string (s) or a regex pattern (p), you can supply a string by putting it in quotes. If you wish to use any [regex](expressions#regular-expressions) notation, wrap the pattern in forward slashes. - -## Boolean functions {#boolean-functions} - -###### and(b1, b2, ...) {#andb1-b2-} - -Uses the logical operator AND on two or more booleans to output a boolean. Evaluates multiple statements into booleans, then returns true if all of the statements are true. For example, `(1 < 3).and(1 < 0)` returns false because one condition is true and one is false. - -###### or(b1, b2, ...) {#orb1-b2-} - -Uses the logical operator OR on two or more booleans to output a boolean. For example, `(1 < 3).or(1 > 7)` returns true because at least one of the conditions (the first one) is true. - -###### not(b) {#notb} - -Uses the logical operator NOT on a boolean to output a boolean. For example, `not(1 > 7)` returns true because 1 > 7 itself is false. - -###### xor(b1, b2, ...) {#xorb1-b2-} - -Uses the logical operator XOR (exclusive-or) on two or more booleans to output a boolean. Evaluates multiple statements, then returns true if only one of them is true. For example, `(1 < 3).xor(1 < 7)` returns false because more than one of the conditions is true. - -## String functions {#string-functions} - -###### length(s) {#lengths} - -Returns the length of string s as a number. - -###### toString(o, string format (optional)) {#tostringo-string-format-optional} - -Takes any value type (string, number, date, boolean, error, null) and gives a string version of that value. - -You can use toString() to convert numbers to strings with rounding, using an [optional string format](https://docs.oracle.com/javase/8/docs/api/java/util/Formatter.html). For example, if you applied the expression `value.toString("%.0f")` to a column: - -|Input|Output| -|-|-| -|3.2|3| -|0.8|1| -|0.15|0| -|100.0|100| - -You can also convert dates to strings, using date parsing syntax built into OpenRefine (see [the toDate() function for details](#todateo-b-monthfirst-s-format1-s-format2-)). For example, `value.toString("MMM-dd-yyyy")` would convert the date value [2024-10-15T00:00:00Z] to “Oct-15-2024”. - -Note: In OpenRefine, using toString() on a null cell outputs the string “null”. - -### Testing string characteristics {#testing-string-characteristics} - -###### startsWith(s, sub) {#startswiths-sub} - -Returns a boolean indicating whether s starts with sub. For example, `"food".startsWith("foo")` returns true, whereas `"food".startsWith("bar")` returns false. - -###### endsWith(s, sub) {#endswiths-sub} - -Returns a boolean indicating whether s ends with sub. For example, `"food".endsWith("ood")` returns true, whereas `"food".endsWith("odd")` returns false. - -###### contains(s, sub or p) {#containss-sub-or-p} - -Returns a boolean indicating whether s contains sub, which is either a substring or a regex pattern. For example, `"food".contains("oo")` returns true whereas `"food".contains("ee")` returns false. - -You can search for a regular expression by wrapping it in forward slashes rather than quotes: `"rose is a rose".contains(/\s+/)` returns true. startsWith() and endsWith() can only take strings, while contains() can take a regex pattern, so you can use contains() to look for beginning and ending string patterns. - -### Basic string modification {#basic-string-modification} - -#### Case conversion {#case-conversion} - -###### toLowercase(s) {#tolowercases} - -Returns string s converted to all lowercase characters. - -###### toUppercase(s) {#touppercases} - -Returns string s converted to all uppercase characters. - -###### toTitlecase(s) {#totitlecases} - -Returns string s converted into titlecase: a capital letter starting each word, and the rest of the letters lowercase. For example, `"Once upon a midnight DREARY".toTitlecase()` returns the string “Once Upon A Midnight Dreary”. - -#### Trimming {#trimming} - -###### trim(s) {#trims} - -Returns a copy of the string s with leading and trailing whitespace removed. For example, `" island ".trim()` returns the string “island”. Identical to strip(). - -###### strip(s) {#strips} - -Returns a copy of the string s with leading and trailing whitespace removed. For example, `" island ".strip()` returns the string “island”. Identical to trim(). - -###### chomp(s, sep) {#chomps-sep} - -Returns a copy of string s with the string sep removed from the end if s ends with sep; otherwise, just returns s. For example, `"barely".chomp("ly")` and `"bare".chomp("ly")` both return the string “bare”. - -#### Substring {#substring} - -###### substring(s, n from, n to (optional)) {#substrings-n-from-n-to-optional} - -Returns the substring of s starting from character index from, and up to (excluding) character index to. If the to argument is omitted, substring will output to the end of s. For example, `"profound".substring(3)` returns the string “found”, and `"profound".substring(2, 4)` returns the string “of”. - -Remember that character indices start from zero. A negative character index counts from the end of the string. For example, `"profound".substring(0, -1)` returns the string “profoun”. - -###### slice(s, n from, n to (optional)) {#slices-n-from-n-to-optional} - -Identical to substring() in relation to strings. Also works with arrays; see [Array functions section](#slicea-n-from-n-to-optional). - -###### get(s, n from, n to (optional)) {#gets-n-from-n-to-optional} - -Identical to substring() in relation to strings. Also works with named fields. Also works with arrays; see [Array functions section](#geta-n-from-n-to-optional). - -#### Find and replace {#find-and-replace} - -###### indexOf(s, sub) {#indexofs-sub} - -Returns the first character index of sub as it first occurs in s; or, returns -1 if s does not contain sub. For example, `"internationalization".indexOf("nation")` returns 5, whereas `"internationalization".indexOf("world")` returns -1. - -###### lastIndexOf(s, sub) {#lastindexofs-sub} - -Returns the first character index of sub as it last occurs in s; or, returns -1 if s does not contain sub. For example, `"parallel".lastIndexOf("a")` returns 3 (pointing at the second “a”). - -###### replace(s, s or p find, s replace) {#replaces-s-or-p-find-s-replace} - -Returns the string obtained by replacing the find string with the replace string in the inputted string. For example, `"The cow jumps over the moon and moos".replace("oo", "ee")` returns the string “The cow jumps over the meen and mees”. Find can be a regex pattern. For example, `"The cow jumps over the moon and moos".replace(/\s+/, "_")` will return “The_cow_jumps_over_the_moon_and_moos”. - -You cannot find or replace nulls with this, as null is not a string. You can instead: - -1. Facet by null and then bulk-edit them to a string, or -2. Transform the column with an expression such as `if(value==null,"new",value)`. - -###### replaceChars(s, s find, s replace) {#replacecharss-s-find-s-replace} - -Returns the string obtained by replacing a character in s, identified by find, with the corresponding character identified in replace. For example, `"Téxt thát was optícálly recógnízéd".replaceChars("áéíóú", "aeiou")` returns the string “Text that was optically recognized”. You cannot use this to replace a single character with more than one character. - -###### find(s, sub or p) {#finds-sub-or-p} - -Outputs an array of all consecutive substrings inside string s that match the substring or [regex](expressions#grel-supported-regex) pattern p. For example, `"abeadsabmoloei".find(/[aeio]+/)` would result in the array [ "a", "ea", "a", "o", "oei" ]. - -You can supply a substring instead of p, by putting it in quotes, and OpenRefine will compile it into a regex pattern. Anytime you supply quotes, OpenRefine interprets the contents as a string, not regex. If you wish to use any regex notation, wrap the pattern in forward slashes. - -###### match(s, p) {#matchs-p} - -Attempts to match the string s in its entirety against the [regex](expressions#grel-supported-regex) pattern p and, if the pattern is found, outputs an array of all [capturing groups](https://www.regular-expressions.info/brackets.html) (found in order). For example, `"230.22398, 12.3480".match(/.*(\d\d\d\d)/)` returns an array of 1 substring: [ "3480" ]. It does not find 2239 as the first sequence with four digits, because the regex indicates the four digits must come at the end of the string. - -You will need to convert the array to a string to store it in a cell, with a function such as toString(). An empty array [] is returned when there is no match to the desired substrings. A null is output when the entire regex does not match. - -Remember to enclose your regex in forward slashes, and to escape characters and use parentheses as needed. Parentheses denote a desired substring (capturing group); for example, “.*(\d\d\d\d)” would return an array containing a single value, while “(.*)(\d\d\d\d)” would return two. So, if you are looking for a desired substring anywhere within a string, use the syntax `value.match(/.*(desired-substring-regex).*/)`. - -For example, if `value` is “hello 123456 goodbye”, the following would occur: - -|Expression|Result| -|-|-| -|`value.match(/\d{6}/)` |null (does not match the full string)| -|`value.match(/.*\d{6}.*/)` |[ ] (no indicated substring)| -|`value.match(/.*(\d{6}).*/)` |[ "123456" ] (array with one value)| -|`value.match(/(.*)(\d{6})(.*)/)` |[ "hello ", "123456", " goodbye" ] (array with three values)| - -### String parsing and splitting {#string-parsing-and-splitting} - -###### toNumber(s) {#tonumbers} - -Returns a string converted to a number. Will attempt to convert other formats into a string, then into a number. If the value is already a number, it will return the number. - -###### split(s, s or p sep, b preserveTokens (optional)) {#splits-s-or-p-sep-b-preservetokens-optional} - -Returns the array of strings obtained by splitting s by sep. The separator can be either a string or a regex pattern. For example, `"fire, water, earth, air".split(",")` returns an array of 4 strings: [ "fire", " water", " earth", " air" ]. Note that the space characters are retained but the separator is removed. If you include “true” for the preserveTokens boolean, empty segments are preserved. - -###### splitByLengths(s, n1, n2, ...) {#splitbylengthss-n1-n2-} - -Returns the array of strings obtained by splitting s into substrings with the given lengths. For example, `"internationalization".splitByLengths(5, 6, 3)` returns an array of 3 strings: [ "inter", "nation", "ali" ]. Excess characters are discarded. - -###### smartSplit(s, s or p sep (optional)) {#smartsplits-s-or-p-sep-optional} - -Returns the array of strings obtained by splitting s by sep, or by guessing either tab or comma separation if there is no sep given. Handles quotes properly and understands cancelled characters. The separator can be either a string or a regex pattern. For example, `value.smartSplit("\n")` will split at a carriage return or a new-line character. - -Note: [`value.escape('javascript')`](#escapes-s-mode) is useful for previewing unprintable characters prior to using smartSplit(). - -###### splitByCharType(s) {#splitbychartypes} - -Returns an array of strings obtained by splitting s into groups of consecutive characters each time the characters change [Unicode categories](https://en.wikipedia.org/wiki/Unicode_character_property#General_Category). For example, `"HenryCTaylor".splitByCharType()` will result in an array of [ "H", "enry", "CT", "aylor" ]. It is useful for separating letters and numbers: `"BE1A3E".splitByCharType()` will result in [ "BE", "1", "A", "3", "E" ]. - -###### partition(s, s or p fragment, b omitFragment (optional)) {#partitions-s-or-p-fragment-b-omitfragment-optional} - -Returns an array of strings [ a, fragment, z ] where a is the substring within s before the first occurrence of fragment, and z is the substring after fragment. Fragment can be a string or a regex. For example, `"internationalization".partition("nation")` returns 3 strings: [ "inter", "nation", "alization" ]. If s does not contain fragment, it returns an array of [ s, "", "" ] (the original unpartitioned string, and two empty strings). - -If the omitFragment boolean is true, for example with `"internationalization".partition("nation", true)`, the fragment is not returned. The output is [ "inter", "alization" ]. - -You can use regex for your fragment. The expresion `"abcdefgh".partition(/c.e/)` will output [“abc”, "cde", defgh” ]. - -###### rpartition(s, s or p fragment, b omitFragment (optional)) {#rpartitions-s-or-p-fragment-b-omitfragment-optional} - -Returns an array of strings [ a, fragment, z ] where a is the substring within s before the last occurrence of fragment, and z is the substring after the last instance of fragment. (Rpartition means “reverse partition.”) For example, `"parallel".rpartition("a")` returns 3 strings: [ "par", "a", "llel" ]. Otherwise works identically to partition() above. - -### Encoding and hashing {#encoding-and-hashing} - -###### diff(s1, s2, s timeUnit (optional)) {#diffs1-s2-s-timeunit-optional} - -Takes two strings and compares them, returning a string. Returns the remainder of s2 starting with the first character where they differ. For example, `"cacti".diff("cactus")` returns "us". Also works with dates; see [Date functions](#diffd1-d2-s-timeunit). - -###### escape(s, s mode) {#escapes-s-mode} - -Escapes s in the given escaping mode. The mode can be one of: "html", "xml", "csv", "url", "javascript". Note that quotes are required around your mode. See the [recipes](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#question-marks--showing-in-your-data) for examples of escaping and unescaping. - -###### unescape(s, s mode) {#unescapes-s-mode} - -Unescapes s in the given escaping mode. The mode can be one of: "html", "xml", "csv", "url", "javascript". Note that quotes are required around your mode. See the [recipes](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#atampampt----att) for examples of escaping and unescaping. - -###### md5(o) {#md5o} - -Returns the [MD5 hash](https://en.wikipedia.org/wiki/MD5) of an object. If fed something other than a string (array, number, date, etc.), md5() will convert it to a string and deliver the hash of the string. For example, `"internationalization".md5()` will return 2c55a1626e31b4e373ceedaa9adc12a3. - -###### sha1(o) {#sha1o} - -Returns the [SHA-1 hash](https://en.wikipedia.org/wiki/SHA-1) of an object. If fed something other than a string (array, number, date, etc.), sha1() will convert it to a string and deliver the hash of the string. For example, `"internationalization".sha1()` will return cd05286ee0ff8a830dbdc0c24f1cb68b83b0ef36. - -###### phonetic(s, s encoding) {#phonetics-s-encoding} - -Returns a phonetic encoding of a string, based on an available phonetic algorithm. See the [section on phonetic clustering](cellediting#clustering-methods) for more information. Can be one of the following supported phonetic methods: [metaphone, doublemetaphone, metaphone3](https://www.wikipedia.org/wiki/Metaphone), [soundex](https://en.wikipedia.org/wiki/Soundex), [cologne](https://en.wikipedia.org/wiki/Cologne_phonetics). Quotes are required around your encoding method. For example, `"Ruth Prawer Jhabvala".phonetic("metaphone")` outputs the string “R0PRWRJHBFL”. - -###### reinterpret(s, s encoderTarget, s encoderSource) {#reinterprets-s-encodertarget-s-encodersource} - -Returns s reinterpreted through the given character encoders. You must supply one of the [supported encodings](http://java.sun.com/j2se/1.5.0/docs/guide/intl/encoding.doc.html) for each of the original source and the target output. Note that quotes are required around your character encoder. - -When an OpenRefine project is started, data is imported and interpreted. A specific character encoding is identified or manually selected at that time (such as UTF-8). You can reinterpret a column into another specificed encoding using this function. This function may not fix your data; it may be better to use this in conjunction with new projects to test the interpretation, and pre-format your data as needed. - -###### fingerprint(s) {#fingerprints} - -Returns the fingerprint of s, a string that is the first step in [fingerprint clustering methods](cellediting#clustering-methods): it will trim whitespaces, convert all characters to lowercase, remove punctuation, sort words alphabetically, etc. For example, `"Ruth Prawer Jhabvala".fingerprint()` outputs the string “jhabvala prawer ruth”. - -###### ngram(s, n) {#ngrams-n} - -Returns an array of the word n-grams of s. That is, it lists all the possible consecutive combinations of n words in the string. For example, `"Ruth Prawer Jhabvala".ngram(2)` would output the array [ "Ruth Prawer", "Prawer Jhabvala" ]. A word n-gram of 1 simply lists all the words in original order; an n-gram larger than the number of words in the string will only return the original string inside an array (e.g. `"Ruth Prawer Jhabvala".ngram(4)` would simply return ["Ruth Prawer Jhabvala"]). - -###### ngramFingerprint(s, n) {#ngramfingerprints-n} - -Returns the [n-gram fingerprint](cellediting#clustering-methods) of s. For example, `"banana".ngram(2)` would output “anbana”, after first generating the 2-grams “ba an na an na”, removing duplicates, and sorting them alphabetically. - -###### unicode(s) {#unicodes} - -Returns an array of strings describing each character of s in their full unicode notation. For example, `"Bernice Rubens".unicode()` outputs [ 66, 101, 114, 110, 105, 99, 101, 32, 82, 117, 98, 101, 110, 115 ]. - -###### unicodeType(s) {#unicodetypes} - -Returns an array of strings describing each character of s by their unicode type. For example, `"Bernice Rubens".unicodeType()` outputs [ "uppercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "space separator", "uppercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter" ]. - -## Format-based functions (JSON, HTML, XML) {#format-based-functions-json-html-xml} - -###### jsonize(o) {#jsonizeo} - -Quotes a value as a JSON literal value. - -###### parseJson(s) {#parsejsons} - -Parses a string as JSON. get() can then be used with parseJson(): for example, `parseJson(" { 'a' : 1 } ").get("a")` returns 1. - -For example, from the following JSON array in `value`, we want to get all instances of “keywords” having the same object string name of “text”, and combine them, using the forEach() function to iterate over the array. - - { - "status":"OK", - "url":"", - "language":"english", - "keywords":[ - { - "text":"York en route", - "relevance":"0.974363" - }, - { - "text":"Anthony Eden", - "relevance":"0.814394" - }, - { - "text":"President Eisenhower", - "relevance":"0.700189" - } - ] - } - -The GREL expression `forEach(value.parseJson().keywords,v,v.text).join(":::")` will output “York en route:::Anthony Eden:::President Eisenhower”. - -### Jsoup XML and HTML parsing {#jsoup-xml-and-html-parsing} - -###### parseHtml(s) {#parsehtmls} -Given a cell full of HTML-formatted text, parseHtml() simplifies HTML tags (such as by removing “ /” at the end of self-closing tags), closes any unclosed tags, and inserts linebreaks and indents for cleaner code. You cannot pass parseHtml() a URL, but you can pre-fetch HTML with the [Add column by fetching URLs](columnediting#add-column-by-fetching-urls) menu option. - -A cell cannot store the output of parseHtml() unless you convert it with toString(): for example, `value.parseHtml().toString()`. - -When parseHtml() simplifies HTML, it can sometimes introduce errors. When closing tags, it makes its best guesses based on line breaks, indentation, and the presence of other tags. You may need to manually check the results. - -You can then extract or [select()](#selects-element) which portions of the HTML document you need for further splitting, partitioning, etc. An example of extracting all table rows from a div using parseHtml().select() together is described more in depth at [StrippingHTML](https://github.com/OpenRefine/OpenRefine/wiki/StrippingHTML). - -###### parseXml(s) {#parsexmls} -Given a cell full of XML-formatted text, parseXml() returns a full XML document and adds any missing closing tags. You can then extract or [select()](#selects-element) which portions of the XML document you need for further splitting, partitioning, etc. Functions the same way as parseHtml() is described above. - -###### select(s, element) {#selects-element} -Returns an array of all the desired elements from an HTML or XML document, if the element exists. Elements are identified using the [Jsoup selector syntax](https://jsoup.org/apidocs/org/jsoup/select/Selector.html). For example, `value.parseHtml().select("img.portrait")[0]` would return the entirety of the first “img” tag with the “portrait” class found in the parsed HTML inside `value`. Returns an empty array if no matching element is found. Use with toString() to capture the results in a cell. A tutorial of select() is shown in [StrippingHTML](https://github.com/OpenRefine/OpenRefine/wiki/StrippingHTML). - -You can use select() more than once: - -``` -value.parseHtml().select("div#content")[0].select("tr").toString() -``` - -###### htmlAttr(s, element) {#htmlattrs-element} -Returns a string from an attribute on an HTML element. Use it in conjunction with parseHtml() as in the following example: `value.parseHtml().select("a.email")[0].htmlAttr("href")` would retrieve the email address attached to a link with the “email” class. - -###### xmlAttr(s, element) {#xmlattrs-element} -Returns a string from an attribute on an XML element. Functions the same way htmlAttr() is described above. Use it in conjunction with parseXml(). - -###### htmlText(element) {#htmltextelement} -Returns a string of the text from within an HTML element (including all child elements), removing HTML tags and line breaks inside the string. Use it in conjunction with parseHtml() and select() to provide an element, as in the following example: `value.parseHtml().select("div.footer")[0].htmlText()`. - -###### xmlText(element) {#xmltextelement} -Returns a string of the text from within an XML element (including all child elements). Functions the same way htmlText() is described above. Use it in conjunction with parseXml() and select() to provide an element. - -###### wholeText(element) {#wholetextelement} - -Selects the (unencoded) text of an element and its children, including any new lines and spaces, and returns a string of unencoded, un-normalized text. Use it in conjunction with parseHtml() and select() to provide an element as in the following example: `value.parseHtml().select("div.footer")[0].wholeText()`. - -###### innerHtml(element) {#innerhtmlelement} -Returns the [inner HTML](https://developer.mozilla.org/en-US/docs/Web/API/Element/innerHTML) of an HTML element. This will include text and children elements within the element selected. Use it in conjunction with parseHtml() and select() to provide an element. - -###### innerXml(element) {#innerxmlelement} -Returns the inner XML elements of an XML element. Does not return the text directly inside your chosen XML element - only the contents of its children. To select the direct text, use ownText(). To select both, use xmlText(). Use it in conjunction with parseXml() and select() to provide an element. - -###### ownText(element) {#owntextelement} -Returns the text directly inside the selected XML or HTML element only, ignoring text inside children elements (for this, use innerXml()). Use it in conjunction with a parser and select() to provide an element. - -## Array functions {#array-functions} - -###### length(a) {#lengtha} -Returns the size of an array, meaning the number of objects inside it. Arrays can be empty, in which case length() will return 0. - -###### slice(a, n from, n to (optional)) {#slicea-n-from-n-to-optional} -Returns a sub-array of a given array, from the first index provided and up to and excluding the optional last index provided. Remember that array objects are indexed starting at 0. If the to value is omitted, it is understood to be the end of the array. For example, `[0, 1, 2, 3, 4].slice(1, 3)` returns [ 1, 2 ], and `[ 0, 1, 2, 3, 4].slice(2)` returns [ 2, 3, 4 ]. Also works with strings; see [String functions](#slices-n-from-n-to-optional). - -###### get(a, n from, n to (optional)) {#geta-n-from-n-to-optional} -Returns a sub-array of a given array, from the first index provided and up to and excluding the optional last index provided. Remember that array objects are indexed starting at 0. - -If the to value is omitted, only one array item is returned, as a string, instead of a sub-array. To return a sub-array from one index to the end, you can set the to argument to a very high number such as `value.get(2,999)` or you can use something like `with(value,a,a.get(1,a.length()))` to count the length of each array. - -Also works with strings; see [String functions](#gets-n-from-n-to-optional). - -###### inArray(a, s) {#inarraya-s} -Returns true if the array contains the desired string, and false otherwise. Will not convert data types; for example, `[ 1, 2, 3, 4 ].inArray("3")` will return false. - -###### reverse(a) {#reversea} -Reverses the array. For example, `[ 0, 1, 2, 3].reverse()` returns the array [ 3, 2, 1, 0 ]. - -###### sort(a) {#sorta} -Sorts the array in ascending order. Sorting is case-sensitive, uppercase first and lowercase second. For example, `[ "al", "Joe", "Bob", "jim" ].sort()` returns the array [ "Bob", "Joe", "al", "jim" ]. - -###### sum(a) {#suma} -Return the sum of the numbers in the array. For example, `[ 2, 1, 0, 3 ].sum()` returns 6. - -###### join(a, sep) {#joina-sep} -Joins the items in the array with sep, and returns it all as a string. For example, `[ "and", "or", "not" ].join("/")` returns the string “and/or/not”. - -###### uniques(a) {#uniquesa} -Returns the array with duplicates removed. Case-sensitive. For example, `[ "al", "Joe", "Bob", "Joe", "Al", "Bob" ].uniques()` returns the array [ "Joe", "al", "Al", "Bob" ]. - -As of OpenRefine 3.4.1, uniques() reorders the array items it returns; in 3.4 beta 644 and onwards, it preserves the original order (in this case, [ "al", "Joe", "Bob", "Al" ]). - -## Date functions {#date-functions} - -###### now() {#now} - -Returns the current time according to your system clock, in the [ISO 8601 extended format](exploring#data-types) (converted to UTC). For example, 10:53am (and 00 seconds) on November 26th 2020 in EST returns [date 2020-11-26T15:53:00Z]. - -###### toDate(o, b monthFirst, s format1, s format2, ...) {#todateo-b-monthfirst-s-format1-s-format2-} - -Returns the inputted object converted to a date object. Without arguments, it returns the ISO 8601 extended format. With arguments, you can control the output format: -* monthFirst: set false if the date is formatted with the day before the month. -* formatN: attempt to parse the date using an ordered list of possible formats. Supply formats based on the [SimpleDateFormat](https://docs.oracle.com/javase/8/docs/api/java/text/SimpleDateFormat.html) syntax (and see the table below for a handy reference). - -For example, you can parse a column containing dates in different formats, such as cells with “Nov-09” and “11/09”, using `value.toDate('MM/yy','MMM-yy').toString('yyyy-MM')` and both will output “2009-11”. For another example, “1/4/2012 13:30:00” can be parsed into a date using `value.toDate('d/M/y H:m:s')`. - -| Letter | Date or Time Component | Presentation | Examples | -|-|-|-|-| -| G | Era designator | Text | AD | -| y | Year | Year | 1996; 96 | -| Y | [Week year](https://en.wikipedia.org/wiki/ISO_week_date#First_week) | Year | 2009; 09 | -| M | Month in year | Month | July; Jul; 07 | -| w | Week in year | Number | 27 | -| W | Week in month | Number | 2 | -| D | Day in year | Number | 189 | -| d | Day in month | Number | 10 | -| F | Day of week in month | Number | 2 | -| E | Day name in week | Text | Tuesday; Tue | -| u | Day number of week (1 = Monday, ..., 7 = Sunday) | Number | 1 | -| a | AM/PM marker | Text | PM | -| H | Hour in day (0-23) | Number | 0 | -| k | Hour in day (1-24) | Number | 24 | -| K | Hour in AM/PM (0-11) | Number | 0 | -| h | Hour in AM/PM (1-12) | Number | 12 | -| m | Minute in hour | Number | 30 | -| s | Second in minute | Number | 55 | -| S | Millisecond | Number | 978 | -| n | Nanosecond | Number | 789000 | -| z | Time zone | General time zone | Pacific Standard Time; PST; GMT-08:00 | -| Z | Time zone | RFC 822 time zone | \-0800 | -| X | Time zone | ISO 8601 time zone | \-08; -0800; -08:00 | - -###### diff(d1, d2, s timeUnit) {#diffd1-d2-s-timeunit} - -Given two dates, returns a number indicating the difference in a given time unit (see the table below). For example, `diff(("Nov-11".toDate('MMM-yy')), ("Nov-09".toDate('MMM-yy')), "weeks")` will return 104, for 104 weeks, or two years. The later date should go first. If the output is negative, invert d1 and d2. - -Also works with strings; see [diff() in string functions](#diffsd1-sd2-s-timeunit-optional). - -###### inc(d, n, s timeUnit) {#incd-n-s-timeunit} - -Returns a date changed by the given amount in the given unit of time (see the table below). The default unit is “hour”. A positive value increases the date, and a negative value moves it back in time. For example, if you want to move a date backwards by two months, use `value.inc(-2,"month")`. - -###### datePart(d, s timeUnit) {#datepartd-s-timeunit} - -Returns part of a date. The data type returned depends on the unit (see the table below). - -OpenRefine supports the following values for timeUnit: - -| Unit | Date part returned | Returned data type | Example using [date 2014-03-14T05:30:04.000789000Z] as value | -|-|-|-|-| -| years | Year | Number | value.datePart("years") → 2014 | -| year | Year | Number | value.datePart("year") → 2014 | -| months | Month | Number | value.datePart("months") → 2 | -| month | Month | Number | value.datePart("month") → 2 | -| weeks | Week of the month | Number | value.datePart("weeks") → 3 | -| week | Week of the month | Number | value.datePart("week") → 3 | -| w | Week of the month | Number | value.datePart("w") → 3 | -| weekday | Day of the week | String | value.datePart("weekday") → Friday | -| hours | Hour | Number | value.datePart("hours") → 5 | -| hour | Hour | Number | value.datePart("hour") → 5 | -| h | Hour | Number | value.datePart("h") → 5 | -| minutes | Minute | Number | value.datePart("minutes") → 30 | -| minute | Minute | Number | value.datePart("minute") → 30 | -| min | Minute | Number | value.datePart("min") → 30 | -| seconds | Seconds | Number | value.datePart("seconds") → 04 | -| sec | Seconds | Number | value.datePart("sec") → 04 | -| s | Seconds | Number | value.datePart("s") → 04 | -| milliseconds | Millseconds | Number | value.datePart("milliseconds") → 789 | -| ms | Millseconds | Number | value.datePart("ms") → 789 | -| S | Millseconds | Number | value.datePart("S") → 789 | -| n | Nanoseconds | Number | value.datePart("n") → 789000 | -| nano | Nanoseconds | Number | value.datePart("n") → 789000 | -| nanos | Nanoseconds | Number | value.datePart("n") → 789000 | -| time | Milliseconds between input and the [Unix Epoch](https://en.wikipedia.org/wiki/Unix_time) | Number | value.datePart("time") → 1394775004000 | - -## Math functions {#math-functions} - -For integer division and precision, you can use simple evaluations such as `1 / 2`, which is equivalent to `floor(1/2)` - that is, it returns only whole number results. If either operand is a floating point number, they both get promoted to floating point and a floating point result is returned. You can use `1 / 2.0` or `1.0 / 2` or `1.0 * x / y` (if you're working with variables of unknown contents). - -:::caution -Some of these math functions don't recognize integers when supplied as the first argument in dot notation (e.g., `5.cos()` simply returns 5 instead of the expected result). To ensure operations are successful, always wrap the first argument in brackets, such as `(value).cos()`. -::: - -|Function|Use|Example| -|-|-|-| -|`abs(n)`|Returns the absolute value of a number.|`abs(-6)` returns 6.| -|`acos(n)`|Returns the arc cosine of an angle, in the range 0 through [PI](https://docs.oracle.com/javase/8/docs/api/java/lang/Math.html#PI).|`acos(0.345)` returns 1.218557541697832.| -|`asin(n)`|Returns the arc sine of an angle in the range of -PI/2 through PI/2.|`asin(0.345)` returns 0.35223878509706474.| -|`atan(n)`|Returns the arc tangent of an angle in the range of -PI/2 through PI/2.|`atan(0.345)` returns 0.3322135507465967.| -|`atan2(n1, n2)`|Converts rectangular coordinates (n1, n2) to polar (r, theta). Returns number theta.|`atan2(0.345,0.6)` returns 0.5218342798144103.| -|`ceil(n)`|Returns the ceiling of a number.|`3.7.ceil()` returns 4 and `-3.7.ceil()` returns -3.| -|`combin(n1, n2)`|Returns the number of combinations for n2 elements as divided into n1.|`combin(20,2)` returns 190.| -|`cos(n)`|Returns the trigonometric cosine of a value.|`cos(5)` returns 0.28366218546322625.| -|`cosh(n)`|Returns the hyperbolic cosine of a value.|`cosh(5)` returns 74.20994852478785.| -|`degrees(n)`|Converts an angle from radians to degrees.|`degrees(5)` returns 286.4788975654116.| -|`even(n)`|Rounds the number up to the nearest even integer.|`even(5)` returns 6.| -|`exp(n)`|Returns [e](https://en.wikipedia.org/wiki/E_(mathematical_constant)) raised to the power of n.|`exp(5)` returns 148.4131591025766.| -|`fact(n)`|Returns the factorial of a number, starting from 1.|`fact(5)` returns 120.| -|`factn(n1, n2)`|Returns the factorial of n1, starting from n2.|`factn(10,3)` returns 280.| -|`floor(n)`|Returns the floor of a number.|`3.7.floor()` returns 3 and `-3.7.floor()` returns -4.| -|`gcd(n1, n2)`|Returns the greatest common denominator of two numbers.|`gcd(95,135)` returns 5.| -|`lcm(n1, n2)`|Returns the least common multiple of two numbers.|`lcm(95,135)` returns 2565.| -|`ln(n)`|Returns the natural logarithm of n.|`ln(5)` returns 1.6094379124341003.| -|`log(n)`|Returns the base 10 logarithm of n.|`log(5)` returns 0.6989700043360189.| -|`max(n1, n2)`|Returns the larger of two numbers.|`max(3,10)` returns 10.| -|`min(n1, n2)`|Returns the smaller of two numbers.|`min(3,10)` returns 3.| -|`mod(n1, n2)`|Returns n1 modulus n2. Note: `value.mod(9)` will work, whereas `74.mod(9)` will not work.|`mod(74, 9)` returns 2. | -|`multinomial(n1, n2 …(optional))`|Calculates the multinomial of one number or a series of numbers.|`multinomial(2,3)` returns 10.| -|`odd(n)`|Rounds the number up to the nearest odd integer.|`odd(10)` returns 11.| -|`pow(n1, n2)`|Returns n1 raised to the power of n2. Note: value.pow(3)` will work, whereas `2.pow(3)` will not work.|`pow(2, 3)` returns 8 (2 cubed) and `pow(3, 2)` returns 9 (3 squared). The square root of any numeric value can be called with `value.pow(0.5)`.| -|`quotient(n1, n2)`|Returns the integer portion of a division (truncated, not rounded), when supplied with a numerator and denominator.|`quotient(9,2)` returns 4.| -|`radians(n)`|Converts an angle in degrees to radians.|`radians(10)` returns 0.17453292519943295.| -|`randomNumber(n lowerBound, n upperBound)`|Returns a random integer in the interval between the lower and upper bounds (inclusively). Will output a different random number in each cell in a column.| -|`round(n)`|Rounds a number to the nearest integer.|`3.7.round()` returns 4 and `-3.7.round()` returns -4.| -|`sin(n)`|Returns the trigonometric sine of an angle.|`sin(10)` returns -0.5440211108893698.| -|`sinh(n)`|Returns the hyperbolic sine of an angle.|`sinh(10)` returns 11013.232874703393.| -|`sum(a)`|Sums the numbers in an array. Ignores non-number items. Returns 0 if the array does not contain numbers.|`sum([ 10, 2, three ])` returns 12.| -|`tan(n)`|Returns the trigonometric tangent of an angle.|`tan(10)` returns 0.6483608274590866.| -|`tanh(n)`|Returns the hyperbolic tangent of a value.|`tanh(10)` returns 0.9999999958776927.| - -## Other functions {#other-functions} - -###### type(o) {#typeo} -Returns a string with the data type of o, such as undefined, string, number, boolean, etc. For example, a [Transform](cellediting#transform) operation using `value.type()` will convert all cells in a column to strings of their data types. - -###### facetCount(choiceValue, s facetExpression, s columnName) {#facetcountchoicevalue-s-facetexpression-s-columnname} -Returns the facet count corresponding to the given choice value, by looking for the facetExpression in the choiceValue in columnName. For example, to create facet counts for the following table, we could generate a new column based on “Gift” and enter in `value.facetCount("value", "Gift")`. This would add the column we've named “Count”: - -| Gift | Recipient | Price | Count | -|-|-|-|-| -| lamp | Mary | 20 | 1 | -| clock | John | 57 | 2 | -| watch | Amit | 80 | 1 | -| clock | Claire | 62 | 2 | - -The facet expression, wrapped in quotes, can be useful to manipulate the inputted values before counting. For example, you could do a textual cleanup using fingerprint(): `(value.fingerprint()).facetCount(value.fingerprint(),"Gift")`. - -###### hasField(o, s name) {#hasfieldo-s-name} -Returns a boolean indicating whether o has a member field called [name](expressions#variables). For example, `cell.recon.hasField("match")` will return false if a reconciliation match hasn’t been selected yet, or true if it has. You cannot chain your desired fields: for example, `cell.hasField("recon.match")` will return false even if the above expression returns true). - -###### coalesce(o1, o2, o3, ...) {#coalesceo1-o2-o3-} -Returns the first non-null from a series of objects. For example, `coalesce(value, "")` would return an empty string “” if `value` was null, but otherwise return `value`. - -###### cross(cell, s projectName (optional), s columnName (optional)) {#crosscell-s-projectname-optional-s-columnname-optional} -Returns an array of zero or more rows in the project projectName for which the cells in their column columnName have the same content as the cell in your chosen column. For example, if two projects contained matching names, and you wanted to pull addresses for people by their names from a project called “People” you would apply the following expression to your column of names: -``` -cell.cross("People","Name").cells["Address"].value[0] -``` - -This would match your current column to the “Name” column in “People” and, using those matches, pull the respective “Address” value into your current project. - -You may need to do some data preparation with cross(), such as using trim() on your key columns or deduplicating values. - -The first argument will be interpreted as `cell.value` if set to `cell`. If you omit projectName and columnName, they will default to the current project and index column (number 0). - -Recipes and more examples for using cross() can be found [on our wiki](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#combining-datasets). diff --git a/OpenRefine/docs/docs/manual/installing.md b/OpenRefine/docs/docs/manual/installing.md deleted file mode 100644 index 74db80926..000000000 --- a/OpenRefine/docs/docs/manual/installing.md +++ /dev/null @@ -1,425 +0,0 @@ ---- -id: installing -title: Installing OpenRefine -sidebar_label: Installing ---- - -## System requirements {#system-requirements} - -OpenRefine does not require internet access to run its basic functions. Once you download and install it, it runs as a small web server on your own computer, and you access that local web server by using your browser. It only requires an internet connection to import data from the web, reconcile data using a web service, or export data to the web. - -OpenRefine requires three things on your computer in order to function: - -#### Compatible operating system {#compatible-operating-system} - -OpenRefine is designed to work with **Windows**, **Mac**, and **Linux** operating systems. [Our team releases packages for each](https://openrefine.org/download.html). - -#### Java {#java} - -[Java](https://java.com/en/download/) must be installed and configured on your computer to run OpenRefine. The Mac version of OpenRefine includes Java; new in OpenRefine 3.4, there is also a Windows package with Java included. - -If you install and start OpenRefine on a Windows computer without Java, it will automatically open up a browser window to the [Java downloads page](https://java.com/en/download/), and you can simply follow the instructions there. - -We recommend you [download](https://java.com/en/download/) and install Java before proceeding with the OpenRefine installation. Please note that OpenRefine 3.5 works with Java 8 to Java 17. - -#### Compatible browser {#compatible-browser} - -OpenRefine works best on browsers based on Webkit, such as: - -* Google Chrome -* Chromium -* Opera -* Microsoft Edge - -We are aware of some minor rendering and performance issues on other browsers such as Firefox. We don't support Internet Explorer. If you are having issues running OpenRefine, see the [section on Running](running.md#troubleshooting). - -### Release versions {#release-versions} - -OpenRefine always has a [latest stable release](https://github.com/OpenRefine/OpenRefine/releases/latest), as well as some more recent developments available in beta, release candidate, or [snapshot releases](https://github.com/OpenRefine/OpenRefine-snapshot-releases/releases). If you are installing for the first time, we recommend [the latest stable release](https://github.com/OpenRefine/OpenRefine/releases/latest). - -If you wish to use an extension that is only compatible with an earlier version of OpenRefine, and do not require the latest features, you may find that [an older stable version is best for you](https://github.com/OpenRefine/OpenRefine/releases) in our list of releases. Look at later releases to see which security vulnerabilities are being fixed, in order to assess your own risk tolerance for using earlier versions. Look for “final release” versions instead of “beta” or “release candidate” versions. - -#### Unstable versions {#unstable-versions} - -If you need a recently developed function, and are willing to risk some untested code, you can look at [the most recent items in the list](https://github.com/OpenRefine/OpenRefine/releases) and see what changes appeal to you. - -“Beta” and “release candidate” versions may both have unreported bugs and are most suitable for people who are willing to help us troubleshoot these versions by [creating bug reports](https://github.com/OpenRefine/OpenRefine/issues). - -For the absolute latest development updates, see the [snapshot releases](https://github.com/OpenRefine/OpenRefine-snapshot-releases/releases). These are created with every commit. - -#### What’s changed {#whats-changed} - -Our [latest version is OpenRefine 3.4.1](https://github.com/OpenRefine/OpenRefine/releases/tag/3.4.1), released September 24th 2020. The major changes in this version are listed on the [3.4.1 release page](https://github.com/OpenRefine/OpenRefine/releases/tag/3.4.1) with the downloadable packages. - -You can find information about all OpenRefine versions on the [Releases page on Github](https://github.com/OpenRefine/OpenRefine/releases). - -:::info Other distributions -OpenRefine may also work in other environments, such as [Chromebooks](https://gist.github.com/organisciak/3e12e5138e44a2fed75240f4a4985b4f) where Linux terminals are available. Look at our list of [Other Distributions on the Downloads page](https://openrefine.org/download.html) for other ways of running OpenRefine, and refer to our contributor community to see new environments in development. -::: - -## Installing or upgrading {#installing-or-upgrading} -### Back up your data {#back-up-your-data} - -If you are upgrading from an older version of OpenRefine and have projects already on your computer, you should create backups of those projects before you install a new version. - -First, [locate your workspace directory](#where-is-data-stored). Then copy everything you find there and paste it into a folder elsewhere on your computer. - -For extra security you can [export your existing OpenRefine projects](exporting#export-a-project). - -:::caution -Take note of the [extensions](#installing-extensions) you have currently installed. They may not be compatible with the upgraded version of OpenRefine. Installations can be installed in two places, so be sure to check both your workspace directory and the existing installation directory. -::: - -### Install or upgrade OpenRefine {#install-or-upgrade-openrefine} - -If you are upgrading an existing OpenRefine installation, you can delete the old program files and install the new files into the same space. Do not overwrite the files as some obsolete files may be left over unnecessarily. - -:::caution -If you have extensions installed, do not delete the `webapp\extensions` folder where you installed them. You may wish to install extensions into the workspace directory instead of the program directory. There is no guarantee that extensions will be forward-compatible with new versions of OpenRefine, and we do not maintain extensions. -::: - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - - -Once you have downloaded the `.zip` file, extract it into a folder where you wish to store program files (such as `D:\Program Files\OpenRefine`). - -You can right-click on `openrefine.exe` or `refine.bat` and pin one of those programs to your Start Menu or create shortcuts for easier access. - - - - - -Once you have downloaded the `.dmg` file, open it and drag the OpenRefine icon onto the Applications folder icon (just like you would normally install Mac applications). - - - - - -The quick version: - -1. Install [Homebrew](http://brew.sh) -2. In Terminal enter ` brew cask install openrefine` -1. Then find OpenRefine in your Applications folder. - -The long version: - -[Homebrew](http://brew.sh) is a popular command-line package manager for Mac. Installing Homebrew is accomplished by pasting the installation command on the Homebrew website into a Terminal window. Once Homebrew is installed, applications like OpenRefine can be installed via a simple command. You can [install Homebrew from their website](http://brew.sh). - -###### Install {#install} - -Install OpenRefine with this command: - -``` -brew cask install openrefine -``` - -You should see output like this: - -``` -==> Downloading https://github.com/OpenRefine/OpenRefine/releases/download/2.7/openrefine-mac-2.7.dmg -########################### 100.0% - ==> Verifying checksum for Cask openrefine - ==> Installing Cask openrefine - ==> Moving App 'OpenRefine.app' to '/Applications/OpenRefine.app'. - 🍺 openrefine was successfully installed! -``` - -Behind the scenes, this command causes Homebrew to download the OpenRefine installer, verify the file’s authenticity (using a SHA-256 checksum), mount the disk image, copy the `OpenRefine.app` application bundle into the Applications folder, unmount the disk image, and save a copy of the installer and metadata about the installation for future use. - -If an existing `OpenRefine.app` is found in the Applications folder, Homebrew will not overwrite it, so installing via Homebrew requires either deleting or renaming previously installed copies. - -###### Uninstall {#uninstall} - -To uninstall OpenRefine, paste this command into the Terminal: - -``` - brew cask uninstall openrefine -``` - -You should see output like this: - -``` - ==> Removing App '/Applications/OpenRefine.app'. -``` - -###### Update {#update} - -To update to the latest version of OpenRefine, paste this command into the Terminal: - -``` - brew cask reinstall openrefine -``` - -You should see output like this: - -``` - ==> Downloading https://github.com/OpenRefine/OpenRefine/releases/download/2.7/openrefine-mac-2.7.dmg -########################### 100.0% - ==> Verifying checksum for Cask openrefine - ==> Removing App '/Applications/OpenRefine.app'. - ==> Moving App 'OpenRefine.app' to '/Applications/OpenRefine.app'. - 🍺 openrefine was successfully installed! -``` - -If you had previously installed the `openrefine-dev` cask (containing a release candidate) and you want to move to the stable release, you need to first uninstall the old cask and then install the new one: - -``` - brew cask uninstall openrefine-dev - brew cask install openrefine -``` - - - - -Once you have downloaded the `.tar.gz` file, open a shell, navigate to the folder containing the download, and type: - -``` -tar xzf openrefine-linux-3.4.tar.gz -``` - - - - - ---- - - -### Set where data is stored {#set-where-data-is-stored} - -OpenRefine stores data in two places: program files in the program directory, wherever it is you’ve installed it; and project files in what we call the “workspace directory.” You can access this folder easily from OpenRefine by going to the [home screen](running#the-home-screen) (at [http://127.0.0.1:3333/](http://127.0.0.1:3333/)) and clicking Browse workspace directory. - -By default this is: - - - - - -Depending on your version of Windows, the data is in one of these directories: -* `%appdata%\OpenRefine` -* `%localappdata%\OpenRefine` -* `C:\Documents and Settings\(user id)\Local Settings\Application Data\OpenRefine` -* `C:\Users\(user id)\AppData\Roaming\OpenRefine` -* `C:\Users\(user id)\AppData\Local\OpenRefine` -* `C:\Users\(user id)\OpenRefine` - -For older Google Refine releases, replace `OpenRefine` with `Google\Refine`. - -You can change this by adding this line to the file `openrefine.l4j.ini` and specifying your desired drive and folder path: - -``` --Drefine.data_dir=D:\MyDesiredFolder -``` - -If your folder path has spaces, use neutral quotation marks around it: - -``` --Drefine.data_dir="D:\My Desired Folder" -``` - -If the folder does not exist, OpenRefine will create it. - - - - - -``` -~/Library/Application Support/OpenRefine/ -``` - -For older versions, as Google Refine: - -``` -~/Library/Application Support/Google/Refine/ -``` - -Logging is to `/var/log/daemon.log` - grep for `com.google.refine.Refine`. - - - - - -``` -~/.local/share/openrefine/ -``` - -You can change this when you run OpenRefine from the terminal, by pointing to the workspace directory through the `-d` parameter: - -``` - ./refine -p 3333 -i 0.0.0.0 -m 6000M -d /My/Desired/Folder -``` - - - - - ---- - - -### Logs {#logs} - -OpenRefine does not currently output an error log, but because the OpenRefine console window is always open (on Linux and Windows) while OpenRefine runs in your browser, you can copy information from the console if an error occurs. - -Using a Mac, you can [run OpenRefine using the terminal](running#starting-and-exiting) in order to capture errors. - ---- - -## Increasing memory allocation {#increasing-memory-allocation} - -OpenRefine relies on having computer memory available to it to work effectively. If you are planning to work with large datasets, you may wish to set up OpenRefine to handle it at the outset. By “large” we generally mean one of the following indicators: -* more than one million total cells -* an input file size of more than 50 megabytes (MB) -* more than 50 [rows per record in records mode](running#records-mode) - -By default OpenRefine is set to operate with 1 gigabyte (GB) of memory (1024MB). If you feel that OpenRefine is running slowly, or you are getting “out of memory” errors (for example, `java.lang.OutOfMemoryError`), you can try allocating more memory. - -A good practice is to start with no more than 50% of whatever memory is left over after the estimated usage of your operating system, to leave memory for your browser to run. - -All of the settings below use a four-digit number to specify the megabytes (MB) used (actually [mebibytes](https://en.wikipedia.org/wiki/Mebibyte)). The default is usually 1024MB, but the new value doesn't need to be a multiple of 1024. - -:::info Dealing with large datasets -If your project is big enough to need more than the default amount of memory, consider turning off Parse cell text into numbers, dates, ... on import. It's convenient, but less efficient than explicitly converting any columns that you need as a data type other than the default “string” type. -::: - - - - - -#### Using openrefine.exe {#using-openrefineexe} - -If you run `openrefine.exe`, you will need to edit the `openrefine.l4j.ini` file found in the program directory and edit the line - -``` -# max memory memory heap size --Xmx1024M -``` - -The line “-Xmx1024M” defines the amount of memory available in megabytes. Change the number “1024” - for example, edit the line to “-Xmx2048M” to make 2048MB [2GB] of memory available. - -:::caution openrefine.exe not running? -Once you increase the memory allocation, you may find that you cannot run `openrefine.exe`. In this case, your computer needs a 64-bit version of [Java](https://www.java.com/en/download/help/index_installing.xml) (this is different from [Java JDK](#install-or-upgrade-java). Look for the “Windows Offline (64-bit)” download on the Downloads page and install that. Your system must also be set to use the 64-bit version of Java by [changing the Java configuration](https://www.java.com/en/download/help/update_runtime_settings.xml). -::: - -#### Using refine.bat {#using-refinebat} - -On Windows, OpenRefine can also be run by using the file `refine.bat` in the program directory. If you start OpenRefine using `refine.bat`, the memory available to OpenRefine can be specified either through command line options, or through the `refine.ini` file. - -To set the maximum amount of memory on the command line when using `refine.bat`, `cd` to the program directory, then type - -```refine.bat /m 2048m``` - -where “2048” is the maximum amount of MB that you want OpenRefine to use. - -To change the default that `refine.bat` uses, edit the `refine.ini` line that reads - -```REFINE_MEMORY=1024M``` - -Note that this file is only read if you use `refine.bat`, not `openrefine.exe`. - - - - -If you have downloaded the `.dmg` package and you start OpenRefine by double-clicking on it: - -* close OpenRefine -* control-click on the OpenRefine icon (opens the contextual menu) -* click on "show package content” (a finder window opens) -* open the “Contents” folder -* open the `Info.plist` file with any text editor (like Mac's default TextEdit) -* Change “-Xmx1024M” into, for example, “-Xmx2048M” or “-Xmx8G” -* save the file -* restart OpenRefine. - - - - -If you have downloaded the `.tar.gz` package and you start OpenRefine from the command line, add the “-m xxxxM” parameter like this: -`./refine -m 2048m` - -#### Setting a default {#setting-a-default} - -If you don't want to set this option on the command line each time, you can also set it in the `refine.ini` file. Edit the line - -``` -REFINE_MEMORY=1024M -``` - -Make sure it is not commented out (that is, that the line doesn't start with a “#” character), and change “1024” to a higher value. Save the file, and when you next start OpenRefine it will use this value. - - - - - ---- - - -## Installing extensions {#installing-extensions} - -Extensions have been created by our contributor community to add functionality or provide convenient shortcuts for common uses of OpenRefine. [We list extensions we know about on our downloads page](https://openrefine.org/download.html). - -:::info Contributing extensions -If you’d like to create or modify an extension, [see our developer documentation here](https://github.com/OpenRefine/OpenRefine/wiki/Documentation-For-Developers). If you’re having a problem, [use our downloads page](https://openrefine.org/download.html) to go to the extension’s page and report the issue there. -::: - -### Two ways to install extensions {#two-ways-to-install-extensions} - -You can [install extensions in one of two places](#set-where-data-is-stored): - -* Into your OpenRefine program folder, so they will only be available to that version/installation of OpenRefine (meaning the extension will not run if you upgrade OpenRefine), or -* Into your workspace, where your projects are stored, so they will be available no matter which version of OpenRefine you’re using. - -We provide these options because you may wish to reinstall a given extension manually each time you upgrade OpenRefine, in order to be sure it works properly. - -### Find the right place to install {#find-the-right-place-to-install} - -If you want to install the extension into the program folder, go to your program directory and then go to `webapp\extensions` (or create it if not does not exist). - -If you want to install the extension into your workspace, you can: -* launch OpenRefine and click Open Project in the sidebar -* At the bottom of the screen, click Browse workspace directory -* A file-explorer or finder window will open in your workspace -* Create a new folder called “extensions” inside the workspace if it does not exist. - -You can also [find your workspace on each operating system using these instructions](#set-where-data-is-stored). - -### Install the extension {#install-the-extension} - -Some extensions have their own instructions: make sure you read the documentation before you begin installing. - -Some extensions may have multiple versions, to match OpenRefine versions, so be sure to choose the right release for your installation. If you have questions about compatibility or want to request or voice your support for an update, [use our downloads page](https://openrefine.org/download.html) to go to the extension’s page and report the issue there. - -Generally, the installation process will be: - -* Download the extension (usually as a zip file from GitHub) -* Extract the zip contents into the “extensions” directory, making sure all the contents go into one folder with the name of the extension -* Start (or restart) OpenRefine. - -To confirm that installation was a success, follow the instructions provided by the extension. Each extension will appear in its own way inside the OpenRefine interface. Make sure you read its documentation to know where the functionality will appear, such as under specific dropdown menus. diff --git a/OpenRefine/docs/docs/manual/jythonclojure.md b/OpenRefine/docs/docs/manual/jythonclojure.md deleted file mode 100644 index a93d18155..000000000 --- a/OpenRefine/docs/docs/manual/jythonclojure.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -id: jythonclojure -title: Jython & Clojure -sidebar_label: Jython & Clojure ---- - -## Jython {#jython} - -Jython 2.7.2 comes bundled with the default installation of OpenRefine 3.4.1. You can add libraries and code by following [this tutorial](https://github.com/OpenRefine/OpenRefine/wiki/Extending-Jython-with-pypi-modules). A large number of Python files (`.py` or `.pyc`) are compatible. - -Python code that depends on C bindings will not work in OpenRefine, which uses Java / Jython only. Since Jython is essentially Java, you can also import Java libraries and utilize those. - -You will need to restart OpenRefine, so that new Jython or Python libraries are initialized during startup. - -OpenRefine now has [most of the Jsoup.org library built into GREL functions](grelfunctions#jsoup-xml-and-html-parsing-functions) for parsing and working with HTML and XML elements. - -### Syntax {#syntax} - -Expressions in Jython must have a `return` statement: - -``` - return value[1:-1] -``` - -``` - return rowIndex%2 -``` - -Fields have to be accessed using the bracket operator rather than dot notation: - -``` - return cells["col1"]["value"] -``` - -For example, to access the [edit distance](reconciling#reconciliation-facets) between a reconciled value and an original cell value using [recon variables](#reconciliation): - -``` - return cell["recon"]["features"]["nameLevenshtein"] -``` - -To return the lower case of `value` (if the value is not null): - -``` - if value is not None: - return value.lower() - else: - return None -``` - -### Tutorials {#tutorials} -- [Extending Jython with pypi modules](https://github.com/OpenRefine/OpenRefine/wiki/Extending-Jython-with-pypi-modules) -- [Working with phone numbers using Java libraries inside Python](https://github.com/OpenRefine/OpenRefine/wiki/Jython#tutorial---working-with-phone-numbers-using-java-libraries-inside-python) - -Full documentation on the Jython language can be found on its official site: [http://www.jython.org](http://www.jython.org). - -## Clojure {#clojure} - -Clojure 1.10.1 comes bundled with the default installation of OpenRefine 3.4.1. At this time, not all [variables](expressions#variables) can be used with Clojure expressions: only `value`, `row`, `rowIndex`, `cell`, and `cells` are available. - -For example, functions can take the form -``` -(.. value (toUpperCase) ) -``` - -Or can look like -``` -(-> value (str/split #" ") last ) -``` - -which functions like `value.split(" ")` in GREL. - -For help with syntax, see the [Clojure website's guide to syntax](https://clojure.org/guides/learn/syntax). - -User-contributed Clojure recipes can be found on our wiki at [https://github.com/OpenRefine/OpenRefine/wiki/Recipes#11-clojure](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#11-clojure). - -Full documentation on the Clojure language can be found on its official site: [https://clojure.org/](https://clojure.org/). \ No newline at end of file diff --git a/OpenRefine/docs/docs/manual/reconciling.md b/OpenRefine/docs/docs/manual/reconciling.md deleted file mode 100644 index 0d14e80ad..000000000 --- a/OpenRefine/docs/docs/manual/reconciling.md +++ /dev/null @@ -1,242 +0,0 @@ ---- -id: reconciling -title: Reconciling -sidebar_label: Reconciling ---- - -## Overview {#overview} - -Reconciliation is the process of matching your dataset with that of an external source. Datasets for comparison might be produced by libraries, archives, museums, academic organizations, scientific institutions, non-profits, or interest groups. You can also reconcile against user-edited data on [Wikidata or other Wikibase instances](wikibase/reconciling), or reconcile against [a local dataset that you yourself supply](https://github.com/OpenRefine/OpenRefine/wiki/Reconcilable-Data-Sources#local-services). - -To reconcile your OpenRefine project against an external dataset, that dataset must offer a web service that conforms to the [Reconciliation Service API standards](https://reconciliation-api.github.io/specs/0.1/). - -You may wish to reconcile in order to: -* fix spelling or variations in proper names -* clean up manually-entered subject headings against authorities such as the [Library of Congress Subject Headings](https://id.loc.gov/authorities/subjects.html) (LCSH) -* link your data to an existing dataset -* add to an editable platform such as [Wikidata](https://www.wikidata.org) -* or see whether entities in your project appear in some specific list, such as the [Panama Papers](https://aleph.occrp.org/datasets/734). - -Reconciliation is semi-automated: OpenRefine matches your cell values to the reconciliation information as best it can, but human judgment is required to review and approve the results. Reconciling happens by default through string searching, so typos, whitespace, and extraneous characters will have an effect on the results. You may wish to [clean and cluster](cellediting) your data before reconciliaton. - -:::info Working iteratively -We recommend planning your reconciliation operations as iterative: reconcile multiple times with different settings, and with different subgroups of your data. -::: - -## Sources {#sources} - -Start with [this current list of reconcilable authorities](https://reconciliation-api.github.io/testbench/), which includes instructions for adding new services via Wikidata editing if you have one to add. - -OpenRefine maintains a [further list of sources on the wiki](https://github.com/OpenRefine/OpenRefine/wiki/Reconcilable-Data-Sources), which can be edited by anyone. This list includes ways that you can reconcile against a [local dataset](https://github.com/OpenRefine/OpenRefine/wiki/Reconcilable-Data-Sources#local-services). - -Other services may exist that are not yet listed in these two places: for example, the [310 datasets hosted by the Organized Crime and Corruption Reporting Project (OCCRP)](https://aleph.occrp.org/datasets/) each have their own reconciliation URL, or you can reconcile against their entire database with the URL [shared on the reconciliation API list](https://reconciliation-api.github.io/testbench/). For another example, you can reconcile against the entire Virtual International Authority File (VIAF) dataset, or [only the contributions from certain institutions](http://refine.codefork.com/). Search online to see if the authority you wish to reconcile against has an available service, or whether you can download a copy to reconcile against locally. - -OpenRefine includes Wikidata reconciliation in the installation package - see the [Wikibase](wikibase/reconciling) page for more information particular to that service. Extensions can add reconciliation services, and can also add enhanced reconciliation capacities. Check the list of extensions on the [Downloads page](https://openrefine.org/download.html) for more information. - -Each source will have its own documentation on how it provides reconciliation. The table on [the reconciliation API list](https://reconciliation-api.github.io/testbench/) indicates whether your chosen service supports the features described below. Refer to the service's documentation if you have questions about its behaviors and which OpenRefine features it supports. - -In addition to the reconciliation services mentioned above, you may also choose to build your own service. You can either start from scratch using the [API specification](https://reconciliation-api.github.io/specs/latest/) or use one of the frameworks mentioned in the [Reconciliation census](https://reconciliation-api.github.io/census/services/). - -Of particular note is [reconcile-csv](http://okfnlabs.org/reconcile-csv/) which allows you to build a reconciliation service from a simple CSV file. Thus if you wanted to reconcile one OpenRefine project against another, you'd simply need to export the target project as a CSV, point `reconcile-csv` at it and you're good to go. A somewhat newer port of this project written in Python can be found at [csv-reconcile](https://github.com/gitonthescene/csv-reconcile) which is more configurable and defaults to parsing tab separated files for convenience. - -Similiarly, you may choose to export some SPARQL output to a TSV to limit the scope of values you're reconciling against and/or for better peformance. - -## Getting started {#getting-started} - -Choose a column to reconcile and use its dropdown menu to select ReconcileStart reconciling. If you want to reconcile only some cells in that column, first use filters and facets to isolate them. - -In the reconciliation window, you will see Wikidata offered as a default service. To add another service, click Add Standard Service... and paste in the URL of a [service](#sources). You should see the name of the service appear in the list of Services if the URL is correct. - -![The reconciliation window.](/img/reconcilewindow.png) - -Once you select a service, your selected column may be sampled in order to suggest [“types” (categories)](#reconciling-by-type) to reconcile against. Other services will suggest their available types without sampling, and some services have no types. - -For example, if you had a list of artists represented in a gallery collection, you could reconcile their names against the Getty Research Institute’s [Union List of Artist Names (ULAN)](https://www.getty.edu/research/tools/vocabularies/ulan/). The same [Getty reconciliation URL](https://services.getty.edu/vocab/reconcile/) will offer you ULAN, AAT (Art and Architecture Thesaurus), and TGN (Thesaurus of Geographic Names). - -![The reconciliation window with types.](/img/reconcilewindow2.png) - -Refer to the [documentation specific to the reconciliation service](https://reconciliation-api.github.io/testbench/) to learn whether types are offered, which types are offered, and which one is most appropriate for your column. You may wish to facet your data and reconcile batches against different types if available. - -Reconciliation can be a time-consuming process, especially with large datasets. We suggest starting with a small test batch. There is no throttle (delay between requests) to set for the reconciliation process. The amount of time will vary for each service, and vary based on the options you select during the process. - -When the process is done, you will see the reconciliation data in the cells. -If the cell was successfully matched, it displays text as a single dark blue link. In this case, the reconciliation is confident that the match is correct, and you should not have to check it manually. -If there is no clear match, one or more candidates are displayed, together with their reconciliation score, with the text in light blue links. You will need to select the correct one. - -For each matching decision you make, you have two options: match this cell only (one checkmark), or also use the same identifier for all other cells containing the same original string (two checkmarks). - -For services that offer the [“preview entities” feature](https://reconciliation-api.github.io/testbench/), you can hover your mouse over the suggestions to see more information about the candidates or matches. Each participating service (and each type) will deliver different structured data that may help you compare the candidates. - -For example, the Getty ULAN shows an artist’s discipline, nationality, and birth and death years: - -![Hovering over matches.](/img/reconcilehover.png) - -Hovering over the suggestion will also offer the two matching options as buttons. - -For matched values (those appearing as dark blue links), the underlying cell value has not been altered - the cell is storing both the original string and the matched entity link at the same time. If you were to copy your column to a new column at this point using `value`, for example, the reconcilation data would not transfer - only the original strings. You can learn more about how OpenRefine stores different pieces of information in each cell in [the Variables section specific to reconciliation data](expressions#reconciliation). - -For each cell, you can manually “Create new item,” which will take the cell’s original value and apply it, as though it is a match. This will not become a dark blue link, because at this time there is nothing to link to: it is a draft entity stored only in your project. You can use this feature to prepare these entries for eventual upload to an editable service such as [Wikibase](wikibase/overview), but most services do not yet support this feature. - -### Reconciliation facets {#reconciliation-facets} - -Under ReconcileFacets there are a number of reconciliation-specific faceting options. OpenRefine automatically creates two facets when you reconcile some cells. - -One is a numeric facet for “best candidate's score,” the range of reconciliation scores of only the best candidate of each cell. Higher scores mean better matches, although each service calculates scores differently and has a different range. You can facet for higher scores using the numeric facet, and then approve them all in bulk, by using Reconcile[Actions](#reconciliation-actions)Match each cell to its best candidate. - -There is also a “judgment” facet created, which lets you filter for the cells that haven't been matched (pick “none” in the facet). As you process each cell, its judgment changes from “none” to “matched” and it disappears from the view. - -You can add other facets by selecting ReconcileFacets on your reconciled column. You can facet by: - -* your judgments (“matched,” or “none” for unreconciled cells, or “new” for entities you've created) -* the action you’ve performed on that cell (chosen a “single” match, or set a “mass” match, or no action, which appears as “unknown”) -* the timestamps on the edits you’ve made so far (these appear as millisecond counts since an arbitrary point: they can be sorted alphabetically to move forward and back in time). - -You can facet only the best candidates for each cell, based on: -* the score (calculated based on each service's own methods) -* the edit distance (using the [Levenshtein distance](cellediting#nearest-neighbor), a number based on how many single-character edits would be required to get your original value to the candidate value, with a larger value being a greater difference) -* the word similarity. - -Word similarity is calculated as a percentage based on how many words (excluding [stop words](https://en.wikipedia.org/wiki/Stop_word)) in the original value match words in the candidate. For example, the value “Maria Luisa Zuloaga de Tovar” matched to the candidate “Palacios, Luisa Zuloaga de” results in a word similarity value of 0.6, or 60%, or 3 out of 5 words. Cells that are not yet matched to one candidate will show as 0.0). - -You can also look at each best candidate’s: -* type (the ones you have selected in successive reconciliation attempts, or other types returned by the service based on the cell values) -* type match (“true” if you selected a type and it succeeded, “false” if you reconciled against no particular type, and “(no type)” if it didn’t reconcile) -* name match (“true” if you’ve matched, “false” if you haven’t yet chosen from the candidates, or “(unreconciled)” if it didn’t reconcile). - -These facets are useful for doing successive reconciliation attempts, against different types, and with different supplementary information. The information represented by these facets are held in the cells themselves and can be called using the [reconciliation variables](expressions#reconciliation) available in expressions. - -### Reconciliation actions {#reconciliation-actions} - -You can use the ReconcileActions menu options to perform bulk changes (which will apply only to your currently viewed set of rows or records): -* Match each cell to its best candidate (by highest score) -* Create a new item for each cell (discard any suggested matches) -* Create one new item for similar cells (a new entity will be created for each unique string) -* Match all filtered cells to... (a specific item from the chosen service, via a search box; only works with services that support the “suggest entities” property) -* Discard all reconciliation judgments (reverts back to multiple candidates per cell, including cells that may have been auto-matched in the original reconciliation process) -* Clear reconciliation data, reverting all cells back to their original values. - -The other options available under Reconcile are: -* Copy reconciliation data... (to an existing column: if the original values in your reconciliation column are identical to those in your chosen column, the matched and new cells will copy over; unmatched values will not change) -* [Use values as identifiers](#reconciling-with-unique-identifiers) (if you are reconciling with unique identifiers instead of by doing string searches) -* [Add entity identifiers column](#add-entity-identifiers-column). - -## Reconciling with unique identifiers {#reconciling-with-unique-identifiers} - -Reconciliation services use unique identifiers for their entities. For example, the 14th Dalai Lama has the VIAF ID [38242123](https://viaf.org/viaf/38242123/) and the Wikidata ID [Q17293](https://www.wikidata.org/wiki/Q37349). You can supply these identifiers directly to your chosen reconciliation service in order to pull more data, but these strings will not be “reconciled” against the external dataset. - -Select the column with unique identifiers and apply the operation ReconcileUse values as identifiers. This will bring up the list of reconciliation services you have already added (to add a new service, open the Start reconciling... window first). If you use this operation on a column of IDs, you will not have access to the usual reconciliation settings. - -Matching identifiers does not validate them. All cells will appear as dark blue “confirmed” matches. You should check before this operation that the identifiers in the column exist on the target service. - -You may get false positives, which you will need to hover over or click on to identify: - -![Hovering over an error.](/img/reconcileIDerror.png) - -## Reconciling by type {#reconciling-by-type} - -Reconciliation services, once added to OpenRefine, may suggest types from their databases. These types will usually be whatever the service specializes in: people, events, places, buildings, tools, plants, animals, organizations, etc. - -Reconciling against a type may be faster and more accurate, but may result in fewer matches. Some services have hierarchical types (such as “mammal” as a subtype of “animal”). When you reconcile against a more specific type, unmatched values may fall back to the broader type; other services will not do this, so you may need to perform successive reconciliation attempts against different types. Refer to the documentation specific to the reconciliation service to learn more. - -When you select a service from the list, OpenRefine will load some or all available types. Some services will sample the first ten rows of your column to suggest types (check the [“Suggest types” column](https://reconciliation-api.github.io/testbench/)). You will see a service’s types in the reconciliation window: - -![Reconciling using a type.](/img/reconcile-by-type.png) - -In this example, “Person” and “Corporate Name” are potential types offered by the reconciliation API for VIAF. You can also use the Reconcile against type: field to enter in another type that the service offers. When you start typing, this field may search and suggest existing types. For VIAF, you could enter “/book/book” if your column contained publications. You may need to enter the service's own strings precisely instead of attempting to search for a match. - -Types are structured to fit their content: the Wikidata “human” type, for example, can include fields for birth and death dates, nationality, etc. The VIAF “person” type can include nationality and gender. You can use this to [include more properties](#reconciling-with-additional-columns) and find better matches. - -If your column doesn’t fit one specific type offered, you can Reconcile against no particular type. This may take longer. - -We recommend working in batches and reconciling against different types, moving from specific to broad. You can create a facet for Best candidate’s types facet to see which types are being represented. Some candidates may return more than one type, depending on the service. Types may appear in facets by their unique IDs, rather than by their semantic labels (for example, Q5 for “human” in Wikidata). - -## Reconciling with additional columns {#reconciling-with-additional-columns} - -Some of your cells may be ambiguous, in the sense that a string can point to more than one entity: there are dozens of places called “Paris” and many characters, people, and pieces of culture, too. Selecting non-geographic or more localized types can help narrow that down, but if your chosen service doesn't provide a useful type, you can include more properties that make it clear whether you're looking for Paris, France. - -![Reconciling sometimes turns up ambiguous matches.](/img/reconcileParis.gif) - -Including supplementary information can be useful, depending on the service (such as including birthdate information about each person you are trying to reconcile). You can re-reconcile unmatched cells with additional properties, in the right side of the Start reconciling window, under “Also use relevant details from other columns.” The column names in your project will appear in the reconciliation window, with an Include? checkbox next to each one. - -Fill in the As Property field with the type of information you are including. When you start typing, potential fields may pop up (depending on the [“suggest properties” feature](https://reconciliation-api.github.io/testbench/)), such as “birthDate” in the case of ULAN or “Geburtsdatum” in the case of Integrated Authority File (GND). Use the documentation for your chosen service to identify the fields in their terms. - -Some services will not be able to search for the exact name of your desired As Property entry, but you can still manually supply the field name. Refer to the service to choose the most appropriate field, and make sure you enter it correctly. - -![Including a birth-date type.](/img/reconcile-with-property.png) - -## Fetching more data {#fetching-more-data} - -One reason to reconcile to some external service is that it allows you to pull data from that service into your OpenRefine project. There are three ways to do this: - -* Add identifiers for your values -* Add columns from reconciled values -* Add column by fetching URLs. - -### Add entity identifiers column {#add-entity-identifiers-column} - -Once you have selected matches for your cells, you can retrieve the unique identifiers for those cells and create a new column for these, with ReconcileAdd entity identifiers column. You will be asked to supply a column name. New items and other unmatched cells will generate null values in this column. - -### Add columns from reconciled values {#add-columns-from-reconciled-values} - -If the reconciliation service supports [data extension](https://reconciliation-api.github.io/testbench/), then you can augment your reconciled data with new columns using Edit columnAdd columns from reconciled values.... - -For example, if you have a column of chemical elements identified by name, you can fetch categorical information about them such as their atomic number and their element symbol: - -![A screenshare of elements fetching related information.](/img/reconcileelements.gif) - -Once you have chosen reconciliation matches for your cells, selecting Add column from reconciled values... will bring up a window to choose which related information you’d like to import into new columns. You can manually enter desired properties, or select from a list of suggestions. - -The quality of the suggested properties will depend on how you have reconciled your data beforehand: reconciling against a specific type will provide you with the associated properties of that type. For example, GND suggests elements about the “people” type after you've reconciled with it, such as their parents, native languages, children, etc. - -![A screenshot of available properties from GND.](/img/reconcileGND.png) - -If you have left any values unreconciled in your column, you will see “<not reconciled>” in the preview. These will generate blank cells if you continue with the column addition process. - -This process may pull more than one property per row in your data (such as multiple occupations), so you may need to switch into records mode after you've added columns. - -### Add columns by fetching URLs {#add-columns-by-fetching-urls} - -If the reconciliation service cannot extend data, look for a generic web API for that data source, or a structured URL that points to their dataset entities via unique IDs (such as “https://viaf.org/viaf/000000”). You can use the Edit column[Add column by fetching URLs](columnediting#add-column-by-fetching-urls) operation to call this API or URL with the IDs obtained from the reconciliation process. This will require using [expressions](expressions). - -You may not want to pull the entire HTML content of the pages at the ends of these URLs, so look to see whether the service offers a metadata endpoint, such as JSON-formatted data. You can either use a column of IDs, or you can pull the ID from each matched cell during the fetching process. - -For example, if you have reconciled artists to the Getty's ULAN, and [have their unique ULAN IDs as a column](#add-entity-identifiers-column), you can generate a new column of JSON-formatted data by using Add column by fetching URLs and entering the GREL expression `"http://vocab.getty.edu/" + value + ".json"`. For this service, the unique IDs are formatted “ulan/000000” and so the generated URLs look like “http://vocab.getty.edu/ulan/000000.json”. - -Alternatively, you can insert the ID directly from the matched column's reconciliation variables, using a GREL expression like `“http://vocab.getty.edu/” + cell.recon.match.id + “.json”` instead. - -Remember to set an appropriate throttle and to refer to the service documentation to ensure your compliance with their terms. See [the section about this operation](columnediting#add-column-by-fetching-urls) to learn more about the fetching process. - -## Keep all the suggestions made {#keep-all-the-suggestions-made} - -To generate a list of each suggestion made, rather than only the best candidate, you can use a [GREL expression](expressions#GREL). Go to Edit columnAdd column based on this column. To create a list of all the possible matches, use something like - -``` -forEach(cell.recon.candidates,c,c.name).join(", ") -``` - -To get the unique identifiers of these matches instead, use - -``` -forEach(cell.recon.candidates,c,c.id).join(", ") -``` - -This information is stored as a string, without any attached reconciliation information. - -## Writing reconciliation expressions {#writing-reconciliation-expressions} - -OpenRefine supplies a number of variables related specifically to reconciled values. These can be used in GREL and Jython expressions. For example, some of the reconciliation variables are: - -* `cell.recon.match.id` or `cell.recon.match.name` for matched values -* `cell.recon.best.name` or `cell.recon.best.id` for best-candidate values -* `cell.recon.candidates` for all listed candidates of each cell -* `cell.recon.judgment` (the values used in the “judgment” facet) -* `cell.recon.judgmentHistory` (the values used in the “judgment action timestamp” facet) -* `cell.recon.matched` (a “true” or “false” value) - -You can find out more in the [reconciliaton variables](expressions#reconciliaton-variables) section. - -## Exporting reconciled data {#exporting-reconciled-data} - -Once you have data that is reconciled to existing entities online, you may wish to export that data to a user-editable service such as Wikidata. See the section on [uploading your edits to Wikidata or other Wikibase instances](wikibase/uploading) for more information, or the section on [exporting](exporting) to see other formats OpenRefine can produce. - -You can share reconciled data in progress through a [project export or import](exporting#export-a-project), with some preparation. The importing user needs to have the appropriate reconciliation services installed on their OpenRefine instance (by going to Start reconciling and clicking on Add Standard Service...) in advance of opening the project, in order to use candidate and match links. Otherwise, the links will be broken and the user will need to add the reconciliation service and re-reconcile the columns in question. [Wikidata](wikibase/reconciling) reconciliation data can be shared more easily as the service comes bundled with OpenRefine. diff --git a/OpenRefine/docs/docs/manual/running.md b/OpenRefine/docs/docs/manual/running.md deleted file mode 100644 index e23317562..000000000 --- a/OpenRefine/docs/docs/manual/running.md +++ /dev/null @@ -1,509 +0,0 @@ ---- -id: running -title: Running OpenRefine -sidebar_label: Running ---- - -## Starting and exiting {#starting-and-exiting} - -OpenRefine does not require internet access to run its basic functions. Once you download and install it, it runs as a small web server on your own computer, and you access that local web server by using your browser. - -You will see a command line window open when you run OpenRefine. Ignore that window while you work on datasets in your browser. - -No matter how you start OpenRefine, it will load its interface in your computer’s default browser. If you would like to use another browser instead, start OpenRefine and then point your chosen browser at the home screen: [http://127.0.0.1:3333/](http://127.0.0.1:3333/). - -OpenRefine works best on browsers based on Webkit, such as: -* Google Chrome -* Chromium -* Opera -* Microsoft Edge - -We are aware of some minor rendering and performance issues on other browsers such as Firefox. We don't support Internet Explorer. - -You can view and work on multiple projects at the same time by simply having multiple tabs or browser windows open. From the Open Project screen, you can right-click on project names and open them in new tabs or windows. - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - - -#### With openrefine.exe {#with-openrefineexe} -You can run OpenRefine by double-clicking `openrefine.exe` or calling it from the command line. - -If you want to [modify the way `openrefine.exe` opens](#starting-with-modifications), you can edit the `openrefine.l4j.ini` file. - -#### With refine.bat {#with-refinebat} -On Windows, OpenRefine can also be run by using the file `refine.bat` in the program directory. If you start OpenRefine using `refine.bat`, you can do so by opening the file itself, or by calling it from the command line. - -If you call `refine.bat` from the command line, you can [start OpenRefine with modifications](#starting-with-modifications). -If you want to modify the way `refine.bat` opens through double-clicking or using a shortcut, you can edit the `refine.ini` file. - -#### Exiting {#exiting} - -To exit OpenRefine, close all the browser tabs or windows, then navigate to the command line window. To close this window and ensure OpenRefine exits properly, hold down `Control` and press `C` on your keyboard. This will save any last changes to your projects. - - - - - -You can find OpenRefine in your Applications folder, or you can open it using Terminal. - -To run OpenRefine using Terminal: -* Find the OpenRefine application / icon in Finder -* Control-click on the icon and select “Show Package Contents” from the context menu -* This should open a new Finder menu: navigate into the “MacOS” folder -* Control-click on “JavaAppLauncher” -* Choose “Open With” from the menu, and select “Terminal.” - -To exit, close all your OpenRefine browser tabs, go back to the terminal window and press `Command` and `Q` to close it down. - -:::caution Problems starting? -If you are using an older version of OpenRefine or are on an older version of MacOS, [check our Wiki for solutions to problems with MacOS](https://github.com/OpenRefine/OpenRefine/wiki/Installation-Instructions#macos). -::: - - - - - -Use a terminal to launch OpenRefine. First, navigate to the installation folder. Then call the program: - -``` -cd openrefine-3.4.1 - ./refine -``` - -This will start OpenRefine and open your browser to the home screen. - -To exit, close all the browser tabs, and then press `control` and `C` in the terminal window. - -:::caution Did you get a JAVA_HOME error? -“Error: Could not find the ‘java’ executable at ‘’, are you sure your JAVA_HOME environment variable is pointing to a proper java installation?” - -If you see this error, you need to [install and configure a JDK package](installing#linux), including setting up `JAVA_HOME`. -::: - - - - - ---- - -### Troubleshooting {#troubleshooting} - -If you are having problems connecting to OpenRefine with your browser, [check our Wiki for information about browser settings and operating-system issues](https://github.com/OpenRefine/OpenRefine/wiki/FAQ#i-am-having-trouble-connecting-to-openrefine-with-my-browser). - -### Starting with modifications {#starting-with-modifications} - -When you run OpenRefine from a command line, you can change a number of default settings. - - - - - -On Windows, use a slash: - -``` -C:>refine /i 127.0.0.2 /p 3334 -``` - -Get a list of all the commands with `refine /?`. - -|Command|Use|Syntax example| -|---|---|---| -|/w|Path to the webapp|refine /w /path/to/openrefine| -|/m|Memory maximum heap|refine /m 6000M| -|/p|Port|refine /p 3334| -|/i|Interface (IP address, or IP and port)|refine /i 127.0.0.2:3334| -|/H|HTTP host to expect on incoming requests|refine /H openrefine.internal| -|/d|Enable debugging (on port 8000)|refine /d| -|/x|Enable JMX monitoring for Jconsole and JvisualVM|refine /x| - - - - - -You cannot start the Mac version with modifications using Terminal, but you can modify the way the application starts with [settings within files](#modifications-set-within-files). - - - - - -To see the full list of command-line options, run `./refine -h`. - -|Command|Use|Syntax example| -|---|---|---| -|-w|Path to the webapp|./refine -w /path/to/openrefine| -|-d|Path to the workspace|./refine -d /where/you/want/the/workspace| -|-m|Memory maximum heap|./refine -m 6000M| -|-p|Port|./refine -p 3334| -|-i|Interface (IP address, or IP and port)|./refine -i 127.0.0.2:3334| -|-H|HTTP host to expect on incoming requests|./refine -H openrefine.internal| -|-k|Add a Google API key|./refine -k YOUR_API_KEY| -|-v|Verbosity (from low to high: error,warn,info,debug,trace)|./refine -v info| -|-x|Additional Java configuration parameters (see Java documentation)|| -|--debug|Enable debugging (on port 8000)|./refine --debug| -|--jmx|Enable JMX monitoring for Jconsole and JvisualVM|./refine --jmx| - - - - - ---- - -#### Modifications set within files {#modifications-set-within-files} - -On Windows, you can modify the way `openrefine.exe` runs by editing `openrefine.l4j.ini`; you can modify the way `refine.bat` runs by editing `refine.ini`. - -You can modify the Mac application by editing `info.plist`. - -On Linux, you can edit `refine.ini`. - -Some settings, such as changing memory allocations, are already set inside these files, and all you have to do is change the values. Some lines need to be un-commented to work. - -For example, inside `refine.ini`, you should see: -``` -no_proxy="localhost,127.0.0.1" -#REFINE_PORT=3334 -#REFINE_HOST=127.0.0.1 -#REFINE_WEBAPP=main\webapp - -# Memory and max form size allocations -#REFINE_MAX_FORM_CONTENT_SIZE=1048576 -REFINE_MEMORY=1400M - -# Set initial java heap space (default: 256M) for better performance with large datasets -REFINE_MIN_MEMORY=1400M -... -``` - -##### JVM preferences {#jvm-preferences} - -Further modifications can be performed by using JVM preferences. These JVM preferences are different options and have different syntax than the key/value descriptions used on the command line. - -Some of the most common keys (with their defaults) are: -* The project [autosave](starting#autosaving) frequency: `-Drefine.autosave` (5 [minutes]) -* The workspace director: `-Drefine.data_dir` (/) -* Development mode: `-Drefine.development` (false) -* Headless mode: `-Drefine.headless` (false) -* IP: `-Drefine.host` (127.0.0.1) -* Port: `-Drefine.port` (3333) -* The application folder: `-Drefine.webapp` (main/webapp) - -The syntax is as follows: - - - - - -Locate the `refine.l4j.ini` file, and insert lines in this way: - -``` --Drefine.port=3334 --Drefine.host=127.0.0.2 --Drefine.webapp=broker/core -``` - -In `refine.ini`, use a similar syntax, but set multiple parameters within a single line starting with `JAVA_OPTIONS=`: - -``` -JAVA_OPTIONS=-Drefine.data_dir=C:\Users\user\Documents\OpenRefine\ -Drefine.port=3334 - -``` - - - - -Locate the `info.plist`, and find the `array` element that follows the line - -``` -JVMOptions -``` - -Typically this looks something like: - -``` -JVMOptions - --Xms256M --Xmx1024M --Drefine.version=2.6-beta.1 --Drefine.webapp=$APP_ROOT/Contents/Resource/webapp - -``` - -Add in values such as: - -``` -JVMOptions - --Xms256M --Xmx1024M --Drefine.version=2.6-beta.1 --Drefine.webapp=$APP_ROOT/Contents/Resource/webapp --Drefine.autosave=2 --Drefine.port=3334 - - -``` - - - - - -Locate the `refine.ini` file, and add `JAVA_OPTIONS=` before the `-Drefine.preference` declaration. You can un-comment and edit the existing suggested lines, or add lines: - -``` -JAVA_OPTIONS=-Drefine.autosave=2 -JAVA_OPTIONS=-Drefine.port=3334 -JAVA_OPTIONS=-Drefine.data_dir=usr/lib/OpenRefineWorkspace -``` - - - - - - ---- - -Refer to the [official Java documentation](https://docs.oracle.com/javase/8/docs/technotes/tools/windows/java.html) for more preferences that can be set. - -## The home screen {#the-home-screen} - -When you first launch OpenRefine, you will see a screen with a menu on the left hand side that includes Create Project, Open Project, Import Project, and Language Settings. This is called the “home screen,” where you can manage your projects and general settings. - -In the lower left-hand corner of the screen, you'll see Preferences, Help, and About. - -### Language settings {#language-settings} - -From the home screen, look in the options to the left for Language Settings. You can set your preferred interface language here. This language setting will persist until you change it again in the future. Languages are translated as a community effort; some languages are partially complete and default back to English where unfinished. Currently OpenRefine supports the following languages for 75% or more of the interface: - -* Cebuano -* German -* English (UK) -* English (US) -* Spanish -* Filipino -* French -* Hebrew -* Magyar -* Italian -* Japanese (日本語) -* Portuguese (Brazil) -* Tagalog -* Chinese (简体中文) - -To leave the Language Settings screen, click on the diamond “OpenRefine” logo. - -:::info Help us Translate OpenRefine -We use Weblate to provide translations for the interface. You can check [our profile on Weblate](https://hosted.weblate.org/projects/openrefine/translations/) to see which languages are in the process of being supported. See [our technical reference if you are interested in contributing translation work](../technical-reference/translating-ui) to make OpenRefine accessible to people in other languages. -::: - -### Preferences {#preferences} - -In the bottom left corner of the screen, look for Preferences. At this time you can set preferences using a key/value pair: that is, selecting one of the keys below and setting a value for it. - -|Setting|Key|Value syntax|Default|Example|Version| -|---|---|---|---|---|---| -|Interface language|userLang|[ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) two-digit code|en|fr|—| -|Maximum facets|ui.browsing.listFacet.limit|Number|2000|5000|—| -|Timeout for Google Drive import|googleReadTimeOut|Number (microseconds)|180000|500000|—| -|Timeout for Google Drive authorization|googleConnectTimeOut|Number (microseconds)|180000|500000|—| -|Maximum lag for Wikibase edit retries|wikibase.upload.maxLag|Number (seconds)|5|10|—| -|Display of the reconciliation preview on hover|cell-ui.previewMatchedCells|Boolean|true|false|v3.2| -|Values for the choice of the number of rows to display|ui.browsing.pageSize|Array of number (JSON)|[ 5, 10, 25, 50 ]|[ 100,500,1000 ]|v3.4| -|Width of the panel for facets/history|ui.browsing.facetsHistoryPanelWidth|Number (pixel)|300|500|v3.5| - -To leave the Preferences screen, click on the diamond “OpenRefine” logo. - -If the preference you’re looking for isn’t here, look at the options you can set from the [command line or in an `.ini` file](#starting-with-modifications). - -## The project screen {#the-project-screen} - -The project screen (or work screen) is where you will spend most of your time once you have [begun to work on a project](starting). This is a quick walkthrough of the parts of the interface you should familiarize yourself with. - -![A screenshot of the project screen.](/img/projectscreen.png) - -### The project bar {#the-project-bar} - -The project bar runs across the very top of the project screen. It contains the the OpenRefine logo, the project title, and the project control buttons on the right side. - -At any time you can close your current project and go back to the home screen by clicking on the OpenRefine logo. If you’d like to open another project in a new browser tab or window, you can right-click on the logo and use “Open in a new tab.” You will lose [your current facets and view settings](#facetfilter) if you close your project (but data transformations will be saved in the [History](#history-undoredo) of the project). - -:::caution -Don’t click the “back” button on your browser - it will likely close your current project and you will lose your facets and view settings. -::: - -You can rename a project at any time by clicking inside the project title, which will turn into a text field. Project names don’t have to be unique, as OpenRefine organizes them based on a unique identifier behind the scenes. - -The Permalink allows you to return to a project at a specific view state - that is, with [facets and filters](facets) applied. The Permalink can help you pick up where you left off if you have to close your project while working with facets and filters. It puts view-specific information directly into the URL: clicking on it will load this current-view URL in the existing tab. You can right-click and copy the Permalink URL to copy the current view state to your clipboard, without refreshing the tab you’re using. - -The Open… button will open up a new browser tab showing the Create Project screen. From here you can change settings, start a new project, or open an existing project. - -Export is a dropdown menu that allows you to pick a format for exporting a dataset. Many of the export options will only export rows and records that are currently visible - the currently selected facets and filters, not the total data in the project. - -Help will open up a new browser tab and bring you to this user manual on the web. - -### The grid header {#the-grid-header} - -The grid header sits below the project bar and above the project grid (where the data of your project is displayed). The grid header will tell you the total number of rows or records in your project, and indicate whether you are in [rows or records mode](exploring#rows-vs-records). - -It will also tell you if you’re currently looking at a select number of rows via facets or filtering, rather than the entire dataset, by displaying either, for example, “180 rows” or “67 matching rows (180 total).” - -Directly below the row number, you have the ability to switch between [row mode and records mode](exploring#rows-vs-records). OpenRefine stores projects persistently in one of the two modes, and displays your data as records by default if you are. - -To the right of the rows/records selection is the array of options for how many rows/records to view on screen at one time. At the far right of the screen you can navigate through your entire dataset one page at a time. - -### Extensions {#extensions} - -The Extensions dropdown offers you options for extending your data - most commonly by uploading your edited statements to Wikidata, or by importing or exporting schema. You can learn more about these functions on the [Wikibase section](wikibase/overview). Other extensions may also add functions to this dropdown menu. - -### The grid {#the-grid} - -The area of the project screen that displays your dataset is called the “grid” (or the “data grid,” or the “project grid”). The grid presents data in a tabular format, which may look like a normal spreadsheet program to you. - -Columns widths are automatically set based on their contents; some column headers may be cut off, but can be viewed by mousing over the headers. - -In each column header you will see a small arrow. Clicking on this arrow brings up a dropdown menu containing column-specific data exploration and transformation options. You will learn about each of these options in the [Exploring data](exploring) and [Transforming data](transforming) sections. - -The first column in every project will always be All, which contains options to flag, star, and do non-column-specific operations. The All column is also where rows/records are numbered. Numbering shows the permanent order of rows and records; a temporary sorting or facet may reorder the rows or show a limited set, but numbering will show you the original identifiers unless you make a permanent change. - -The project grid may display with both vertical and horizontal scrolling, depending on the number and width of columns, and the number of rows/records displayed. You can control the display of the project grid by using [Sort and View options](exploring#sort-and-view). - -Mousing over individual cells will allow you to [edit cells individually](cellediting#edit-one-cell-at-a-time). - -### Facet/Filter {#facetfilter} - -The Facet/Filter tab is one of the main ways of exploring your data: displaying the patterns and trends in your data, and helping you narrow your focus and modify that data. [Facets](facets) and [filters](facets#text-filter) are explained more in [Exploring data](exploring). - -![A screenshot of facets and filters in action.](/img/facetfilter.png) - -In the tab, you will see three buttons: Refresh, Reset all, and Remove all. - -Refreshing your facets will ensure you are looking at the latest information about each facet, for example if you have changed the counts or eliminated some options. - -Resetting your facets will remove any inclusion or exclusion you may have set - the facet options will stay in the sidebar, but your view settings will be undone. - -Removing your facets will clear out the sidebar entirely. If you have written custom facets using [expressions](expressions), these will be lost. - -You can preserve your facets and filters for future use by copying a [Permalink](#the-project-bar). - -### History (Undo/Redo) {#history-undoredo} - -In OpenRefine, any activity that changes the data can be undone. Changes are tracked from the very beginning, when a project is first created. The change history of each project is saved with the project's data, so quitting OpenRefine does not erase the steps you've taken. When you restart OpenRefine, you can view and undo changes that you made before you quit OpenRefine. OpenRefine [autosaves](starting#autosaving) your actions every five minutes by default, and when you close OpenRefine properly (using Ctrl + C). You can [change this interval](running#jvm-preferences). - -Project history gets saved when you export a project archive, and restored when you import that archive to a new installation of OpenRefine. - -![A screenshot of the History (Undo/Redo) tab with 13 steps.](/img/history.png "A screenshot of the History (Undo/Redo) tab with 13 steps.") - -When you click on the Undo / Redo tab in the sidebar of any project, that project’s history is shown as a list of changes in order, with the first “change” being the action of creating the project itself. (That first change, indexed as step zero, cannot be undone.) Here is a sample history with 3 changes: - -``` -0. Create project -1. Remove 7 rows -2. Create new column Last Name based on column Name with grel:value.split(" ") -3. Split 230 cell(s) in column Address into several columns by separator -``` - -The current state of the project is highlighted with a dark blue background. If you move back and forth on the timeline you will see the current state become highlighted, while the actions that came after that state will be grayed out. - -To revert your data back to an earlier state, simply click on the last action in the timeline you want to keep. In the example above, if we keep the removal of 7 rows but revert everything we did after that, then click on “Remove 7 rows.” The last 2 changes will be undone, in order to bring the project back to state #1. - -In this example, changes #2 and #3 will now be grayed out. You can redo a change by clicking on it in the history - everything up to and including it will be redone. - -If you have moved back one or more states, and then you perform a new operation on your data, the later actions (everything that’s greyed out) will be erased and cannot be re-applied. - -The Undo/Redo tab will indicate which step you’re on, and if you’re about to risk erasing work - by saying something like “4/5" or “1/7” at the end. - -#### Reusing operations {#reusing-operations} - -Operations that you perform in OpenRefine can be reused. For example, a formula you wrote inside one project can be copied and applied to another project later. - -To reuse one or more operations, first extract it from the project where it was first applied. Click to the Undo/Redo tab and click Extract…. This brings up a box that lists all operations up to the current state (it does not show undone operations). Select the operation or operations you want to extract using the checkboxes on the left, and they will be encoded as JSON on the right. Copy that JSON to the clipboard. - -Move to the second project, go to the Undo/Redo tab, click Apply… and paste in that JSON. - -Not all operations can be extracted. Edits to a single cell, for example, can’t be replicated. - -## Advanced OpenRefine uses {#advanced-openrefine-uses} - -### Running OpenRefine's Linux version on a Mac {#running-openrefines-linux-version-on-a-mac} - -You can run OpenRefine from the command line in Mac by using the Linux installation package. We do not promise support for this method. Follow the instructions in the Linux section. - -### Running as a server {#running-as-a-server} - -:::caution -Please note that if your machine has an external IP (is exposed to the Internet), you should not do this, or should protect it behind a proxy or firewall, such as nginx. Proceed at your own risk. -::: - -By default (and for security reasons), OpenRefine only listens to TCP requests coming from localhost (127.0.0.1) on port 3333. If you want to share your OpenRefine instance with colleagues and respond to TCP requests to any IP address of the machine, start it from the command line like this: -``` -./refine -i 0.0.0.0 -``` - -or set this option in `refine.ini`: -``` -REFINE_HOST=0.0.0.0 -``` - -or set this JVM option: -``` --Drefine.host=0.0.0.0 -``` - -On Mac, you can add a specific entry to the `Info.plist` file located within the app bundle (`/Applications/OpenRefine.app/Contents/Info.plist`): -``` -JVMOptions - - - -Drefine.host=0.0.0.0 - … - -``` - -:::caution -OpenRefine has no built-in security or version control for multi-user scenarios. OpenRefine has a single data model that is not shared, so there is a risk of data operations being overwritten by other users. Care must be taken by users. -::: - -### Automating OpenRefine {#automating-openrefine} - -Some users may wish to employ OpenRefine for batch processing as part of a larger automated pipeline. Not all OpenRefine features can work without human supervision and advancement (such as clustering), but many data transformation tasks can be automated. - -:::caution -The following are all third-party extensions and code; the OpenRefine team does not maintain them and cannot guarantee that any of them work. -::: - -Some examples: - -* This project allows OpenRefine to be run from the command line using [operations saved in a JSON file](#reusing-operations): [OpenRefine batch processing](https://github.com/opencultureconsulting/openrefine-batch) -* A Python project for applying a JSON file of operations to a data file, outputting the new file, and deleting the temporary project, written by David Huynh and Max Ogden: [Python client library for Google Refine](https://github.com/maxogden/refine-python) -* And the same in Ruby: [Refine-Ruby](https://github.com/maxogden/refine-ruby) -* Another Python client library, by Paul Makepeace: [OpenRefine Python Client Library](https://github.com/PaulMakepeace/refine-client-py) - -To look for other instances, search our Google Groups [for users](https://groups.google.com/g/openrefine) and [for developers](https://groups.google.com/g/openrefine-dev), where [these projects were originally posted](https://groups.google.com/g/openrefine/c/GfS1bfCBJow/m/qWYOZo3PKe4J). diff --git a/OpenRefine/docs/docs/manual/sortview.md b/OpenRefine/docs/docs/manual/sortview.md deleted file mode 100644 index fbb52717c..000000000 --- a/OpenRefine/docs/docs/manual/sortview.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -id: sortview -title: Sort and view -sidebar_label: Sort and view ---- - -## Sort {#sort} - -You can temporarily sort your rows by one column. You can sort based on [data type](exploring#data-types): -* text alphabetically or reverse -* numbers by largest or smallest -* dates by earliest or latest -* boolean values by false first or true first. - -You can also choose where to place errors and blank cells in the sorting. Text can be case-sensitive or not: if so, cells that start with lowercase characters will appear ahead of those that start with uppercase characters. - -![A screenshot of the Sort window.](/img/sort.png) - -After you apply a sorting method, you can make it permanent, remove it, reverse it, or apply a subsequent sorting. When it is applied, you’ll find Sort in the project grid header to the right of the rows-display setting, which will show all current sorting settings. - -If you have multiple sorting methods applied, they will work in the order you applied them (represented in order in the Sort menu). For example, you can sort an “authors” column alphabetically, and then sort their books by publication date, for those authors that have more than one book. If you apply those in a different order - sort all the publication dates in the dataset first, and then alphabetically by author - your dataset will look different. - -![Temporarily sorted rows.](/img/sort2.png) - -When the sorting method you've applied is temporary, you will see that the rows retain their original numbering. When you make that sorting method permanent, by selecting Reorder rows permanently, the row numbers will change and the Sort menu in the project grid header will disappear. This will apply all current sorting methods. - -## View {#view} - -You can control what data you view in the grid. On each column, you will see a View menu option. From there, you can “collapse” (hide) that specific column, all other columns, all columns to the left, and all columns to the right. Using the View option that appears in the All column’s dropdown menu, you can collapse all columns, and expand all the columns that you previously collapsed. - -### Show/hide “null” {#showhide-null} - -You can find, under AllView, the option to show and hide [“null” values](exploring#data-types). A small grey “null” will appear in each applicable cell. Remember that a null cell is not the same thing as an empty cell. - -![A screenshot of what a null value looks like.](/img/null.png) - -## Page navigation {#page-navigation} - -You can go directly to any page by changing the page number on the right-hand side. Using the up and down arrow keys in this input lets you go to the next and previous pages. You can also change the number of rows or records per page on the left-hand side of this view header bar. - -![A screenshot of the Page Navigation Feature.](/img/goto.png) diff --git a/OpenRefine/docs/docs/manual/starting.md b/OpenRefine/docs/docs/manual/starting.md deleted file mode 100644 index f40ec237c..000000000 --- a/OpenRefine/docs/docs/manual/starting.md +++ /dev/null @@ -1,194 +0,0 @@ ---- -id: starting -title: Starting a project -sidebar_label: Starting a project ---- - -## Overview {#overview} - -An OpenRefine project is started by importing in some existing data - OpenRefine doesn’t allow you to create a dataset from nothing. - -No matter where your data comes from, OpenRefine won’t modify your original data source. It copies all the information from your input, creates its own project file, and stores it in your [workspace directory](installing#set-where-data-is-stored). - -The data and all of your edits are [automatically saved](#autosaving) inside the project file. When you’re finished modifying the data, you can [export it back out](exporting) into the file format of your choice. - -You can also receive and open other people’s projects, or send them yours, by [exporting a project archive](exporting#export-a-project) and [importing it](#import-a-project). - -## Create a project by importing data {#create-a-project-by-importing-data} - -When you start OpenRefine, you’ll be taken to the Create Project screen. You’ll see on the left side of the screen that your options are to: - -* import data from one or more files on your computer -* import data from one or more links on the web -* import data by pasting in text from your clipboard -* import data from a database (using SQL), and -* import one or more Sheets from Google Drive. - -From these sources, you can load any of the following file formats: - -* comma-separated values (CSV) or text-separated values (TSV) -* Text files -* Fixed-width columns -* JSON -* XML -* OpenDocument spreadsheet (ODS) -* Excel spreadsheet (XLS or XLSX) -* PC-Axis (PX) -* MARC -* RDF data (JSON-LD, N3, N-Triples, Turtle, RDF/XML) -* Wikitext - -More formats can be imported by [adding extensions to provide that functionality](https://openrefine.org/download.html). - -If you supply two or more files for one project, the files’ rows will be loaded in the order that you specify, and OpenRefine will create a column at the beginning of the dataset with the source URL or file name in it to help you identify where each row came from. If the files have columns with identical names, the data will load in those columns; if not, the successive files will append all of their new columns to the end of the dataset: - -|File|Fruit|Quantity|Berry|Berry source| -|---|---|---|---|---| -|fruits.csv|Orange|4| -|fruits.csv|Apple|6| -|berries.csv||9|Mulberry|Greece| -|berries.csv||2|Blueberry|Canada| - -You cannot combine two datasets into one project by appending data within rows. You can, however, combine two projects later using functions such as [cross()](grelfunctions/#crosscell-s-projectname-s-columnname), or [fetch further data](columnediting) using other methods. - -For whichever method you choose to start your project, when you click Next >> you will be given a preview and a chance to configure the way OpenRefine interprets the data you input. - -### Get data from this computer {#get-data-from-this-computer} - -Click on Browse… and select a file (or several) on your hard drive. All files will be shown, not just compatible ones. - -If you import an archive file (something with the extension `.zip`, `.tar.gz`, `.tgz`, `.tar.bz2`, `.gz`, or `.bz2`), OpenRefine detects the files inside it, shows you a preview screen, and allows you to select which ones to load. This does not work with `.rar` files. When importing multiple archives you can store the name of the archive each file was extracted from by ticking the `Store archive file` option upon import. - -### Web addresses (URLs) {#web-addresses-urls} - -Type or paste the URL to a data file into the field provided. You can add as many fields as you want. OpenRefine will download the file and preview the project for you. - -If you supply two or more file URLs, OpenRefine will identify each one and ask you to choose which (or all) to load. - -Do not use this form to load a Google Sheet by its link; use [the Google Data form instead](#google-data). - -### Clipboard {#clipboard} - -You can copy and paste in data from anywhere. OpenRefine will recognize comma-separated, tab-separated, or table-formatted information copied from sources such as word-processing documents, spreadsheets, and tables in PDFs. You can also just paste in a list of items that you want to turn into rows. OpenRefine recognizes each new text line as a row. - -This can be useful if you want to pre-select a specific number of rows from your source data, or paste together rows from different places, rather than delete unwanted rows later in the project interace. - -This can also be useful if you would like to paste in a list of URLs, which you can use later to [fetch more data](columnediting). - -### Database (SQL) {#database-sql} - -If you are an administrator or have SQL access to a database of information, you may want to pull the latest dataset directly from there. This could include an online catalogue, a content management system, or a digital repository or collection management system. You can also load a database (`.db`) file saved locally. You will need to use an [SQL query](https://www.w3schools.com/sql/) to import your intended data. - -There are some publicly-accessible databases you can query, such as [one provided by Rfam](https://docs.rfam.org/en/latest/database.html). The instructions provided by Rfam can help you understand how to connect to and query from other databases. - -OpenRefine can connect to PostgreSQL, MySQL, MariaDB, and SQLite database systems. It will automatically populate the Port field based on which of these you choose, but you can manually edit this if needed. - -If you have a `.db` file, you can supply the path to the file on your computer in the Database field at the bottom of the form. You can leave the rest of the fields blank. - -To import data directly from a database, you will need the database type (such as MySQL), database name, the hostname (either an IP address or the domain that hosts the database), and the port on the host. You will need an account authorized for access, and you may need to add OpenRefine's IP address or host to the "allowable hosts" for that account. You can find that information by pressing Test and getting the IP address from the error message that results. - -You can either connect just once to gather data, or save the connection to use it again later. If you press Connect without saving, OpenRefine will forget all the information you just entered. If you’d like to save the connection, name your connection in a way you will recognize later. Click Save and it will appear in the Saved Connections list on the left. From now on, you can click on the ... ellipsis to the right of the connection you’ve saved, and click Connect. - -If your connection is successful, you will see a Query Editor where you can run your SQL query. OpenRefine will give you an error if you write a statement that tries to modify the source database in any way. - -### Google data {#google-data} - -You have two ways to load in data from Google Sheets: -* providing a link to an accessible Google Sheet (that is, one with link-sharing turned on), and -* selecting a Google Sheet in your Google Drive. - -#### Google Sheet by URL {#google-sheet-by-url} - -You can import data from any Google Sheet that has link-sharing turned on. Paste in a URL that looks something like - -``` -https://docs.google.com/spreadsheets/………/edit?usp=sharing -``` - -This will only work with Sheets, not with any other Google Drive file that might have an available link, including `.xls` and other valid files that are hosted in Google Drive. These links will not work when attempting to start a project [by URL](#web-addresses-urls) either, so you need to download those files to your computer. - -#### Google Sheet from Drive {#google-sheet-from-drive} - -You can authorize OpenRefine to access your Google Drive data and import data from any Google Sheet it finds there. This will include Sheets that belong to you and Sheets that are shared with you, as well as Sheets that are in your trash. - -When you select a Google option (either here, or [when exporting project data to Google Drive or Google Sheets](exporting), you will see a pop-up window that asks you to select a Google account to authorize with. You may see an error message when you authorize: if so, try your import or export operation again and it should succeed. - -OpenRefine will not show spreadsheets that are in your email inbox or stored in any other Google property - only in Drive. It also won’t show all compatible file formats, only Sheets files. - -OpenRefine will generate a list of all Sheets it finds, with the most recently modified Sheets at the top. If a file you’ve just added isn’t showing in this list, you can close and restart OpenRefine, or simply navigate to an existing project, open it, then head back to the Create Project window and check again. - -When you click Preview the Sheet will open in a new browser tab. When you click the Sheet title, OpenRefine will begin to process the data. - - -## Project preview {#project-preview} - -Once OpenRefine is ready to import the data, you will see a screen with Configure Parsing Options at the top. You’ll see a preview of the first 100 rows and all identified columns. - -At the bottom of the screen you will find options for telling OpenRefine how to process what it has found. You can tell it which row(s) to parse as column headers, as well as to ignore any number of rows at the top. You can also select a specific range of rows to work with, by discarding some rows at the top (excluding the header) and limiting the total number of rows it loads. - -OpenRefine tries to guess how to parse your data based on the file extension. For example, `.xml` files are going to be parsed as though they are formatted in XML. An unknown file extension (or your clipboard copy-paste) is assumed to be either tab-separated or comma-separated. OpenRefine looks for a tab character, and if one is found, it assumes you have imported tab-separated data. - -If OpenRefine isn’t certain what format you imported, it will provide a list of possibilities under Parse data as and some settings. You can specify a custom separator now, or split columns later while [transforming your data](transforming). - -If you imported a spreadsheet with multiple worksheets, they will be listed along with the number of rows they contain. You can only select data from one worksheet. - -Note that OpenRefine does not preserve any formatting, such as cell or text colour, that my have been in the original data file. Hyperlinked text will be input as plain text, but OpenRefine will recognize links and make them clickable inside the project interface. - -:::info Encoding issues? -Look for character encoding issues at this stage. You may want to manually select an encoding, such as UTF-8, UTF-16, or ASCII, if OpenRefine does not display some characters correctly in the preview. Once your project is created, you can specify another encoding for specific columns using the [reinterpret() function](grelfunctions#reinterprets-s-encoder). -::: - -You should create a project name at this stage. You can also supply tags to keep your projects organized. When you’re happy with the preview, click Create Project. - - -## Import a project {#import-a-project} - -Because OpenRefine only runs locally on your computer, you can’t have a project accessible to more than one person at the same time. - -The best way to collaborate with another person is to export and import projects that save all your changes, so that you can pick up where someone else left off. You can also [export projects](exporting#export-a-project) and import them to other computers, such as for working on the same project from the office and from home. - -An exported project will include all of the [history](running#history-undoredo), so you can see (and undo) all the changes from the previous user. It is essentially a point-in-time snapshot of their work. OpenRefine only exports projects as `.tar.gz` files at this time. -:::caution -If you wish to hide the original state of your data and your history of edits (for example, if you are using OpenRefine to anonymize information), export your cleaned dataset only and do not share your project archive. -::: - -Once someone has sent you a project archive file from their computer, you can save it anywhere. OpenRefine will import it like a new project and save its information to your workspace directory. - -In the left-hand menu of the home screen, click Import Project. Click Browse… and navigate to wherever you saved the file you were sent (for example, your Downloads folder). - -You can rename the project if you’d like - we recommend adding your name, a date, or a version number, if you’re planning to continue collaborating with another person (or working from multiple computers). - -Then, click Import Project. Your project should appear with a step count beside Undo/Redo if steps were saved by the exporter. - -OpenRefine will store the project in its own workspace directory, so you can now delete the original file that was sent to you. - - -## Project management {#project-management} - -You can access all of your created projects by clicking on Open Project. Your project list can be organized by modification date, title, row count, and other metadata you can supply (such as subject, descripton, tags, or creator). To edit the fields you see here, click About to the left of each project. There you can edit a number of available fields. You can also see the project ID that corresponds to the name of the folder in your work directory. - -### Naming projects {#naming-projects} - -You may have multiple projects from the same dataset, or multiple versions from sharing a project with another person. OpenRefine automatically generates a project name from the imported file, or “clipboard” when you use Clipboard importing. Project names don’t have to be unique, and OpenRefine will create many projects with the same name unless you intervene. - -You can edit a project's name when you create it or import it, and you can rename a project later by opening it and clicking on the project name at the top of the screen. - -### Autosaving {#autosaving} - -OpenRefine [saves all of your actions](running#history-undoredo) (everything you can see in the Undo/Redo panel). That includes flagging and starring rows. - -It doesn’t, however, save your facets, filters, or any kind of view you may have in place while you work. This includes the number of rows showing, and any sorting or column collapsing you may have done. A good rule of thumb is: if it’s not showing in Undo/Redo, you will lose it when you leave the project workspace. - -Autosaving happens by default every five minutes. You can [change this preference by following these directions](running#jvm-preferences). - -You can only save and share facets and filters, not any other type of view. To save current facets and filters, click Permalink. The project will reload with a different URL, which you can then copy and save elsewhere. This permalink will save both the facets and filters you’ve set, and the settings for each one (such as sorting by count rather than by name). - -### Deleting projects {#deleting-projects} - -You can delete projects, which will erase the project files from the workspace directory on your computer. This is immediate and cannot be undone. - -Go to Open Project and find the project you want to delete. Click on the X to the left of the project name. There will be a confirmation dialog. - -### Project files {#project-files} - -You can find all of your raw project files in your work directory. They will be named according to the unique “Project ID” that OpenRefine has assigned them, which you can find on the Open Project screen, under the “About” link for each project. diff --git a/OpenRefine/docs/docs/manual/transforming.md b/OpenRefine/docs/docs/manual/transforming.md deleted file mode 100644 index 53a436370..000000000 --- a/OpenRefine/docs/docs/manual/transforming.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -id: transforming -title: Transforming data -sidebar_label: Overview ---- - -## Overview {#overview} - -OpenRefine gives you powerful ways to clean, correct, codify, and extend your data. Without ever needing to type inside a single cell, you can automatically fix typos, convert things to the right format, and add structured categories from trusted sources. - -This section of ways to improve data are organized by their appearance in the menu options in OpenRefine. You can: - -* change the order of [rows](#edit-rows) or [columns](columnediting#rename-remove-and-move) -* edit [cell contents](cellediting) within a particular column -* [transform](transposing) rows into columns, and columns into rows -* [split or join columns](columnediting#split-or-join) -* [add new columns](columnediting) based on existing data, with fetching new information, or through [reconciliation](reconciling) -* convert your rows of data into [multi-row records](exploring#rows-vs-records). - -## Edit rows {#edit-rows} - -Moving rows around is a permanent change to your data. - -You can [sort your data](sortview#sort) based on the values in one column, but that change is a temporary view setting. With that setting applied, you can make that new order permanent. - -![A screenshot of where to find the Sort menu with a sorting applied.](/img/sortPermanent.png) - -In the project grid header, the word “Sort” will appear when a sort operation is applied. Click on it to show the dropdown menu, and select Reorder rows permanently. You will see the numbering of the rows change under the All column. - -:::info Reordering all rows -Reordering rows permanently will affect all rows in the dataset, not just those currently viewed through [facets and filters](facets). -::: - -You can undo this action using the [History tab](running#history-undoredo). \ No newline at end of file diff --git a/OpenRefine/docs/docs/manual/transposing.md b/OpenRefine/docs/docs/manual/transposing.md deleted file mode 100644 index 6b5ec0dc8..000000000 --- a/OpenRefine/docs/docs/manual/transposing.md +++ /dev/null @@ -1,234 +0,0 @@ ---- -id: transposing -title: Transposing -sidebar_label: Transposing ---- - -## Overview {#overview} - -These functions were created to solve common problems with reshaping your data: pivoting cells from a row into a column, or pivoting cells from a column into a row. You can also transpose from a repeated set of values into multiple columns. - -## Transpose cells across columns into rows {#transpose-cells-across-columns-into-rows} - -Imagine personal data with addresses in this format: - -|Name|Street|City|State/Province|Country|Postal code| -|---|---|---|---|---|---| -|Jacques Cousteau|23, quai de Conti|Paris||France|75270| -|Emmy Noether|010 N Merion Avenue|Bryn Mawr|Pennsylvania|USA|19010| - -You can transpose the address information from this format into multiple rows. Go to the “Street” column and select TransposeTranspose cells across columns into rows. From there you can select all of the five columns, starting with “Street” and ending with “Postal code,” that correspond to address information. Once you begin, you should put your project into [records mode](exploring#rows-vs-records) to associate the subsequent rows with “Name” as the key column. - -![A screenshot of the transpose across columns window.](/img/transpose1.png) - -### One column {#one-column} - -You can transpose the multiple address columns into a series of rows: - -|Name|Address| -|---|---| -|Jacques Cousteau|23, quai de Conti| -| |Paris| -| |France| -| |75270| -|Emmy Noether|010 N Merion Avenue| -||Bryn Mawr| -||Pennsylvania| -||USA| -||19010| - -You can choose one column and include the column-name information in each cell by prepending it to the value, with or without a separator: - -|Name|Address| -|---|---| -|Jacques Cousteau|Street: 23, quai de Conti| -| |City: Paris| -| |Country: France| -| |Postal code: 75270| -|Emmy Noether|Street: 010 N Merion Avenue| -||City: Bryn Mawr| -||State/Province: Pennsylvania| -||Country: USA| -||Postal code: 19010| - -### Two columns {#two-columns} - -You can retain the column names as separate cell values, by selecting Two new columns and naming the key and value columns. - -|Name|Address part|Address| -|---|---|---| -|Jacques Cousteau|Street|23, quai de Conti| -| |City|Paris| -| |Country|France| -| |Postal code|75270| -|Emmy Noether|Street|010 N Merion Avenue| -||City|Bryn Mawr| -||State/Province|Pennsylvania| -||Country|USA| -||Postal code|19010| - -## Transpose cells in rows into columns {#transpose-cells-in-rows-into-columns} - -Imagine employee data in this format: - -|Column| -|---| -|Employee: Karen Chiu| -|Job title: Senior analyst| -|Office: New York| -|Employee: Joe Khoury| -|Job title: Junior analyst| -|Office: Beirut| -|Employee: Samantha Martinez| -|Job title: CTO| -|Office: Tokyo| - -The goal is to sort out all of the information contained in one column into separate columns, but keep it organized by the person it represents: - -|Name |Job title |Office| -|---|---|---| -|Karen Chiu |Senior analyst |New York| -|Joe Khoury |Junior analyst |Beirut| -|Samantha Martinez |CTO |Tokyo| - -By selecting TransposeTranspose cells in rows into columns... a window will appear that simply asks how many rows to transpose. In this case, each employee record has three rows, so input “3” (do not subtract one for the original column). The original column will disappear and be replaced with three columns, with the name of the original column plus a number appended. - -|Column 1 |Column 2 |Column 3| -|---|---|---| -|Employee: Karen Chiu |Job title: Senior analyst |Office: New York| -|Employee: Joe Khoury |Job title: Junior analyst |Office: Beirut| -|Employee: Samantha Martinez |Job title: CTO |Office: Tokyo| - -From here you can use Cell editingReplace to remove “Employee: ”, “Job title: ”, and “Office: ” if you wish, or use [expressions](expressions) with Edit cellsTransform... to clean out the extraneous characters: - -``` -value.replace("Employee: ", "") -``` - -If your dataset doesn't have a predictable number of cells per intended row, such that you cannot specify easily how many columns to create, try Columnize by key/value columns. - -## Columnize by key/value columns {#columnize-by-keyvalue-columns} - -This operation can be used to reshape a dataset that contains key and value columns: the repeating strings in the key column become new column names, and the contents of the value column are moved to new columns. This operation can be found at TransposeColumnize by key/value columns. - -![A screenshot of the Columnize window.](/img/transpose2.png) - -Consider the following example, with flowers, their colours, and their International Union for Conservation of Nature (IUCN) identifiers: - -|Field |Data | -|--------|----------------------| -|Name |Galanthus nivalis | -|Color |White | -|IUCN ID |162168 | -|Name |Narcissus cyclamineus | -|Color |Yellow | -|IUCN ID |161899 | - -In this format, each flower species is described by multiple attributes on consecutive rows. The “Field” column contains the keys and the “Data” column contains the values. In the Columnize by key/value columns window you can select each of these from the available columns. It transforms the table as follows: - -| Name | Color | IUCN ID | -|-----------------------|----------|---------| -| Galanthus nivalis | White | 162168 | -| Narcissus cyclamineus | Yellow | 161899 | - -### Entries with multiple values in the same column {#entries-with-multiple-values-in-the-same-column} - -If a new row would have multiple values for a given key, then these values will be grouped on consecutive rows, to form a [record structure](exploring#rows-vs-records). - -For instance, flowers can have multiple colors: - -| Field | Data | -|-------------|-----------------------| -| Name | Galanthus nivalis | -| _Color_ | _White_ | -| _Color_ | _Green_ | -| IUCN ID | 162168 | -| Name | Narcissus cyclamineus | -| Color | Yellow | -| IUCN ID | 161899 | - -This table is transformed by the Columnize operation to: - -| Name | Color | IUCN ID | -|-----------------------|----------|---------| -| Galanthus nivalis | White | 162168 | -| | Green | | -| Narcissus cyclamineus | Yellow | 161899 | - -The first key encountered by the operation serves as the record key, so the “Green” value is attached to the “Galanthus nivalis” name. See the [Row order](#row-order) section for more details about the influence of row order on the results of the operation. - -### Notes column {#notes-column} - -In addition to the key and value columns, you can optionally add a column for notes. This can be used to store extra metadata associated to a key/value pair. - -Consider the following example: - -| Field | Data | Source | -|---------|---------------------|-----------------------| -| Name | Galanthus nivalis | IUCN | -| Color | White | Contributed by Martha | -| IUCN ID | 162168 | | -| Name | Narcissus cyclamineus | Legacy | -| Color | Yellow | 2009 survey | -| IUCN ID | 161899 | | - -If the “Source” column is selected as the notes column, this table is transformed to: - -| Name | Color | IUCN ID | Source: Name | Source: Color | -|-----------------------|----------|---------|---------------|-----------------------| -| Galanthus nivalis | White | 162168 | IUCN | Contributed by Martha | -| Narcissus cyclamineus | Yellow | 161899 | Legacy | 2009 survey | - -Notes columns can therefore be used to preserve provenance or other context about a particular key/value pair. - -### Row order {#row-order} - -The order in which the key/value pairs appear matters. The Columnize operation will use the first key it encounters as the delimiter for entries: every time it encounters this key again, it will produce a new row, and add the following key/value pairs to that row. - -Consider for instance the following table: - -| Field | Data | -|----------|-----------------------| -| _Name_ | Galanthus nivalis | -| Color | White | -| IUCN ID | 162168 | -| _Name_ | Crinum variabile | -| _Name_ | Narcissus cyclamineus | -| Color | Yellow | -| IUCN ID | 161899 | - -The occurrences of the “Name” value in the “Field” column define the boundaries of the entries. Because there is no other row between the “Crinum variabile” and the “Narcissus cyclamineus” rows, the “Color” and “IUCN ID” columns for the “Crinum variabile” entry will be empty: - -| Name | Color | IUCN ID | -|-----------------------|----------|---------| -| Galanthus nivalis | White | 162168 | -| Crinum variabile | | | -| Narcissus cyclamineus | Yellow | 161899 | - -This sensitivity to order is removed if there are extra columns: in that case, the first extra column will serve as the key for the new rows. - -### Extra columns {#extra-columns} - -If your dataset contains extra columns, that are not being used as the key, value, or notes columns, they can be preserved by the operation. For this to work, they must have the same value in all old rows corresponding to a new row. - -In the following example, the “Field” and “Data” columns are used as key and value columns respectively, and the “Wikidata ID” column is not selected: - -| Field | Data | Wikidata ID | -|---------|-----------------------|-------------| -| Name | Galanthus nivalis | Q109995 | -| Color | White | Q109995 | -| IUCN ID | 162168 | Q109995 | -| Name | Narcissus cyclamineus | Q1727024 | -| Color | Yellow | Q1727024 | -| IUCN ID | 161899 | Q1727024 | - -This will be transformed to: - -| Wikidata ID | Name | Color | IUCN ID | -|-------------|-----------------------|----------|---------| -| Q109995 | Galanthus nivalis | White | 162168 | -| Q1727024 | Narcissus cyclamineus | Yellow | 161899 | - -This actually changes the operation: OpenRefine no longer looks for the first key (“Name”) but simply pivots all information based on the first extra column's values. Every old row with the same value gets transposed into one new row. If you have more than one extra column, they are pivoted as well but not used as the new key. - -You can use [Fill down](cellediting#fill-down-and-blank-down) to put identical values in the extra columns if you need to. \ No newline at end of file diff --git a/OpenRefine/docs/docs/manual/troubleshooting.md b/OpenRefine/docs/docs/manual/troubleshooting.md deleted file mode 100644 index 5388fc351..000000000 --- a/OpenRefine/docs/docs/manual/troubleshooting.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -id: troubleshooting -title: Troubleshooting -sidebar_label: Troubleshooting ---- - -## Frequently asked questions {#frequently-asked-questions} - -We collect and share FAQs and responses on Github at [https://github.com/OpenRefine/OpenRefine/wiki/FAQ](https://github.com/OpenRefine/OpenRefine/wiki/FAQ). - -If you don’t find your problem and solution there, continue on to the resources in the Community section below to see more conversations and look for solutions. - -## Community {#community} - -### If you’re having a problem: {#if-youre-having-a-problem} -* Search the [User forum](https://groups.google.com/g/openrefine) to see if the problem is already reported -* Search [Github issues](https://github.com/OpenRefine/OpenRefine/issues) to see if the problem is already reported -* Read [Stack Overflow](https://stackoverflow.com/questions/tagged/openrefine) to see if others had a similar problem -* Check [Twitter](https://twitter.com/search?f=tweets&vertical=default&q=OpenRefine%20OR%20%22Open%20Refine%22%20OR%20%23OpenRefine&src=typd) to see if others are discussing the problem -* Report an issue: - * First as a new thread (conversation) in the [User forum](https://groups.google.com/g/openrefine). - * Then, if you wish, you can create a Github issue. - -### If you want to contribute: {#if-you-want-to-contribute} -* [Help us translate the tool into more languages](../technical-reference/translating-ui), using Weblate -* [We have a guide to contributing](../technical-reference/contributing) in the [Technical Reference](../technical-reference/technical-reference-index) section -* Contribute your feature requests in the [User forum](https://groups.google.com/g/openrefine) or as [Github issues](https://github.com/OpenRefine/OpenRefine/issues/new/choose) -* Join the User Forum and/or the [Developer Forum](https://groups.google.com/g/openrefine-dev) -* Share your successes and use cases with us, in the User forum -* Add your [blog posts, guides, tips, tricks, tutorials to our list](https://github.com/OpenRefine/OpenRefine/wiki/External-Resources) -* Keep an eye out for and respond to our biennial user survey. diff --git a/OpenRefine/docs/docs/manual/wikibase/advanced-schemas.md b/OpenRefine/docs/docs/manual/wikibase/advanced-schemas.md deleted file mode 100644 index 45f91800c..000000000 --- a/OpenRefine/docs/docs/manual/wikibase/advanced-schemas.md +++ /dev/null @@ -1,72 +0,0 @@ -Sometimes your data is not as simple as a normal table, or the sort of -statements that you want to do varies on each row. This document -explains how to work around these cases. - -## Hierarchical data {#hierarchical-data} - -Sometimes your source provides data in a structured format, such as XML, -JSON or RDF. OpenRefine can import these files and will convert them to -tables. These tables will reflect some of the hierarchy in the file by -means of null cells, using the [records mode](/manual/exploring#rows-vs-records). - -The Wikibase extension always works in rows mode, so if we want to add -statements which reference both the artist and the song, we need to fill -the null cells with the corresponding artist. You can do this with the -**Fill down** operation (in the **Edit cells** menu for this column). -This function will copy not just cell values but also reconciliation -results. - -## Conditional additions {#conditional-additions} - -Sometimes you want to add a statement only in some conditions. - -The workflow to achieve this looks like this: -- Use facets to select the rows where you do not want to add any - information; -- Blank out the cells in the column that contain the information you - want to add. If you do not want to lose this information, you can - create a copy of the column beforehand; -- Remove your facets to see all rows again; -- Create a schema using the column you partially blanked out as - statement value. - -## Varying properties {#varying-properties} - -Sometimes you wish you could use column variables for properties in your -schema. It is currently not possible, first because we do not have a -reconciliation service for properties yet, but also because allowing -varying properties in a statement would mean that these properties could -potentially have different datatypes, which would break the structure of -the schema. - -If you only want to use a few properties, there is a way to go around -this problem. For instance, say you have a first column of altitudes and a -second column that indicates whether you should add it as -[operating altitude (P2254)](https://www.wikidata.org/wiki/Property:P2254) or as -[elevation above sea level (P2044)](https://www.wikidata.org/wiki/Property:P2044). - -Create a text facet on the first column. Filter to keep only the -*altitude* values. Add a new column based on the second column, by -keeping the default expression (`value`) which just copies the existing -values. Then, select the *maximum operating altitude* value in the facet -and do the same. Reset the facet, you should have obtained two new columns -which partition the original column. You can now create a schema which adds -two statements, with values taken from those columns. Since blank values are -ignored, exactly one statement will be added for each item, with the desired property. - -## Adapting to existing data on Wikibase {#adapting-to-existing-data-on-wikibase} - -Sometimes you want to create statements only if there are no such -statements on the item yet. Here is one way to achieve this: - -- first, retrieve the existing values from Wikidata first, using the - **Edit columns** → **Add columns from reconciled values** action; -- second, create a *facet by null* on the newly created column that - contains the information you want to control against; -- select the non-null rows (value **false**); -- clear the contents of the column where your source values are - (**Edit cells** → **Common transformations** → **To null**). - -You can now construct your schema as usual - null values will be ignored -when generating the statements. - diff --git a/OpenRefine/docs/docs/manual/wikibase/configuration.md b/OpenRefine/docs/docs/manual/wikibase/configuration.md deleted file mode 100644 index aa7fd1d98..000000000 --- a/OpenRefine/docs/docs/manual/wikibase/configuration.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -id: configuration -title: Connecting OpenRefine to a Wikibase instance -sidebar_label: Connecting to Wikibase ---- - -This page explains how to connect OpenRefine to any Wikibase instance. If you just want to work with [Wikidata](https://www.wikidata.org/), you can ignore this page as Wikidata is configured out of the box in OpenRefine. - -## For Wikibase end users {#for-wikibase-end-users} - -All you need to configure OpenRefine to work with a Wikibase instance is a *manifest* for that instance, which provides some metadata and links required for the integration to work. - -We offer some off-the-shelf manifests for some public Wikibase instances in the [wikibase-manifests](https://github.com/OpenRefine/wikibase-manifests) repository. But the administrators of your Wikibase instance should provide one that is potentially more -up to date, so it makes sense to request it to them first. - -## For Wikibase administrators {#for-wikibase-administrators} - -To let your users contribute to your Wikibase instance with OpenRefine, you will need to write a manifest as described above. There is currently no canonical location where this manifest should be hosted - just make sure can be found easily by your users. This section explains the format of the manifest. - -### Requirements {#requirements} - -To work with OpenRefine, your Wikibase instance needs an associated reconciliation service. For instance you can use [a Python wrapper](https://github.com/wetneb/openrefine-wikibase) for this. Also, in addition to Wikibase, the [UniversalLanguageSelector extension](https://www.mediawiki.org/wiki/Special:MyLanguage/Extension:UniversalLanguageSelector) should be installed. - - -### The format of the manifest {#the-format-of-the-manifest} - -Here is the manifest of Wikidata: - -```json -{ - "version": "1.0", - "mediawiki": { - "name": "Wikidata", - "root": "https://www.wikidata.org/wiki/", - "main_page": "https://www.wikidata.org/wiki/Wikidata:Main_Page", - "api": "https://www.wikidata.org/w/api.php" - }, - "wikibase": { - "site_iri": "http://www.wikidata.org/entity/", - "maxlag": 5, - "properties": { - "instance_of": "P31", - "subclass_of": "P279" - }, - "constraints": { - "property_constraint_pid": "P2302", - "exception_to_constraint_pid": "P2303", - "constraint_status_pid": "P2316", - "mandatory_constraint_qid": "Q21502408", - "suggestion_constraint_qid": "Q62026391", - "distinct_values_constraint_qid": "Q21502410", - // ... - } - }, - "oauth": { - "registration_page": "https://meta.wikimedia.org/wiki/Special:OAuthConsumerRegistration/propose" - }, - "reconciliation": { - "endpoint": "https://wikidata.reconci.link/${lang}/api" - }, - "editgroups": { - "url_schema": "([[:toollabs:editgroups/b/OR/${batch_id}|details]])" - } -} -``` - -In general, there are several parts of the manifest: version, mediawiki, wikibase, oauth, reconciliation and editgroups. - -#### version {#version} - -The version should in the format "1.x". The minor version should be increased when you update the manifest in a backward-compatible manner. The major version should be "1" if the manifest is in the format specified by [wikibase-manifest-schema-v1.json](https://github.com/afkbrb/wikibase-manifest/blob/master/wikibase-manifest-schema-v1.json). - -#### mediawiki {#mediawiki} - -This part contains some basic information of the Wikibase. - -##### name {#name} - -The name of the Wikibase, should be unique for different Wikibase instances. - -##### root {#root} - -The root of the Wikibase. Typically in the form "https://foo.bar/wiki/". The trailing slash cannot be omitted. - -##### main_page {#main_page} - -The main page of the Wikibase. Typically in the form "https://foo.bar/wiki/Main_Page". - -##### api {#api} - -The MediaWiki API endpoint of the Wikibase. Typically in the form "https://foo.bar/w/api.php". - -#### wikibase {#wikibase} - -This part contains configurations of the Wikibase extension. - -##### site_iri {#site_iri} - -The IRI of the Wikibase, in the form 'http://foo.bar/entity/'. This should match the IRI prefixes used in RDF serialization. Be careful about using "http" or "https", because any variation will break comparisons at various places. The trailing slash cannot be omitted. - -##### maxlag {#maxlag} - -Maxlag is a parameter that controls how aggressive a mass-editing tool should be when uploading edits to a Wikibase instance. See https://www.mediawiki.org/wiki/Manual:Maxlag_parameter for more details. The value should be adapted according to the actual traffic of the Wikibase. - -##### properties {#properties} - -Some special properties of the Wikibase. - -###### instance_of {#instance_of} - -The ID of the property "instance of". - -###### subclass_of {#subclass_of} - -The ID of the property "subclass of". - -##### constraints {#constraints} - -Not required. Should be configured if the Wikibase has the [WikibaseQualityConstraints extension](https://www.mediawiki.org/wiki/Extension:WikibaseQualityConstraints) installed. Configurations of constraints consists of IDs of constraints related properties and items. For Wikidata, these IDs are retrieved from [extension.json](https://github.com/wikimedia/mediawiki-extensions-WikibaseQualityConstraints/blob/master/extension.json). To configure this for another Wikibase instance, you should contact an admin of the Wikibase instance to get the content of `extension.json`. - -#### oauth {#oauth} - -Not required. Should be configured if the Wikibase has the [OAuth extension](https://www.mediawiki.org/wiki/Extension:OAuth) installed. - -##### registration_page {#registration_page} - -The page to register an OAuth consumer of the Wikibase. Typically in the form "https://foo.bar/wiki/Special:OAuthConsumerRegistration/propose". - -#### reconciliation {#reconciliation} - -The Wikibase instance must have at least a reconciliation service endpoint linked to it. If there is no reconciliation service for the Wikibase, you can run one with [openrefine-wikibase](https://github.com/wetneb/openrefine-wikibase). - -##### endpoint {#endpoint} - -The default reconciliation service endpoint of the Wikibase instance. The endpoint must contain the "${lang}" variable such as "https://wikidata.reconci.link/${lang}/api", since the reconciliation service is expected to work for different languages. - -#### editgroups {#editgroups} - -Not required. Should be configured if the Wikibase instance has [EditGroups](https://github.com/Wikidata/editgroups) service(s). - -##### url_schema {#url_schema} - -The URL schema used in edits summary. This is used for EditGroups to extract the batch id from a batch of edits and for linking to the EditGroups page of the batch. The URL schema must contains the variable '${batch_id}', such as '([[:toollabs:editgroups/b/OR/${batch_id}|details]])' for Wikidata. - -#### Check the format of the manifest {#check-the-format-of-the-manifest} - -As mentioned above, the manifest should be in the format specified by [wikibase-manifest-schema-v1.json](https://github.com/afkbrb/wikibase-manifest/blob/master/wikibase-manifest-schema-v1.json). You can check the format by adding the manifest directly to OpenRefine, and OpenRefine will complain if there is anything wrong with the format. - -![test-validate-manifest-format](https://user-images.githubusercontent.com/29347603/90506110-52d85d00-e186-11ea-8077-683d2f234c46.gif) diff --git a/OpenRefine/docs/docs/manual/wikibase/new-entities.md b/OpenRefine/docs/docs/manual/wikibase/new-entities.md deleted file mode 100644 index 8dbc4f362..000000000 --- a/OpenRefine/docs/docs/manual/wikibase/new-entities.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -id: new-entities -title: Creating new items -sidebar_label: New items ---- - -OpenRefine can create new items. This page explains how they are -generated. - -## Words of caution {#words-of-caution} - -- The fact that OpenRefine does not propose any item when reconciling - a cell does not mean that the item is not present in the Wikibase instance: - it can be missed for all sorts of reasons. Please make - sure that you are not creating any duplicates! - -- Make sure that the items that you want to create are admissible in - the Wikibase instance. For Wikidata, see the [notability guidelines](https://www.wikidata.org/wiki/Wikidata:Notability); - -- Deleting items generally requires special rights: if you want to revert an - edit group that includes new items in Wikidata, you will need to ask an - administrator to do it. - -## Workflow overview {#workflow-overview} - -Here is how you would typically create new items with OpenRefine: - -- Reconcile a column; -- Mark some of its cells as new items. This will not create items yet. - If you need to mark many rows as new items, use the **Reconcile** → - **Actions** → **Create a new item for each cell** operation. -- Create a Wikibase schema as usual, using the column where your new - items are marked; -- Perform the edits: the new items will be created on Wikidata at this - point; -- The cells that you had marked as new items will now be reconciled to - the newly-created items. - -It is often useful (but not mandatory) to treat new items in isolation -and use a dedicated schema for them. This helps you add many statements -on the new items (including labels and descriptions) without risking to -clutter existing items with redundant edits. Use a facet on the judgment -status of the reconciled column to isolate new items and perform their -edits separately. As always in OpenRefine, only the rows covered by your -facets will be considered when uploading the edits to Wikidata: if a -cell is reconciled to a new item but is excluded by the facet, no new -item will be created for it.[^1] - -Note that even if you know that all items in your column are new, you -will still need to make a first reconciliation pass by selecting the -Wikidata reconciliation service, and then setting all reconciliation -statuses to \"new\". If you skip the first part, OpenRefine will not -know that this column is reconciled against your Wikibase instance (it could be -reconciled to other services) so it will not let you use it in place of -an item in a Wikibase schema. - -You can also perform the edits with QuickStatements - in this case, your -OpenRefine project will not be updated with the newly created Qids. - -## Adding labels to new items {#adding-labels-to-new-items} - -The text that is in a cell reconciled to \"new\" is not automatically -used as label for the newly-created item. This is because OpenRefine has -no way to guess in which language this label should be. When adding new -items, you need to explicitly add a label in the schema. This label can -use the reconciled column as source, but if you have other cells matched -to existing items, be careful not to override the labels of these items -(if it is not your intention). - -OpenRefine will refuse to perform edits where new items are created -without any labels (as this is considered a critical issue). Other -issues will be raised if insufficient basic information is added on the -items (but these other warnings will not prevent you from performing the -edits). - -## Marking multiple cells as identical items {#marking-multiple-cells-as-identical-items} - -If you mark individual cells as new items, one new item per cell will be -created. Sometimes multiple rows refer to the same item. OpenRefine -makes it possible to mark all the corresponding cells as the *same* new -item. Two conditions have to be met: -- the reconciled cells must be in the same column (it is not possible - to mark two cells in different colums as the same new item); -- the cells must contain the same initial text value. - -If these two conditions are met, then isolate these cells with facets -and go to **Reconcile** → **Actions** → **Create one item for similar -cells**. This will mark the cells as new and referring to the same item. - -## Retrieving the Qids of the newly-created items {#retrieving-the-qids-of-the-newly-created-items} - -Once you have performed your edits with OpenRefine, any new cells -covered by the facet will be updated with their new Qids. You can -retrieve these Qids with the **Edit column** → **Add column based on -this column** action and using the `cell.recon.match.id` expression. -Note that you will no longer be able to isolate new items with a -judgment facet at this stage (because the judgment will be updated to -**matched**) so it can be worth marking these rows (for instance with a -star or flag) before performing the edits. - -[^1]: The only exception to this rule is when marking multiple cells as - identical items: in this case, if one of such cells are included in - the facet, then all the others will be updated with the newly - created Qid once the edits are made. diff --git a/OpenRefine/docs/docs/manual/wikibase/overview.md b/OpenRefine/docs/docs/manual/wikibase/overview.md deleted file mode 100644 index a202101fd..000000000 --- a/OpenRefine/docs/docs/manual/wikibase/overview.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -id: overview -title: Overview of Wikibase support -sidebar_label: Overview ---- - -[Wikibase](https://wikiba.se/) is a platform for collaborative knowledge base editing. Its flagship instance [Wikidata](https://www.wikidata.org/) offers structured data about the world and can be edited by anyone. OpenRefine provides powerful ways to both pull data from Wikibase and add data to it. - -OpenRefine's Wikibase integration is provided by an extension which is available by default in OpenRefine. In this page, we present the functionalities for Wikidata, but [any Wikibase instance can be connected to OpenRefine](./configuration) to obtain a similar integration. - -## Editing Wikidata with OpenRefine {#editing-wikidata-with-openrefine} - -As a user-maintained data source, Wikidata can be edited by anyone. OpenRefine makes it simple to upload information in bulk. You simply need to get your information into the correct format, and ensure that it is new (not redundant to information already on Wikidata) and does not conflict with existing Wikidata information. - -You do not need a Wikidata account to reconcile your local OpenRefine project to Wikidata, but to upload your cleaned dataset to Wikidata, you will need an [autoconfirmed](https://www.wikidata.org/wiki/Wikidata:Autoconfirmed_users) account, and you must [authorize OpenRefine with that account](#manage-wikidata-account). - -Wikidata is built by creating entities (such as people, organizations, or places, identified with unique numbers starting with Q), defining properties (unique numbers starting with P), and using properties to define relationships between entities (a Q has a property P, with a value of another Q). - -For example, you may wish to create entities for local authors and the books they've set in your community. Each writer will be an entity with the occupation [author (Q482980)](https://www.wikidata.org/wiki/Q482980), each book will be an entity with the property “instance of” ([P31](https://www.wikidata.org/wiki/Property:P31)) linking it to a class such as [literary work (Q7725634)](https://www.wikidata.org/wiki/Q7725634), and books will be related to authors through a property [author (P50)](https://www.wikidata.org/wiki/Property:P50). Books can have places where they are set, with the property [narrative location (P840)](https://www.wikidata.org/wiki/Property:P840). - -To do this with OpenRefine, you'll need a column of publication titles that you have reconciled (and create new items where needed); each publication will have one or more locations in a “setting” column, which is also reconciled to municipalities or regions where they exist (and create new items where needed). Then you can add those new relationships, and create new entities for authors, books, and places where needed. You do not need columns for properties; those are defined later, in the creation of your [schema](#edit-wikidata-schema). - -There is a list of [tutorials and walkthroughs on Wikidata](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing) that will allow you to see the full process. You can save your schemas and drafts in OpenRefine, and your progress stays in draft until you are ready to upload it to Wikidata. - -Batches of edits to Wikidata that are created with OpenRefine can be undone. You can test out the uploading process by reconciling to several “sandbox” entities created specifically for drafting edits and learning about Wikidata: -* https://www.wikidata.org/wiki/Q4115189 -* https://www.wikidata.org/wiki/Q13406268 -* https://www.wikidata.org/wiki/Q15397819 -* https://www.wikidata.org/wiki/Q64768399 - -If you upload edits that are redundant (that is, all the statements you want to make have already been made), nothing will happen. If you upload edits that conflict with existing information (such as a different birthdate than one already in Wikidata), it will be added as a second statement. OpenRefine produces no warnings as to whether your data replicates or conflicts with existing Wikidata elements. - -You can use OpenRefine's reconciliation preview to look at the target Wikidata elements and see what information they already have, and whether the elements' histories have had similar edits reverted in the past. - -### Wikidata schema {#wikidata-schema} - -A [schema](https://en.wikipedia.org/wiki/Database_schema) is a plan for how to structure information in a database. In OpenRefine, the schema operates as a template for how Wikidata edits should be applied: how to translate your tabular data into statements. With a schema, you can: -* preview the Wikidata edits and inspect them manually; -* analyze and fix any issues highlighted by OpenRefine; -* upload your changes to Wikidata by logging in with your own account; -* export the changes to the QuickStatements v1 format. - -For example, if your dataset has columns for authors, publication titles, and publication years, your schema can be conceptualized as: [publication title] has the author [author], and was published in [publication year]. To establish these facts, you need to establish one or more columns as “items,” for which you will make “statements” that relate them to other columns. - -You can export any schema you create, and import an existing schema for use with a new dataset. This can help you work in batches on a large amount of data while minimizing redundant labor. - -Once you select Edit Wikidata schema under the Extensions dropdown menu, your project interface will change. You’ll see new tabs added to the right of “X rows/records" in the grid header: “Schema,” “Issues,” and “Preview.” You can now switch between the tabular grid format of your dataset and the screens that allow you to prepare data for uploading. - -OpenRefine presents you with an easy visual way to map out the relationships in your dataset. Each of the columns of your project will appear at the top of the sceren, and you can simply drag and drop them into the appropriate slots. To get start, select one column as an item. - -![A screenshot of the schema construction window in OpenRefine.](/img/wikidata-schema.png) - -You may wish to refer to [this Wikidata tutorial on how OpenRefine handles Wikidata schema](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Tutorials/Basic_editing). For details about how each data type is handled in the Wikibase schema, see [Schema alignment](./schema-alignment). - -#### Editing terms with your schema {#editing-terms-with-your-schema} - -With OpenRefine, you can edit the terms (labels, aliases, descriptions, or sitelinks) of Wikidata entities as well as establish relationships between entities. For example, you may wish to upload pseudonyms, pen names, maiden names, or married names for authors. - -![An author with a number of aliases indicating pseudonyms.](/img/wikidata-terms.png) - -You can do so by putting the preferred names in one column of your dataset and alternative names in another column. In the schema interface, add an item for the preferred values, then click “Add term” on the right-hand side of the screen. Select “Alias” from the dropdown, enter in “English” in the language field, and drop your alternative names column into the space. For this example, you should also consider adding those alternative names to the authors' entries using the property [pseudonym (P742)](https://www.wikidata.org/wiki/Property:P742). The "description" and "label" terms can only contain one value, so there is an option to override existing values if needed. Aliases can be potentially infinite. - -![The schema window showing a term being edited.](/img/wikidata-terms2.png) - -Terms must always have an associated language. You can select the term's language by typing in the “lang” field, which will auto-complete for you. You cannot edit multiple languages at once, unless you supply a suitable column instead. For example, suppose you had translated publication titles, with data in the following format: - -|English title|Translated title|Translation language| -|---|---|---| -|Possession|Besessen|German| -||Обладать|Russian| -|Disgrace|Disgrâce|French| -||Vergogna|Italian| -|Wolf Hall|En la corte del lobo|Spanish| -||ウルフ・ホール|Japanese| - -You could upload the “Translated titles” to “Label” with the language specified by “Translation language.” You may wish to fetch the two-letter language code and use that instead for better language matches. - -![Constructing a schema with aliases and languages.](/img/wikidata-translated.png) - -### Manage Wikidata account {#manage-wikidata-account} - -To edit Wikidata directly from OpenRefine, you must log in with a Wikidata account. OpenRefine can only upload edits with Wikidata user accounts that are “[autoconfirmed](https://www.wikidata.org/wiki/Wikidata:Autoconfirmed_users)” - at this time, that means accounts that have more than 50 edits and have existed for longer than four days. - -Use the Extensions menu to select Manage Wikidata account and you will be presented with the following window: - -![The Wikidata authorization window in OpenRefine.](/img/wikidata-login.png) - -For security reasons, you should not use your main account authorization with OpenRefine. Wikidata allows you to set special passwords to access your account through software. You can find [this setting for your account here](https://www.wikidata.org/wiki/Special:BotPasswords) once logged in. Creating bot access will prompt you for a unique name. You should then enable the following required settings: -* High-volume editing -* Edit existing pages -* Create, edit, and move pages - -It will then generate a username (in the form of “yourwikidatausername@yourbotname”) and password for you to use with OpenRefine. - -If your account or your bot is not properly authorized, OpenRefine will not display a warning or error when you try to upload your edits. - -You can store your unencrypted username and password in OpenRefine, saved locally to your computer and available for future use. For security reasons, you may wish to leave this box unchecked. You can also save your OpenRefine-specific bot password in your browser or with a password management tool. - -### Import and export schema {#import-and-export-schema} - -You can save time on repetitive processes by defining a schema on one project, then exporting it and importing for use on new datasets in the future. Or you and your colleagues can share a schema with each other to coordinate your work. - -You can export a schema from a project using ExportWikidata schema, or by using ExtensionsExport schema. OpenRefine will generate a JSON file for you to save and share. You may experience issues with pop-up windows in your browser: consider allowing pop-ups from the OpenRefine URL (`127.0.0.1`) from now on. - -You can import a schema using ExtensionsImport schema. You can upload a JSON file, or paste JSON statements directly into a field in the window. An imported schema will look for columns with the same names, and you will see an error message if your project doesn't contain matching columns. - -### Upload edits to Wikidata {#upload-edits-to-wikidata} - -There are two menu options in OpenRefine for applying your edits to Wikidata, and the details of the differences between the two can be found in the [Uploading page](./uploading). Under Export you will see Wikidata edits... and under Extensions you will see Upload edits to Wikidata. Both will bring up the same window for you to [log in with a Wikidata account](#manage-wikidata-account). - -Once you are authorized, you will see a window with any outstanding issues. You can ignore these issues, but we recommend you resolve them. - -If you are ready to upload your edits, you can provide an “Edit summary” - a short message describing the batch of edits you are making. It can be helpful to leave notes for yourself, such as “batch 1: authors A-G” or other indicators of your workflow progress. OpenRefine will show the progress of the upload as it is happening, but does not show a confirmaton window. - -If your edits have been successful, you will see them listed on [your Wikidata user contributions page](https://www.wikidata.org/wiki/Special:Contributions/), and on the [Edit groups page](https://editgroups.toolforge.org/). All edits can be undone from this second interface. - -### QuickStatements export {#quickstatements-export} - -Your OpenRefine data can be exported in a format recognized by [QuickStatements](https://www.wikidata.org/wiki/Help:QuickStatements), a tool that creates Wikidata edits using text commands. OpenRefine generates “version 1” QuickStatements commands. - -There are advantages to using QuickStatements rather than uploading your edits directly to Wikidata, including the way QuickStatements resolves duplicates and redundancies. You can learn more on QuickStatements' [Help page](https://www.wikidata.org/wiki/Help:QuickStatements), and on OpenRefine's [Uploading page](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Uploading). - -In order to use QuickStatements, you must authorize it with a Wikidata account that is [autoconfirmed](https://www.wikidata.org/wiki/Wikidata:Autoconfirmed_users) (it may appear as “MediaWiki” when you authorize). - -Follow the [steps listed on this page](https://www.wikidata.org/wiki/Help:QuickStatements#Running_QuickStatements). -To prepare your OpenRefine data into QuickStatements, select ExportQuickStatements file, or ExtensionsExport to QuickStatements. Exporting your schema from OpenRefine will generate a text file called `statements.txt` by default. Paste the contents of the text file into a new QuickStatements batch using version 1. You can find [version 1 of the tool (no longer maintained) here](https://wikidata-todo.toolforge.org/quick_statements.php). The text commands will be processed into Wikidata edits and previewed for you to review before submitting. - -### Issue detection {#issue-detection} - -This section is an overview of the [Quality assurance page](./quality-assurance). - -OpenRefine will analyze your schema and make suggestions. It does not check for conflicts in your proposed edits, or tell you about redundancies. - -One of the most common suggestions is to attach [a reference to your edits](https://www.wikidata.org/wiki/Help:Sources) - a citation for where the information can be found. This can be a book or newspaper citation, a URL to an online page, a reference to a physical source in an archival or special collection, or another source. If the source is itself an item on Wikidata, use the relationship [stated in (P248)](https://www.wikidata.org/wiki/Property:P248); otherwise, use [reference URL (P854)](https://www.wikidata.org/wiki/Property:P854) to identify an external source. diff --git a/OpenRefine/docs/docs/manual/wikibase/quality-assurance.md b/OpenRefine/docs/docs/manual/wikibase/quality-assurance.md deleted file mode 100644 index a8ead43e3..000000000 --- a/OpenRefine/docs/docs/manual/wikibase/quality-assurance.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -id: quality-assurance -title: Quality assurance for Wikibase uploads -sidebar_label: Quality assurance ---- - -This page explains how the Wikidata extension of OpenRefine analyzes edits before they are uploaded to the Wikibase -instance. Most of these checks rely on the use of the [Wikibase Quality Constraints](https://gerrit.wikimedia.org/g/mediawiki/extensions/WikibaseQualityConstraints) extension and the configuration of the property and item identifiers in the [Wikibase manifest](./configuration). - -## Overview {#overview} - -Changes are scrutinized before they are uploaded, but also before the current content of the corresponding items is retrieved and merged with the updates. This means that some constraint violations cannot be predicted by the software (for instance, adding a new statement that conflicts with an existing statement on the item). However, this makes it possible to run the checks quickly, even for relatively large batches of edits. Issues are therefore refreshed in real time while the user builds the schema. - -As a consequence, not all constraint violations can be detected: the ones that are supported are listed in the [Constraint violations](#constraint-violations) section. Conversely, not all issues reported will be flagged as constraint violations on the Wikibase site: see [Generic issues](#generic-issues) for these. - -## Reconciliation {#reconciliation} - -You should always assess the quality of your reconciliation results first. OpenRefine has various tools for quality assurance of reconciliation results. For instance: - -* you can analyze the string similarity between your original names and those of the reconciled items (for instance with ReconcileFacetsBest candidate's name edit distance); -* you can compare the values in your table with those on the items (via a text facet defined by a custom expression); -* you can facet by type on the reconciled items (add a new column with the types and use a text facet ordered by counts to get a sense of the distribution of types in your reconciled items). - -## Constraint violations {#constraint-violations} - -Constraints are retrieved as defined on the properties, using [ (P2302)](https://www.wikidata.org/wiki/Property:P2302). - -The following constraints are supported: -* [format constraint (Q21502404)](https://www.wikidata.org/wiki/Q21502404), checked on all values -* [inverse constraint (Q21510855)](https://www.wikidata.org/wiki/Q21510855): OpenRefine assumes that the inverses of the candidate statements are not in Wikidata yet. If you know that the inverse statements are already in Wikidata, you can safely ignore this issue. -* [used for values only constraint (Q21528958)](https://www.wikidata.org/wiki/Q21528958), [used as qualifier constraint (Q21510863)](https://www.wikidata.org/wiki/Q21510863) and [used as reference constraint (Q21528959)](https://www.wikidata.org/wiki/Q21528959) -* [allowed qualifiers constraint (Q21510851)](https://www.wikidata.org/wiki/Q21510851) -* [required qualifier constraint (Q21510856)](https://www.wikidata.org/wiki/Q21510856) -* [single-value constraint (Q19474404)](https://www.wikidata.org/wiki/Q19474404): this will only trigger if you are adding more than one statement with the property on the same item, but will not detect any existing statement with this property. -* [distinct values constraint (Q21502410)](https://www.wikidata.org/wiki/Q21502410): similarly, this only checks for conflicts inside your edit batch. - -A comparison of the supported constraints with respect to other implementations is available [here](https://www.wikidata.org/wiki/Wikidata:WikiProject_property_constraints/reports/implementations). - -## Generic issues {#generic-issues} - -OpenRefine also detects issues that are not flagged (yet) by constraint violations on Wikidata: -* Statements without references. This does not rely on [citation needed constraint (Q54554025)](https://www.wikidata.org/wiki/Q54554025): all statements are expected to have references. (The idea is that when importing a dataset, every statement you add -* should link to this dataset - it does not hurt to do it even for generic properties such as [instance of (P31)](https://www.wikidata.org/wiki/Property:P31).) -* Spurious whitespace and non-printable characters in strings (including labels, descriptions and aliases); -* Self-referential statements (statements which mention the item they belong to); -* New items created without any label; -* New items created without any description; -* New items created without any [instance of (P31)](https://www.wikidata.org/wiki/Property:P31) or [subclass of (P279)](https://www.wikidata.org/wiki/Property:P279) statement. diff --git a/OpenRefine/docs/docs/manual/wikibase/reconciling.md b/OpenRefine/docs/docs/manual/wikibase/reconciling.md deleted file mode 100644 index 0e5625c9f..000000000 --- a/OpenRefine/docs/docs/manual/wikibase/reconciling.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -id: reconciling -title: Reconciling with Wikibase -sidebar_label: Reconciling with Wikibase ---- - -The Wikidata [reconciliation service](reconciling) for OpenRefine [supports](https://reconciliation-api.github.io/testbench/): -* A large number of potential types to reconcile against -* Previewing and viewing entities -* Suggesting entities, types, and properties -* Augmenting your project with more information pulled from Wikidata. - -You can find documentation and further resources on the reconciliation API [here](https://wikidata.reconci.link/). - -For the most part, Wikidata reconciliation behaves the same way other reconciliation services do, but there are a few processes and features specific to Wikidata. - -## Language settings {#language-settings} - -You can install a version of the Wikidata reconciliation service that uses your language. First, you need the language code: this is the [two-letter code found on this list](https://en.wikipedia.org/wiki/List_of_Wikipedias), or in the domain name of the desired Wikipedia/Wikidata (for instance, “fr” if your Wikipedia is https://fr.wikipedia.org/wiki/). - -Then, open the reconciliation window (under ReconcileStart reconciling...) and click Add Standard Service. The URL to enter is `https://wikidata.reconci.link/fr/api`, where “fr” is your desired language code. - -When reconciling using this interface, items and properties will be displayed in your chosen language if the label is available. The matching score of the reconciliation is not influenced by your choice of language for the service: items are matched by considering all labels and returning the best possible match. The language of your dataset is also irrelevant to your choice of language for the reconciliation service; it simply determines which language labels to return based on the entity chosen. - -## Restricting matches by type {#restricting-matches-by-type} - -In Wikidata, types are items themselves. For instance, the [university of Ljubljana (Q1377)](https://www.wikidata.org/wiki/Q1377) has the type [public university (Q875538)](https://www.wikidata.org/wiki/Q875538), using the [instance of (P31)](https://www.wikidata.org/wiki/Property:P31) property. Types can be subclasses of other types, using the [subclass of (P279)](https://www.wikidata.org/wiki/Property:P279) property. For instance, [public university (Q875538)](https://www.wikidata.org/wiki/Q875538) is a subclass of [university (Q3918)](https://www.wikidata.org/wiki/Q3918). You can visualize these structures with the [Wikidata Graph Builder](https://angryloki.github.io/wikidata-graph-builder/). - -When you select or enter a type for reconciliation, OpenRefine will include that type and all of its subtypes. For instance, if you select [university (Q3918)](https://www.wikidata.org/wiki/Q3918), then [university of Ljubljana (Q1377)](https://www.wikidata.org/wiki/Q1377) will be a possible match, though that item isn't directly linked to Q3918 - because it is directly linked to Q875538, the subclass of Q3918. - -Some items and types may not yet be set as an instance or subclass of anything (because Wikidata is crowdsourced). If you restrict reconciliation to a type, items without the chosen type will not appear in the results, except as a fallback, and will have a lower score. - -## Reconciling via unique identifiers {#reconciling-via-unique-identifiers} - -You can supply a column of unique identifiers (in the form "Q###" for entities) directly to Wikidata in order to pull more data, but [these strings will not be “reconciled” against the external dataset](reconciling#reconciling-with-unique-identifiers). Apply the operation ReconcileUse values as identifiers on your column of QIDs. All cells will appear as dark blue “confirmed” matches. Some of the “matches” may be errors, which you will need to hover over or click on to identify. You cannot use this to reconcile properties (in the form "P###"). - -If the identifier you submit is assigned to multiple Wikidata items (because Wikidata is crowdsourced), all of the items are returned as candidates, with none automatically matched. - -## Property paths, special properties, and subfields {#property-paths-special-properties-and-subfields} - -Wikidata's hierarchical property structure can be called by using property paths (using |, /, and . symbols). Labels, aliases, descriptions, and sitelinks can also be accessed. You can also match values against subfields, such as latitude and longitude subfields of a geographical coordinate. - -For information on how to do this, read the [documentation and further resources here](https://wikidata.reconci.link/#documentation). - - diff --git a/OpenRefine/docs/docs/manual/wikibase/schema-alignment.md b/OpenRefine/docs/docs/manual/wikibase/schema-alignment.md deleted file mode 100644 index 112c1a45c..000000000 --- a/OpenRefine/docs/docs/manual/wikibase/schema-alignment.md +++ /dev/null @@ -1,253 +0,0 @@ ---- -id: schema-alignment -title: Schema alignment -sidebar_label: Schema alignment ---- - -A Wikibase schema is a template of Wikidata edits that is applied -to each row in the project. This page describes how each part of this -template works, and how it generates edits depending on the contents of -the table cells. - -## Items {#items} - -An item in the schema represents a set of changes on a particular -Wikidata item, generated by a single row. This item can contain changes -in [terms](#terms) (labels, descriptions and aliases) or -[statements](#statements). - -It is possible to make edits on different items for each row of your -table: just add multiple items in your schema. Each item has a subject, -which can be either entered manually (when the item on which the edits -should be made is the same for all rows), or any reconciled column can -be dropped in this field. In this case, the edits will depend on the -reconciliation status of each cell: - -- If the cell is matched to an item, edits will be made on that item; -- If the cell is marked as corresponding to a new item, a new item - will be created for it. See [New items](./new-entities) for more - details about how this works; -- If the cell has reconciliation candidates but has not been matched - to any of them, the edit will be skipped (even if there is only one - candidate with a high reconciliation score); -- If the cell is not reconciled or blank, the edit will be skipped. - -Do not worry about the ordering of items in the schema or the order of -your rows, as OpenRefine will rearrange your edits to optimize their -upload. If your project makes edits on the same item across multiple -rows, these edits will be merged together and performed in one edit. See -[Uploading your changes](./uploading) about that. - -## Terms {#terms} - -**Terms** are the language-specific strings that you find at the top of -Wikidata items: labels, descriptions and aliases. OpenRefine lets you -edit these terms via the Wikidata schema. - -### Languages {#languages} - -Each term belongs to a particular language. Wikidata supports [hundreds -of languages](https://www.wikidata.org/wiki/Help:Wikimedia_language_codes/lists/all), which -are designated by language codes. For each term that you want to add to -an item, you will need to specify the language for this term. There are -two cases: - -- Either the language is constant across your dataset: you know that - all the names in a given column are spelled in the same language. In - this case, type the name of the language in the input and select the - language in the drop-down suggestion dialog. This will place the - appropriate language code in the input. -- Or the language varies across your dataset. In this case, you need - to provide a column of Wikimedia language codes that indicates the - language for each term that you want to add. Just drag and drop this - column to the language field. If there are any invalid language - codes in this column, the corresponding terms will be ignored. - OpenRefine will translate any deprecated language codes to their - preferred values silently. - -### Labels {#labels} - -This is because Wikidata items can have at most one label per language, -so you need to choose whether to override any existing label (default -behaviour before 3.2) or only insert your label if there is no such -label in the given language (default behaviour starting from 3.2). When -the content of the cell providing the label is blank, nothing will be -changed (so, it is not possible to remove labels). - -### Descriptions {#descriptions} - -Descriptions work like labels: there is at most one description per -language, and OpenRefine can override existing descriptions or leave -them unchanged. It is not possible to remove descriptions either. - -### Aliases {#aliases} - -Aliases are added to the list of existing aliases in the given language. -When adding an alias in a language where no label has been added yet, -the alias is automatically promoted to a label for this language. It is -not possible to remove aliases or to override any existing aliases. - -## Statements {#statements} - -You can add statements in the schema: this will generate new statements -on the corresponding items. These statements will be merged with any -existing statements on the actual Wikidata items and [this merging process depends on the upload medium](./uploading#Merging-strategies-for-statements). -It is forecast to give more control over the merging strategy in the -near future. - -### Main values {#main-values} - -Statements must have main values: \"novalue\" or \"somevalue\" -statements are not supported yet. The main value of a statement is a -data value whose type depends on the property used for the statement. If -the main value cannot be evaluated (for instance because one of the -cells it depends on is empty), then the entire statement will be -skipped. - -See the [data values](#data-values) section for more details -about how to specify each type of data value and when they are skipped. - -### Qualifiers {#qualifiers} - -Qualifiers can be added on each statement. When their values are -skipped, only the qualifier will be discarded: the rest of the statement -will still be added. - -### References {#references} - -References can (and should) be added to back each statement. If values -inside the reference are skipped, the corresponding part of the -reference will be discarded but the reference will still be added -(unless the reference becomes empty). - -### Ranks {#ranks} - -All statements ranks are set to **Normal**. It is currently not possible -to set a different rank. - -## Data values {#data-values} - -Data values are the data that you can find as target of a statement (or -qualifier, or part of a reference). Each property dictates a particular -type of data value. In each case, OpenRefine uses a particular process -to translate cell contents to a data value of the appropriate type. This -section explains the process for all data types. - -### Items {#items-1} - -Items are evaluated in the same way as the subjects of items in the -schema. They can be input directly using the auto-suggest service -provided, or any column reconciled against Wikidata can be used. Refer to -[the first Items section](#items) to see how they are -evaluated. - -### Strings and external identifiers {#strings-and-external-identifiers} - -Bare strings and external identifiers can be input directly as constants -(if they do not change across rows) or using any column. If a reconciled -column is used for a string value, it is the value of the cell that is -going to be used, not the name of the reconciled item (which is what -OpenRefine displays). Values are skipped when the column is blank or -null. - -### Monolingual texts {#monolingual-texts} - -Monolingual texts consist of two parts: - -- the language: see [Languages](#languages) for their - structure; -- the value of the text: see [the section above](#strings-and-external-identifiers). - -A monolingual text is skipped when any of its parts is skipped (that is, -if the language or the text are invalid). - -### Dates {#dates} - -Dates are parsed from cell contents (or from any constant provided in -the schema) and the precision of the date is inferred from its format. -Here are the valid formats: - -- `YYYYM`, such as `2001M` (millenium precision) -- `YYYYC`, such as `1901C` (century precision) -- `YYYYD`, such as `1981D` (decade precision) -- `YYYY`, such as `1984` (year precision) -- `YYYY-MM`, such as `2019-03` (month precision) -- `YYYY-MM-DD`, such as `1897-08-14` (day precision) - -Any value that does not match any of these formats will be ignored. All -dates are represented in UTC, Gregorian calendar. - -In OpenRefine 3.3, the following new formats have been introduced: - -- `TODAY` returns today's date with day precision. This will be - evaluated when performing the edits (or exporting to - QuickStatements); -- `YYYY-MM-DD_QID` can be used to specify a date in a particular - calendar (such as the [proleptic Julian calendar (Q1985786)](https://www.wikidata.org/wiki/Q1985786). - -In OpenRefine 3.5, the following new format has been introduced: - -- `-234` represents the year 234 [BCE](https://en.wikipedia.org/wiki/Common_Era) - -### Quantities {#quantities} - -Quantities consist of two parts: the amount and the unit. - -- the amount is mandatory and must be a string, such as `18,229.1020`. - The precision that is displayed will be respected (the same number - of trailing zeros will be shown in Wikidata). By default, no upper - and lower bounds will be set. To define these, one needs to use the - engineering notation, such as `3.45E+3`, which will be interpreted - as `3,450±5`. As usual, the amount can be provided as a constant or - as a column variable. In the latter case, the values in the column - must be strings. -- the unit is optional. It is an item, so it can be provided either - with the auto-suggest dialog or as a reconciled column. It is - important to note that if a reconciled column is used, any - unreconciled cells will discard the entire quantity value. So a - template for a quantity value is either always unit-less, or always - has a unit. - -### Globe coordinates {#globe-coordinates} - -Geographic coordinates are specified as strings with the following -formats, where all components are floating point numbers in degrees: - -- `latitude,longitude` for a default precision of ten micro degrees - (for instance: - [`49.265278,4.028611`](https://tools.wmflabs.org/geohack/geohack.php?params=49.265277777778_N_4.0286111111111_E_globe:earth&language=en) - can be used indicate the position of Reims, France. - - -- `latitude,longitude,precision` when specifying an explicit precision - (for instance: `49.265278,4.028611,0.1` can be used indicate the - position of Reims within a tenth of a degree). - -All globe coordinates are on Earth ([Q2](https://www.wikidata.org/wiki/Q2)). - -If your coordinates are in a different format, such as -`49° 15′ 55″ N, 4° 1′ 43″ E`, you will need to convert them to decimal -format first. - -### Media on Commons {#media-on-commons} - -Media on Wikimedia Commons is treated like strings, whose values must -exactly match filenames on Commons. These values are not checked during -schema evaluations: if they are wrong, uploading the statements will -fail. - -Tabular data and Geoshapes must be prefixed with the `Data:` namespace. -This is indicated by the placeholder in the field that appears when -constructing the schema. - -### Properties {#properties} - -Properties are always constants: there is currently no way to reconcile -a column against properties. They have to be selected with the -auto-suggest dialog. - -### Other data types {#other-data-types} - -URLs, mathematical expressions and other textual datatypes are supported -and treated as strings. At the time of writing, all datatypes supported -by Wikidata are supported by OpenRefine. diff --git a/OpenRefine/docs/docs/manual/wikibase/uploading.md b/OpenRefine/docs/docs/manual/wikibase/uploading.md deleted file mode 100644 index 99da1259d..000000000 --- a/OpenRefine/docs/docs/manual/wikibase/uploading.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -id: uploading -title: Uploading edits to Wikibase -sidebar_label: Uploading edits ---- - -This page explains how to upload your edits to the target Wikibase. It assumes you already have a created a Wikibase schema in your OpenRefine project. - -## Uploading with OpenRefine {#uploading-with-openrefine} - -* Click WikidataUpload edits to Wikidata. -* Log in with your personal account or your bot account depending on which account you want to use to make the edits. It is a good practice to use a [bot password](https://www.mediawiki.org/wiki/Manual:Bot_passwords). -* Supply a meaningful edit summary. This is especially important because OpenRefine condenses all your changes on the same item as one edit: if you are making multiple changes, the edit summary generated by Wikibase will not indicate clearly what sort of change you made. If you are making atomic changes, such as adding a single alias or statement, the automatic edit summaries will be more meaningful. If supported by your Wikibase instance, OpenRefine will append a link to the [EditGroups](https://editgroups.toolforge.org/) tool, which lets you track and analyze your edit batch after upload. -* Click Perform edits and wait for the operation to complete. You can watch your edits being made by checking your wiki contributions or the EditGroups tool. - -Because performing edits in OpenRefine counts as an operation, you can extract this operation and reapply it to other projects. If you do so, you should also include the operation that saves the schema (only the last one is required), and make sure that the column names in the schema match those of the OpenRefine project where you are applying the operation. - -## Uploading with QuickStatements {#uploading-with-quickstatements} - -This requires that the Wikibase site has an associated [QuickStatements](https://meta.wikimedia.org/wiki/QuickStatements) tool. - -* Click WikibaseExport to QuickStatements and copy the contents of the file; -* Go to QuickStatements (for Wikidata it can be found at https://quickstatements.toolforge.org/) and login to authorize the tool to use your account; -* Click Version 1 format; -* Paste the generated changes in the text area; -* Perform the edits with Run or Run in background. - -## Notable differences between the two methods {#notable-differences-between-the-two-methods} - -### Merging strategy for statements {#merging-strategy-for-statements} - -OpenRefine checks for existing statements which match not only the property and the target value, but also the qualifiers. On the other hand, QuickStatements ignores qualifiers when matching statements. -Both merging strategies can be useful depending on the properties. It is forecast to let the user configure the matching method in OpenRefine. - -If references are provided, both tools merge references in matching statements. - -### New item creation {#new-item-creation} - -OpenRefine supports creating new items with arbitrary relations between them. - -QuickStatements supports creating new items with the CREATE instruction, and subsequent instructions can use the LAST placeholder to use the Qid of the last created item. When generating QuickStatements instructions, OpenRefine reorders your edits so that this syntax can be used. In rare cases, such as when a statement links two newly-created items, it is impossible to use QuickStatements to perform the edit. In this case, no QuickStatements script will be generated. - -### Speed and number of edits {#speed-and-number-of-edits} - -OpenRefine generally performs one edit per item touched by an edit batch and at most two in general (in the case where new items contain links between them). This was chosen to minimize server load, speed up the upload and keep item histories compact. The downside is that the edit summaries can be less meaningful - it is therefore important that users supply informative summaries when uploading their batches. OpenRefine asymptotically edits at the rate of 60 edits per minute (so, usually 60 items per minute). The first edits are made more quickly, which is convenient for small batches. - -QuickStatements performs incremental edits (for instance, when adding a statement with a qualifier and a reference, it will make three edits). That generally means lower speed, but more explicit item histories. - diff --git a/OpenRefine/docs/docs/technical-reference/architecture.md b/OpenRefine/docs/docs/technical-reference/architecture.md deleted file mode 100644 index e80d7015a..000000000 --- a/OpenRefine/docs/docs/technical-reference/architecture.md +++ /dev/null @@ -1,283 +0,0 @@ ---- -id: architecture -title: Architecture -sidebar_label: Architecture ---- - -OpenRefine is a web application, but is designed to be run locally on your own machine. The server-side maintains states of the data (undo/redo history, long-running processes, etc.) while the client-side maintains states of the user interface (facets and their selections, view pagination, etc.). The client-side makes GET and POST ajax calls to cause changes to the data and to fetch data and data-related states from the server-side. - -This architecture provides a good separation of concerns (data vs. UI); allows the use of familiar web technologies (HTML, CSS, Javascript) to implement user interface features; and enables the server side to be called by third-party software through standard GET and POST operations. - -## Technology stack {#technology-stack} - -The server-side part of OpenRefine is implemented in Java as one single servlet which is executed by the [Jetty](http://jetty.codehaus.org/jetty/) web server + servlet container. The use of Java strikes a balance between performance and portability across operating systems (there is very little OS-specific code and has mostly to do with starting the application). - -OpenRefine has no database. It uses its own in-memory data-store that is built up-front to be optimized for the operations required by faceted browsing and infinite undo. - -The client-side part of OpenRefine is implemented in HTML, CSS and Javascript and uses the following libraries: -* [jQuery](http://jquery.com/) -* [jQueryUI](http:jqueryui.com/) -* [Recurser jquery-i18n](https://github.com/recurser/jquery-i18n) - -The functional extensibility of OpenRefine is provided by a fork of the [SIMILE Butterfly](https://github.com/OpenRefine/simile-butterfly) modular web application framework. - -Several projects provide the functionality to read and write custom format files (POI, opencsv, JENA, marc4j). - -String clustering is provided by the [SIMILE Vicino](http://code.google.com/p/simile-vicino/) project. - -OAuth functionality is provided by the [Signpost](https://github.com/mttkay/signpost) project. - -## Server-side architecture {#server-side-architecture} - -OpenRefine's server-side is written entirely in Java (`main/src/`) and its entry point is the Java servlet `com.google.refine.RefineServlet`. By default, the servlet is hosted in the lightweight Jetty web server instantiated by `server/src/com.google.refine.Refine`. Note that the server class itself is under `server/src/`, not `main/src/`; this separation leaves the possibility of hosting `RefineServlet` in a different servlet container. - -The web server configuration is in `main/webapp/WEB-INF/web.xml`; that's where `RefineServlet` is hooked up. `RefineServlet` itself is simple: it just reacts to requests from the client-side by routing them to the right `Command` class in the packages `com.google.refine.commands.**`. - -As mentioned before, the server-side maintains states of the data, and the primary class involved is `com.google.refine.ProjectManager`. - -### Projects {#projects} - -In OpenRefine there's the concept of a workspace similar to that in Eclipse. When you run OpenRefine it manages projects within a single workspace, and the workspace is embodied in a file directory with sub-directories. The default workspace directories are listed in the [FAQs](https://github.com/OpenRefine/OpenRefine/wiki/FAQ-Where-Is-Data-Stored). You can get OpenRefine to use a different directory by specifying a -d parameter at the command line. - -The class `ProjectManager` is what manages the workspace. It keeps in memory the metadata of every project (in the class `ProjectMetadata`). This metadata includes the project's name and last modified date, and any other information necessary to present and let the user interact with the project as a whole. Only when the user decides to look at the project's data would `ProjectManager` load the project's actual data. The separation of project metadata and data is to minimize the amount of stuff loaded into memory. - -A project's _actual_ data includes the columns, rows, cells, reconciliation records, and history entries. - -A project is loaded into memory when it needs to be displayed or modified, and it remains in memory until 1 hour after the last time it gets modified. Periodically the project manager tries to save modified projects, and it saves as many modified projects as possible within 30 seconds. - -### Data Model {#data-model} - -A project's data consists of - -- _raw data_: a list of rows, each row consisting of a list of cells -- _models_ on top of that raw data that give high-level presentation or interpretation of that data. This design lets the same raw data be viewed in different ways by different models, and let the models be changed without costly changes to the raw data. - -#### Column Model {#column-model} - -Cells in rows are not named and can only be addressed by their list position indices. So, a _column model_ is needed to give a name to each list position. The column model also stores other metadata for each column, including the type that cells in the column have been reconciled to and the overall reconciliation statistics of those cells. - -Each column also acts as a cache for data computed from the raw data related to that column. - -Columns in the column model can be removed and re-ordered without changing the raw data--the cells in the rows. This makes column removal and ordering operations really quick. - -##### Column Groups {#column-groups} - -Consider the following data: - -![Illustration of row groups in OpenRefine](https://raw.github.com/OpenRefine/OpenRefine/2.0/graphics/row-groups.png) - -Although the data is in a grid, we humans can understand that it is a tree. First of all, all rows contain data ultimately linked to the movie Austin Powers, although only one row contains the text "Austin Powers" in the "movie title" column. We also know that "USA" and "Germany" are not related to Elizabeth Hurley and Mike Myers respectively (say, as their nationality), but rather, "USA" and "Germany" are related to the movie (where it was released). We know that Mike Myers played both the character "Austin Powers" and the character "Dr. Evil"; and for the latter he received 2 awards. We humans can understand how to interpret the grid as a tree based on its visual layout as well as some knowledge we have about the movie domain but is not encoded in the table. - -OpenRefine can capture our knowledge of this transformation from grid to tree using _column groups_, also stored in the column model. Each column group illustrated as a blue bracket above specifies which columns are grouped together, as well as which of those columns is the key column in that group (blue triangle). One column group can span over columns grouped by another column group, and in this way, column groups form a hierarchy determined by which column group envelopes another. This hierarchy of column groups allows the 2-dimensional (grid-shaped) table of rows and cells to be interpreted as a list of hierarchical (tree-shaped) data records. - -Blank cells play a very important role. The blank cell in a key column of a row (e.g., cell "character" on row 4) makes that row (row 4) _depend_ on the first preceding row with that column filled in (row 3). This means that "Best Comedy Perf" on row 4 applies to "Dr. Evil" on row 3. Row 3 is said to be a _context row_ for row 4. Similarly, since rows 2 - 6 all have blank cells in the first column, they all depend on row 1, and all their data ultimately applies to the movie Austin Powers. Row 1 depends on no other row and is said to be a _record row_. Rows 1 - 6 together form one _record_. - -Currently (as of 12th December 2017) only the XML and JSON importers create column groups, and while the data table view does display column groups but it doesn't support modifying them. - -### Changes, History, Processes, and Operations {#changes-history-processes-and-operations} - -All changes to the project's data are tracked (N.B. this does not include changes to a project's metadata - such as the project name.) - -Changes are stored as `com.google.refine.history.Change` objects. `com.google.refine.history.Change` is an interface, and implementing classes are in `com.google.refine.model.changes.**`. Each change object stores enough data to modify the project's data when its `apply()` method is called, and enough data to revert its effect when its `revert()` method is called. It's only supposed to _store_ data, not to actually _compute_ the change. In this way, it's like a .diff patch file for a code base. - -Some change objects can be huge, as huge as the project itself. So change objects are not kept in memory except when they are to be applied or reverted. However, since we still need to show the user some information about changes (as displayed in the History panel in the UI), we keep metadata of changes separate from the change objects. For each change object there is one corresponding `com.google.refine.history.HistoryEntry` for storing its metadata, such as the change's human-friendly description and timestamp. - -Each project has a `com.google.refine.history.History` object that contains an ordered list of all `HistoryEntry` objects storing metadata for all changes that have been done since after the project was created. Actually, there are 2 ordered lists: one for done changes that can be reverted (undone), an done for undone changes that can be re-applied (redone). Changes must be done or redone in their exact orders in these lists because each change makes certain assumptions about the state of the project before and after it is applied. As changes cannot be undone/redone out of order, when one change fails to revert, it blocks the whole history from being reverted to any state preceding that change (as happened in [Issue #2](https://github.com/OpenRefine/OpenRefine/issues/2)). - -As mentioned before, a change contains only the diff and does not actually compute that diff. The computation is performed by a `com.google.refine.process.Process` object--every change object is created by a process object. A process can be immediate, producing its change object synchronously within a very short period of time (e.g., starring one row); or a process can be long-running, producing its change object after a long time and a lot of computation, including network calls (e.g., reconciling a column). - -As the user interacts with the UI on the client-side, their interactions trigger ajax calls to the server-side. Some calls are meant to modify the project. Those are handled by commands that instantiates processes. Processes are queued in a first-in-first-out basis. The first-in process gets run and until it is done all the other processes are stuck in the queue. - -A process can effect a change in one thing in the project (e.g., edit one particular cell, star one particular row), or a process can effect changes in _potentially_ many things in the project (e.g., edit zero or more cells sharing the same content, starring all rows filtered by some facets). The latter kind of process is generalizable: it is meaningful to apply them on another similar project. Such a process is associated with an _abstract operation_ `com.google.refine.model.AbstractOperation` that encodes the information necessary to create another instance of that process, but potentially for a different project. When you click "extract" in the History panel, these abstract operations are called to serialize their information to JSON; and when you click "apply" in the History panel, the JSON you paste in is used to re-construct these abstract operations, which in turn create processes, which get run sequentially in a queue to generate change object and history entry pairs. - -In summary, - -- change objects store diffs -- history entries store metadata of change objects -- processes compute diffs and create change object and history entry pairs -- some processes are long-running and some are immediate; processes are run sequentially in a queue -- generalizable processes can be re-constructed from abstract operations - -## Client-side architecture {#client-side-architecture} -The client-side part of OpenRefine is implemented in HTML, CSS and Javascript and uses the following Javascript libraries: -* [jQuery](http://jquery.com/) -* [jQueryUI](http:jqueryui.com/) -* [Recurser jquery-i18n](https://github.com/recurser/jquery-i18n) - -### Importing architecture {#importing-architecture} - -OpenRefine has a sophisticated architecture for accommodating a diverse and extensible set of importable file formats and work flows. The formats range from simple CSV, TSV to fixed-width fields to line-based records to hierarchical XML and JSON. The work flows allow the user to preview and tweak many different import settings before creating the project. In some cases, such as XML and JSON, the user also has to select which elements in the data file to import. Additionally, a data file can also be an archive file (e.g., .zip) that contains many files inside; the user can select which of those files to import. Finally, extensions to OpenRefine can inject functionalities into any part of this architecture. - -### The Index Page and Action Areas {#the-index-page-and-action-areas} - -The opening screen of OpenRefine is implemented by the file refine/main/webapp/modules/core/index.vt and will be referred to here as the index page. Its default implementation contains 3 finger tabs labeled Create Project, Open Project, and Import Project. Each tab selects an "action area". The 3 default action areas are for, obviously, creating a new project, opening an existing project, and importing a project .tar file. - -Extensions can add more action areas in Javascript. For example, this is how the Create Project action area is added (refine/main/webapp/modules/core/scripts/index/create-project-ui.js): - -```javascript -Refine.actionAreas.push({ - id: "create-project", - label: "Create Project", - uiClass: Refine.CreateProjectUI -}); -``` - -The UI class is a constructor function that takes one argument, a jQuery-wrapped HTML element where the tab body of the action area should be rendered. - -If your extension requires a very unique importing work flow, or a very novel feature that should be exposed on the index page, then add a new action area. Otherwise, try to use the existing work flows as much as possible. - -### The Create Project Action Area {#the-create-project-action-area} - -The Create Project action area is itself extensible. Initially, it embeds a set of finger tabs corresponding to a variety of "source selection UIs": you can select a source of data by specifying a file on your computer, or you can specify the URL to a publicly accessible data file or data feed, or you can paste in from the clipboard a chunk of data. - -There are actually 3 points of extension in the Create Project action area, and the first is invisible. - -#### Importing Controllers {#importing-controllers} - -The Create Project action area manages a list of "importing controllers". Each controller follows a particular work flow (in UI terms, think "wizard"). Refine comes with a "default importing controller" (refine/main/webapp/modules/core/scripts/index/default-importing-controller/controller.js) and its work flow assumes that the data can be retrieved and cached in whole before getting processed in order to generate a preview for the user to inspect. (If the data cannot be retrieved and cached in whole before previewing, then another importing controller is needed.) - -An importing controller is just programming logic, but it can manifest itself visually by registering one or more data source UIs and one or more custom panels in the Create Project action area. The default importing controller registers 3 such custom panels, which act like pages of a wizard. - -An extension can register any number of importing controller. Each controller has a client-side part and a server-side part. Its client-side part is just a constructor function that takes an object representing the Create Project action area (usually named `createProjectUI`). The controller (client-side) is expected to use that object to register data source UIs and/or create custom panels. The controller is not expected to have any particular interface method. The default importing controller's client-side code looks like this (refine/main/webapp/modules/core/scripts/index/default-importing-controller/controller.js): - -```javascript -Refine.DefaultImportingController = function(createProjectUI) { - this._createProjectUI = createProjectUI; // save a reference to the create project action area - - this._progressPanel = createProjectUI.addCustomPanel(); // create a custom panel - this._progressPanel.html('...'); // render the custom panel - ... do other stuff ... -}; -Refine.CreateProjectUI.controllers.push(Refine.DefaultImportingController); // register the controller -``` - -We will cover the server-side code below. - -#### Data Source Selection UIs {#data-source-selection-uis} - -Data source selection UIs are another point of extensibility in the Create Project action area. As mentioned previously, by default there are 3 data source UIs. Those are added by the default importing controller. - -Extensions can also add their own data source UIs. A data source selection UI object can be registered like so - -```javascript -createProjectUI.addSourceSelectionUI({ - label: "This Computer", - id: "local-computer-source", - ui: theDataSourceSelectionUIObject -}); -``` - -`theDataSourceSelectionUIObject` is an object that has the following member methods: - -- `attachUI(bodyDiv)` -- `focus()` - -If you want to install a data source selection UI that is managed by the default importing controller, then register its UI class with the default importing controller, like so (refine/main/webapp/modules/core/scripts/index/default-importing-sources/sources.js): - -```javascript -Refine.DefaultImportingController.sources.push({ - "label": "This Computer", - "id": "upload", - "uiClass": ThisComputerImportingSourceUI -}); -``` - -The default importing controller will assume that the `uiClass` field is a constructor function and call it with one argument--the controller object itself. That constructor function should save the controller object for later use. More specifically, for data source UIs that use the default importing controller, they can call the controller to kickstart the process that retrieves and caches the data to import: - -```javascript -controller.startImportJob(form, "... status message ..."); -``` - -The argument `form` is a jQuery-wrapped FORM element that will get submitted to the server side at the command /command/core/create-importing-job. That command and the default importing controller will take care of uploading or downloading the data, caching it, updating the client side's progress display, and then showing the next importing step when the data is fully cached. - -See refine/main/webapp/modules/core/scripts/index/default-importing-sources/sources.js for examples of such source selection UIs. While we write about source selection UIs managed by the default importing controller here, chances are your own extension will not be adding such a new source selection UI. Your extension probably adds with a new importing controller as well as a new source selection UI that work together. - -#### File Selection Panel {#file-selection-panel} -Documentation not currently available - -#### Parsing UI Panel {#parsing-ui-panel} -Documentation not currently available - -### Server-side Components {#server-side-components} - -#### ImportingController {#importingcontroller} -Documentation not currently available - -#### UrlRewriter {#urlrewriter} -Documentation not currently available - -#### FormatGuesser {#formatguesser} -Documentation not currently available - -#### ImportingParser {#importingparser} -Documentation not currently available - - -## Faceted browsing architecture {#faceted-browsing-architecture} - -Faceted browsing support is core to OpenRefine as it is the primary and only mechanism for filtering to a subset of rows on which to do something _en masse_ (ie in bulk). Without faceted browsing or an equivalent querying/browsing mechanism, you can only change one thing at a time (one cell or row) or else change everything all at once; both kinds of editing are practically useless when dealing with large data sets. - -In OpenRefine, different components of the code need to know which rows to process from the faceted browsing state (how the facets are constrained). For example, when the user applies some facet selections and then exports the data, the exporter serializes only the matching rows, not all rows in the project. Thus, faceted browsing isn't only hooked up to the data view for displaying data to the user, but it is also hooked up to almost all other parts of the system. - -### Engine Configuration {#engine-configuration} - -As OpenRefine is a web app, there might be several browser windows opened on the same project, each in a different faceted browsing state. It is best to maintain the faceted browsing state in each browser window while keeping the server side completely stateless with regard to faceted browsing. Whenever the client-side needs something done by the server, it transfers the entire faceted browsing state over to the server-side. The faceted browsing state behaves much like the `WHERE` clause in a SQL query, telling the server-side how to select the rows to process. - -In fact, it is best to think of the faceted browsing state as just a database query much like a SQL query. It can be passed around the whole system, to any component needing to know which rows to process. It is serialized into JSON to pass between the client-side and the server side, or to save in an abstract operation's specification. The job of the faceted browsing subsystem on the client-side is to let the user interactively modify this "faceted browsing query", and the job of the faceted browsing subsystem on the server side is to resolve that query. - -In the code, the faceted browsing state, or faceted browsing query, is actually called the *engine configuration* or *engine config* for short. It consists mostly of an array facet configurations. For each facet, it stores the name of the column on which the facet is based (or an empty string if there is no base column). Each type of facet has different configuration. Text search facets have queries and flags for case-sensitivity mode and regular expression mode. Text facets (aka list facets) and numeric range facets have expressions. Each list facet also has an array of selected choices, an invert flag, and flags for whether blank and error cells are selected. Each numeric range facet has, among other things, a "from" and a "to" values. If you trace the AJAX calls, you'd see the engine configs being shuttled, e.g., - -```json -{ - "facets" : [ - { - "type": "text", - "name": "Shrt_Desc", - "columnName": "Shrt_Desc", - "mode": "text", - "caseSensitive": false, - "query": "cheese" - }, - { - "type": "list", - "name": "Shrt_Desc", - "columnName": "Shrt_Desc", - "expression": "grel:value.toLowercase().split(\",\")", - "omitBlank": false, - "omitError": false, - "selection": [], - "selectBlank":false, - "selectError":false, - "invert":false - }, - { - "type": "range", - "name": "Water", - "expression": "value", - "columnName": "Water", - "selectNumeric": true, - "selectNonNumeric": true, - "selectBlank": true, - "selectError": true, - "from": 0, - "to": 53 - } - ], - "includeDependent": false - } -``` - -### Server-Side Subsystem {#server-side-subsystem} - -From an engine configuration like the one above, the server-side faceted browsing subsystem is capable of producing: - -- an iteration over the rows matching the facets' constraints -- information on how to render the facets (e.g., choice and count pairs for a list facet, histogram for a numeric range facet) - -When the engine config JSON arrives in an HTTP request on the server-side, a `com.google.refine.browsing.Engine` object is constructed and initialized with that JSON. It in turns constructs zero or more `com.google.refine.browsing.facets.Facet` objects. Then for each facet, the engine calls its `getRowFilter()` method, which returns `null` if the facet isn't constrained in anyway, or a `com.google.refine.browsing.filters.RowFilter` object. Then, to when iterating over a project's rows, the engine calls on all row filters' `filterRow()` method. If and only if all row filters return `true` the row is considered to match the facets' constraints. How each row filter works depends on the corresponding type of facet. - -To produce information on how to render a particular facet in the UI, the engine follows the same procedure described in the previous except it skips over the facet in question. In other words, it produces an iteration over all rows constrained by the other facets. Then it feeds that iteration to the facet in question by calling the facet's `computeChoices()` method. This gives the method a chance to compute the rendering information for its UI counterpart on the client-side. When all facets have been given a chance to compute their rendering information, the engine calls all facets to serialize their information as JSON and returns the JSON to the client-side. Only one HTTP call is needed to compute all facets. - -### Client-side subsystem {#client-side-subsystem} - -On the client-side there is also an engine object (implemented in Javascript rather than Java) and zero or more facet objects (also in Javascript, obviously). The engine is responsible for distributing the rendering information computed on the server-side to the right facets, and when the user interacts with a facet, the facet tells the engine to update the whole UI. To do so, the engine gathers the configuration of each facet and composes the whole engine config as a single JSON object. Two separate AJAX calls are made with that engine config, one to retrieve the rows to render, and one to re-compute the rendering information for the facets because changing one facet does affect all the other facets. diff --git a/OpenRefine/docs/docs/technical-reference/build-test-run.md b/OpenRefine/docs/docs/technical-reference/build-test-run.md deleted file mode 100644 index 300784ac8..000000000 --- a/OpenRefine/docs/docs/technical-reference/build-test-run.md +++ /dev/null @@ -1,283 +0,0 @@ ---- -id: build-test-run -title: How to build, test and run -sidebar_label: How to build, test and run ---- - -import useBaseUrl from '@docusaurus/useBaseUrl'; - - -You will need: -* [OpenRefine source code](https://github.com/OpenRefine/OpenRefine) -* [Java JDK](http://java.sun.com/javase/downloads/index.jsp) (Get [OpenJDK from here](https://jdk.java.net/15/).) -* [Apache Maven](https://maven.apache.org) (OPTIONAL) -* A Unix/Linux shell environment OR the Windows command line - -From the top level directory in the OpenRefine application you can build, test and run OpenRefine using the `./refine` shell script (if you are working in a \*nix shell), or using the `refine.bat` script from the Windows command line. Note that the `refine.bat` on Windows only supports a subset of the functionality, supported by the `refine` shell script. The example commands below are using the `./refine` shell script, and you will need to use `refine.bat` if you are working from the Windows command line. - -### Get OpenRefine source code - -With Git installed, use the `git clone` command to download the [project's repo](https://github.com/OpenRefine/OpenRefine) to a directory of your choice. - -### Set up JDK {#set-up-jdk} - -You must [install JDK](https://jdk.java.net/15/) and set the JAVA_HOME environment variable (please ensure it points to the JDK, and not the JRE). - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - - -1. On Windows 10, click the Start Menu button, type `env`, and look at the search results. Click Edit the system environment variables. (If you are using an earlier version of Windows, use the “Search” or “Search programs and files” box in the Start Menu.) - -![A screenshot of the search results for 'env'.](/img/env.png "A screenshot of the search results for 'env'.") - -2. Click Environment Variables… at the bottom of the Advanced window. -3. In the Environment Variables window that appears, click New… and create a variable with the key `JAVA_HOME`. You can set the variable for only your user account, as in the screenshot below, or set it as a system variable - it will work either way. - -![A screenshot of 'Environment Variables'.](/img/javahome.png "A screenshot of 'Environment Variables'.") - -4. Set the `Value` to the folder where you installed JDK, in the format `D:\Programs\OpenJDK`. You can locate this folder with the Browse directory... button. - - - - - -First, find where Java is on your computer with this command: - -``` -which java -``` - -Check the environment variable `JAVA_HOME` with: - -``` -$JAVA_HOME/bin/java --version -``` - -To set the environment variable for the current Java version of your MacOS: - -``` -export JAVA_HOME="$(/usr/libexec/java_home)" -``` - -Or, for Java 13.x: - -``` -export JAVA_HOME="$(/usr/libexec/java_home -v 13)" -``` - - - - - -##### With the terminal {#with-the-terminal} - -Enter the following: - -``` -sudo apt install default-jre -``` - -This probably won’t install the latest JDK package available on the Java website, but it is faster and more straightforward. (At the time of writing, it installs OpenJDK 11.0.7.) - -##### Manually {#manually} - -First, [extract the JDK package](https://openjdk.java.net/install/) to the new directory `usr/lib/jvm`: - -``` -sudo mkdir -p /usr/lib/jvm -sudo tar -x -C /usr/lib/jvm -f /tmp/openjdk-14.0.1_linux-x64_bin.tar.gz -``` - -Then, navigate to this folder and confirm the final path (in this case, `usr/lib/jvm/jdk-14.0.1`). Open a terminal and type - -``` -sudo gedit /etc/profile -``` - -In the text window that opens, insert the following lines at the end of the `profile` file, using the path above: - -``` -JAVA_HOME=/usr/lib/jvm/jdk-14.0.1 -PATH=$PATH:$HOME/bin:$JAVA_HOME/bin -export JAVA_HOME -export PATH -``` - -Note: OpenRefine on Linux currently supports jdk versions 8 to 15. Reference: [Issue 4106](https://github.com/OpenRefine/OpenRefine/issues/4106). - -Save and close the file. When you are back in the terminal, type - -``` -source /etc/environment -``` - -Exit the terminal and restart your system. You can then check that `JAVA_HOME` is set properly by opening another terminal and typing -``` -echo $JAVA_HOME -``` - -It should show the path you set above. - - - - - ---- - - - -### Maven (Optional) {#maven-optional} -OpenRefine's build script will download Maven for you and use it, if not found already locally installed. - -If you will be using your Maven installation instead of OpenRefine's build script download installation, then set the `MVN_HOME` environment variable. You may need to reboot your machine after setting these environment variables. If you receive a message `Could not find the main class: com.google.refine.Refine. Program will exit.` it is likely `JAVA_HOME` is not set correctly. - -Ensure that you set your `MAVEN_HOME` environment variable, for example: - -```shell -MAVEN_HOME=E:\Downloads\apache-maven-3.5.4-bin\apache-maven-3.5.4\ -``` - -NOTE: You can use Maven commands directly, but running some goals in isolation might fail (try adding the `compile test-compile` goals in your invocation if that is the case). - -### Building {#building} - -To see what functions are supported by OpenRefine's build system, type -```shell -./refine -h -``` - -To build the OpenRefine application from source type: -```shell -./refine clean -./refine build -``` - -### Testing {#testing} -Since OpenRefine is composed of two parts, a server and a in-browser UI, the testing system reflects that: - -* on the server side, it's powered by [TestNG](http://testng.org/) and the unit tests are written in Java; -* on the client side, we use [Cypress](https://www.cypress.io/) and the tests are written in Javascript - -To run all tests, use: -```shell -./refine test -``` -**this option is not available when using refine.bat** - - -If you want to run only the server side portion of the tests, use: -```shell -./refine server_test -``` - -If you are running the UI tests for the first time, [you must go through the installation process.](functional-tests) -If you want to run only the client side portion of the tests, use: -```shell -./refine ui_test chrome -``` - -## Running {#running} -To run OpenRefine from the command line (assuming you have been able to build from the source code successfully) -```shell -./refine -``` -By default, OpenRefine will use [refine.ini](https://github.com/OpenRefine/OpenRefine/blob/master/refine.ini) for configuration. You can copy it and rename it to `refine-dev.ini`, which will be used for configuration instead. `refine-dev.ini` won't be tracked by Git, so feel free to put your custom configurations into it. - -## Building Distributions (Kits) {#building-distributions-kits} - -The Refine build system uses Apache Ant to automate the creation of the installation packages for the different operating systems. The packages are currently optimized to run on Mac OS X which is the only platform capable of creating the packages for all three OS that we support. - -To build the distributions type - -```shell -./refine dist -``` -where 'version' is the release version. - -## Building, Testing and Running OpenRefine from Eclipse {#building-testing-and-running-openrefine-from-eclipse} -OpenRefine' source comes with Maven configuration files which are recognized by [Eclipse](http://www.eclipse.org/) if the Eclipse Maven plugin (m2e) is installed. - -At the command line, go to a directory **not** under your Eclipse workspace directory and check out the source: - -```shell -git clone https://github.com/OpenRefine/OpenRefine.git -``` - -In Eclipse, invoke the `Import...` command and select `Existing Maven Projects`. - -![Screenshot of Import a Maven project option](/img/eclipse-import-maven-project-1.png) - -Choose the root directory of your clone of the repository. You get to choose which modules of the project will be imported. You can safely leave out the `packaging` module which is only used to generate the Linux, Windows and MacOS distributions. - -Screenshot of Select maven projects to import - -To run and debug OpenRefine from Eclipse, you will need to add an execution configuration on the `server` sub-project. -Right click on the `server` subproject, click `Run as...` and `Run configurations...` and create a new `Maven Build` run configuration. Rename the run configuration `OpenRefine`. Enter the root directory of the project as `Base directory` and use `exec:java` as a Maven goal. - -![Screenshot of Add a run configuration with the exec:java goal](/img/eclipse-exec-config.png) - -This will add a run configuration that you can then use to run OpenRefine from Eclipse. - -## Code style in Eclipse - -You can apply the supplied Eclipse code style (in `IDEs/eclipse/Refine.style.xml`) to make sure Eclipse lints your files according to the existing style. -Pull requests deviating from this style will fail in the CI. - -You can manually apply the code style (regardless of your IDE) with the `mvn formatter:format` command. - -## Testing in Eclipse {#testing-in-eclipse} - -You can run the server tests directly from Eclipse. To do that you need to have the TestNG launcher plugin installed, as well as the TestNG M2E plugin (for integration with Maven). If you don't have it, you can get it by [installing new software](https://help.eclipse.org/2020-03/index.jsp?topic=/org.eclipse.platform.doc.user/tasks/tasks-129.htm) from this update URL http://dl.bintray.com/testng-team/testng-eclipse-release/ - -Once the TestNG launching plugin is installed in your Eclipse, right click on the source folder "main/tests/server/src", select `Run As` -> `TestNG Test`. This should open a new tab with the TestNG launcher running the OpenRefine tests. - -### Test coverage in Eclipse {#test-coverage-in-eclipse} - -It is possible to analyze test coverage in Eclipse with the `EclEmma Java Code Coverage` plugin. It will add a `Coverage as…` menu similar to the `Run as…` and `Debug as…` menus which will then display the covered and missed lines in the source editor. - -### Debug with Eclipse {#debug-with-eclipse} -Here's an example of putting configuration in Eclipse for debugging, like putting values for the Google Data extension. Other type of configurations that can be set are memory, Wikidata login information and more. - -![Screenshot of Eclipse debug configuration](/img/eclipse-debug-config.png) - -## Building, Testing and Running OpenRefine from IntelliJ idea {#building-testing-and-running-openrefine-from-intellij-idea} - -At the command line, go to a directory you want to save the OpenRefine project and execute the following command to clone the repository: - -```shell -git clone https://github.com/OpenRefine/OpenRefine.git -``` - -Then, open the IntelliJ idea and go to `file -> open` and select the location of the cloned repository. - -![Screenshot of Open option on the IntelliJ File menu](/img/intellij-setup-1.png) - -It will prompt you to add as a maven project as the source code contains a pom.xml file in it. Allow `auto-import` so that it can add it as a maven project. -If it doesn't prompt something like this then you can go on the right side of the IDE and click on maven then, click on `reimport all the maven projects` that will add all the dependencies and jar files required for the project. - -![Screenshot of Maven project controls in IntelliJ](/img/intellij-maven.png) - -After this, you will be able to properly build, test, and run the OpenRefine project from the terminal. -But if you will go to any of the test folders and open some file it will show you some import errors because the project isn't yet set up at the module level. - -For removing those errors, and enjoying the features of the IDE like ctrl + click, etc you need to set up the project at the module level too. Open the different modules like `extensions/wikidata`, `main` as a project in the IDE. Then, right-click on the project folder and open the module settings. - -![Screenshot of open module settings menu in IntelliJ](/img/intellij-open-module-settings.png) - -In the module settings, add the source folder and test source folders of that module. - -![Screenshot of module settings in IntelliJ](/img/intellij-module-settings.png) - -Then, do the same thing for the main OpenRefine project and now you are good to go. diff --git a/OpenRefine/docs/docs/technical-reference/contributing.md b/OpenRefine/docs/docs/technical-reference/contributing.md deleted file mode 100644 index a9151abf7..000000000 --- a/OpenRefine/docs/docs/technical-reference/contributing.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -id: contributing -title: Contributing -sidebar_label: Contributing ---- - -Please read the general [guidelines on contributing to OpenRefine](https://github.com/OpenRefine/OpenRefine/blob/master/CONTRIBUTING.md) first, then review the information on [reporting and tracking issues](#reporting-and-tracking-issues), and on making your [first pull request](#your-first-pull-request) below) - -## Reporting and tracking issues {#reporting-and-tracking-issues} - -If you need to file a bug or request a feature, [create an Issue in the OpenRefine Github repository](https://github.com/OpenRefine/OpenRefine/issues). Github issues should be used for reporting specific bugs and requesting specific features. If you just don't know how to do something using OpenRefine, or want to discuss some ideas, please: - -- [Try the user manual](/) -- [post to our OpenRefine mailing list](http://groups.google.com/group/openrefine/) - -## Contributing to the documentation {#contributing-to-the-documentation} - -We use [Docusaurus](https://docusaurus.io/) for our docs. For small documentation changes, you should be able to edit the Markdown files directly and submit them as a pull request. A preview of the docs will be generated automatically. But it is also -possible to preview your changes locally. Assuming you have [Node.js](https://nodejs.org/en/download/) installed (which includes npm), you can install Docusaurus with: - -You will need to install [Yarn](https://yarnpkg.com/getting-started/install) before you can build the site. -```sh -npm install -g yarn -``` - -Once you have installed yarn, navigate to docs directory & set-up the dependencies. - -```sh -cd docs -yarn -``` - -Once this is done, generate the docs with: - -```sh -yarn build -``` - -You can also spin a local web server to serve the docs for you, with auto-refresh when you edit the source files, with: -```sh -yarn start -``` - -## Your first code pull request {#your-first-code-pull-request} - -This describes the overall steps to your first code contribution in OpenRefine. If you have trouble with any of these steps feel free to reach out on the [developer mailing list](https://groups.google.com/forum/#!forum/openrefine-dev) or the [Gitter channel](https://gitter.im/OpenRefine/OpenRefine). - -- Install OpenRefine, learn to use it by following some tutorials or watching [some videos](http://openrefine.org/). That will ensure you understand the user workflows and get familiar with the terminology used in the tool. - -- Fork the GitHub repository, clone it on your machine and set up your IDE to work on it. We have [instructions for this](https://github.com/OpenRefine/OpenRefine/wiki/Building-OpenRefine-From-Source). - -- Browse through the list of issues to find an issue that you find interesting. You should pick one where you understand what the problem is as a user, you can see why fixing it would be an improvement to the tool. It is also a good idea to pick an issue that matches your technical skills: some require work on the backend (in Java) or in the frontend (Javascript), often both. We try to maintain a list of [good first issues](https://github.com/OpenRefine/OpenRefine/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) which should be easier than others and should not require any difficult design decision. - -- Reproduce the issue locally, by following the steps described in the issue. You might need to locate a particular dialog, use a specific importer on a sample file, or follow any other user workflow. If you have followed all the steps described in the issue and cannot observe the issue mentioned, write a comment on the issue explaining that you are not able to reproduce it (perhaps it was fixed by another change). - -- Locate the code that is relevant for the issue you want to solve. Text search across files is often useful for that. For instance, if the issue you want to solve is about a dialog entitled "Columnize by key/values", you can search for "Columnize" in the entire source code. For more details about this technique, see [this comment](https://github.com/OpenRefine/OpenRefine/issues/3137#issuecomment-691649962). - -- Study how the current code works. You might want to use a debugger to put breakpoints at the relevant locations (for inspecting the backend, use your IDE's debugger, for the frontend, use your browser's developer tools). - -- Create a git branch for your fix. The name of your branch should contain the issue number, and a few words to describe the topic of the fix, for instance "issue-1234-columnize-layout". - -- Make changes to the code to fix the issue. If you are changing backend code, it would be great if you could also write a test in Java to demonstrate the fix. You can imitate existing tests for that. We currently do not have frontend tests. - -- If you made Java changes, run linting to make sure they conform to our code style, with `mvn formatter:format`. - -- commit your changes, using a message that contains one of the special words "closes" and "fixes" which are detected by Github, followed by the issue number, e.g. "closes #1234" or "fixes #1234", this will link the commit to the issue you are working on. - -- push your branch to your fork and create a pull request for it, explaining the approach you have used, any design decisions you have made. - -Thank you! diff --git a/OpenRefine/docs/docs/technical-reference/development-roadmap.md b/OpenRefine/docs/docs/technical-reference/development-roadmap.md deleted file mode 100644 index da35f2f77..000000000 --- a/OpenRefine/docs/docs/technical-reference/development-roadmap.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -id: development-roadmap -title: Development roadmap -sidebar_label: Development roadmap ---- - -Please be aware that the OpenRefine roadmap is subject to change at any time, so please check back regularly, and monitor [milestones](https://github.com/OpenRefine/OpenRefine/milestones), [projects](https://github.com/OpenRefine/OpenRefine/projects) and [issues](https://github.com/OpenRefine/OpenRefine/issues) in Github to keep up to date with current plans. - -If there are features you would like to see that are not currently listed here or in current [milestones](https://github.com/OpenRefine/OpenRefine/milestones), [projects](https://github.com/OpenRefine/OpenRefine/projects) and [issues](https://github.com/OpenRefine/OpenRefine/issues), please add them to the [issue tracker](https://github.com/OpenRefine/OpenRefine/issues). - - -## Planned releases {#planned-releases} - -### 4.0 {#40} -[New backend storage option to allow using much bigger datasets at the expense of real-time feedback.](https://github.com/OpenRefine/OpenRefine/milestone/7) - -New UI (possibly Vue or React based) - -## Work in progress {#work-in-progress} -Alongside the planned releases there are often smaller pieces of work in progress. Check for [recently updated issues](https://github.com/OpenRefine/OpenRefine/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc) and [pull requests](https://github.com/OpenRefine/OpenRefine/pulls?q=is%3Apr+is%3Aopen+sort%3Aupdated-desc) to see what is currently in the works. - -## On the back burner {#on-the-back-burner} -Some aspects of OpenRefine have previously been targeted for release, but have not made it into a release and have not been worked on recently. If you would like to see features in these areas, please create an issue the describes what development you would like to see: - -- Streamlining traditional features -- Views: map, timeline, protovis (D3.js) charts -- Better machinery to guess and re-encode cell values (useful for fixing encoding issues) -- Collaborative editing support (see documentation on the '[broker protocol](https://github.com/OpenRefine/OpenRefine/wiki/Broker-Protocol)' to see where this work was going) -- Column groups diff --git a/OpenRefine/docs/docs/technical-reference/functional-tests.md b/OpenRefine/docs/docs/technical-reference/functional-tests.md deleted file mode 100644 index afdfbe43c..000000000 --- a/OpenRefine/docs/docs/technical-reference/functional-tests.md +++ /dev/null @@ -1,241 +0,0 @@ ---- -id: functional-tests -title: Functional tests -sidebar_label: Functional tests ---- - -import useBaseUrl from '@docusaurus/useBaseUrl'; - -## Introduction {#introduction} - -OpenRefine interface is tested with the [Cypress framework](https://www.cypress.io/). -With Cypress, tests are performing assertions using a real browser, the same way a real user would use the software. - -Cypress tests can be ran - -- using the Cypress test runner (development mode) -- using a command line (CI/CD mode) - -If you are writing tests, the Cypress test runner is good enough, and the command-line is mainly used by the CI/CD platform (Github actions) - -## Cypress brief overview {#cypress-brief-overview} - -Cypress operates insides a browser, it's internally using NodeJS. -That's a key difference with tools such as Selenium. - -**From the Cypress documentation:** - -> But what this also means is that your test code **is being evaluated inside the browser**. Test code is not evaluated in Node, or any other server side language. The **only** language we will ever support is the language of the web: JavaScript. - -Good starting points with Cypress are the [Getting started guide](https://docs.cypress.io/guides/getting-started/writing-your-first-test.html#Write-your-first-test), and the [Trade-offs](https://docs.cypress.io/guides/references/trade-offs.html#Permanent-trade-offs-1) - -The general workflow of a Cypress test is to - -- Start a browser (yarn run cypress open) -- Visit a URL -- Trigger user actions -- Assert that the DOM contains expected texts and elements using selectors - -## Getting started {#getting-started} - -If this is the first time you use Cypress, it is recommended for you to get familiar with the tool. - -- [Cypress overview](https://docs.cypress.io/guides/overview/why-cypress.html) -- [Cypress examples of tests and syntax](https://example.cypress.io/) - -### 1. Install Cypress {#1-install-cypress} - -You will need: - -- [Node.js 10 or 12 and above](https://nodejs.org) -- [Yarn or NPM](https://yarnpkg.com/) -- A Unix/Linux shell environment or the Windows command line - -To install Cypress and dependencies, run : - -```shell -cd ./main/tests/cypress -yarn install -``` - -### 2. Start the test runner {#2-start-the-test-runner} - -The test runner assumes that OpenRefine is up and running on the local machine, the tests themselves do not launch OpenRefine, nor restarts it. - -Start OpenRefine with - -```shell -./refine -``` - -Then start Cypress - -```shell -yarn --cwd ./main/tests/cypress run cypress open -``` - -### 3. Run the existing tests {#3-run-the-existing-tests} - -Once the test runner is up, you can choose to run one or several tests by selecting them from the interface. -Click on one of them and the test will start. - -### 4. Add your first test {#4-add-your-first-test} - -- Add a `test.spec.js` into the `main/tests/cypress/cypress/integration` folder. -- The test is instantly available in the list -- Click on the test -- Start to add some code - -## Tests technical documentation {#tests-technical-documentation} - -### A typical test {#a-typical-test} - -A typical OpenRefine test starts with the following code - -```javascript -it('Ensure cells are blanked down', function () { - cy.loadAndVisitProject('food.mini'); - cy.get('.viewpanel-sorting a').contains('Sort').click(); - cy.get('.viewpanel').should('to.contain', 'Something'); -}); -``` - -The first noticeable thing about a test is the description (`Ensure cells are blanked down`), which describes what the test is doing. -Lines usually starts with `cy.something...`, which is the main way to interact with the Cypress framework. - -A few examples: - -- `cy.get('a.my-class')` will retrieve the `` element -- `cy.click()` will click on the element -- eventually, `cy.should()` will perform an assertion, for example that the element contains an expected text with `cy.should('to.contains', 'my text')` - -On top of that, OpenRefine contributors have added some functions for common OpenRefine interactions. -For example - -- `cy.loadAndVisitProject` will create a fresh project in OpenRefine -- `cy.assertCellEquals` will ensure that a cell contains a given value - -See below on the dedicated section 'Testing utilities' - -### Testing guidelines {#testing-guidelines} - -- `cy.wait` should be used in the last resort scenario. It's considered a bad practice, though sometimes there is no other choice -- Tests should remain isolated from each other. It's best to try one feature at the time -- A test should always start with a fresh project -- The name of the files should mirror the OpenRefine UI organization - -### Testing utilities {#testing-utilities} - -OpenRefine contributors have added some utility methods on the top of the Cypress framework. -Those methods perform some common actions or assertions on OpenRefine, to avoid code duplication. - -Utilities can be found in `cypress/support/commands.js`. - -The most important utility method is `loadAndVisitProject`. -This method will create a fresh OpenRefine project based on a dataset given as a parameter. -The fixture parameter can be - -- An arbitrary array, the first row is for the column names, other rows are for the values - Use an arbitrary array **only** if the test requires some specific grid values - **Example:** - - ```javascript - const fixture = [ - ['Column A', 'Column B', 'Column C'], - ['0A', '0B', '0C'], - ['1A', '1B', '1C'], - ['2A', '2B', '2C'], - ]; - cy.loadAndVisitProject(fixture); - ``` - -- A referenced dataset: `food.small` or `food.mini` - Most of the time, tests does not require any specific grid values - Use food.mini as much as possible, it loads 2 rows and very few columns in the grid - Use food.small if the test requires a few hundred rows in the grid - - Those datasets live in `cypress/fixtures` - -### Browsers {#browsers} - -In terms of browsers, Cypress is using what is installed on your operating system. -See the [Cypress documentation](https://docs.cypress.io/guides/guides/launching-browsers.html#Browsers) for a list of supported browsers - -### Folder organization {#folder-organization} - -Tests are located in `main/tests/cypress/cypress` folder. -The test should not use any file outside the cypress folder. - -- `/fixtures` contains CSVs and OpenRefine project files used by the tests -- `/integration` contains the tests -- `/plugins` contains custom plugins for the OR project -- `/screenshots` and `/videos` contains the recording of the tests, Git ignored -- `/support` is a custom library of assertion and common user actions, to avoid code duplication in the tests themselves - -### Configuration {#configuration} - -Cypress execution can be configured with environment variables, they can be declared at the OS level, or when running the test - -Available variables are - -- OPENREFINE_URL, determine on which scheme://url:port to access OpenRefine, default to http://localhost:333 -- DISABLE_PROJECT_CLEANUP, If set to 1, projects will not be deleted after each run. Default to 0 to keep the OpenRefine instance clean - - -Cypress contains [exaustive documentation](https://docs.cypress.io/guides/guides/environment-variables.html#Setting) about configuration, but here are two simple ways to configure the execution of the tests: - -#### Overriding with a cypress.env.json file {#overriding-with-a-cypressenvjson-file} - -This file is ignored by Git, and you can use it to configure Cypress locally - -#### Command-line {#command-line} - -You can pass variables at the command-line level - -```shell -yarn --cwd ./main/tests/cypress run cypress open --env OPENREFINE_URL="http://localhost:1234" -``` - -### Visual testing {#visual-testing} - -Tests generally ensure application behavior by making assertions against the DOM, to ensure specific texts or css attributes are present in the document body. -Visual testing, on the contrary, is a way to test applications by comparing images. -A reference screenshot is taken the first time the test runs, and subsequent runs will compare a new screenshot against the reference, at the pixel level. - -Here is an [introduction to visual testing by Cypress](https://docs.cypress.io/plugins/directory#visual-testing). - -In some cases, we are using visual testing. -We are using [Cypress Image Snapshot](https://github.com/jaredpalmer/cypress-image-snapshot) - -Identified cases are so far: - -- testing images created by OpenRefine backend (scatterplots for example) - -Reference screenshots (Called snapshots), are stored in /cypress/snapshots. -And a snapshot can be taken for the whole page, or just a single part of the page. - -#### When a visual test fails {#when-a-visual-test-fails} - -First, Cypress will display the following error message: - -![Diff image when a visual test fails](/img/visual-test-cypress-failure.png) - -Then, a diff image will be created in /cypress/snapshots, this directory is ignored by Git. -The diff images shows the reference image on the left, the image that was taken during the test run on the right, and the diff in the middle. - -![Diff image when a visual test fails](/img/failed-visual-test.png) - -## CI/CD {#cicd} - -In CI/CD, tests are run headless, with the following command-line - -```shell -./refine ui_test chrome -``` - -Results are displayed in the standard output - -## Resources {#resources} - -[Cypress command line options](https://docs.cypress.io/guides/guides/command-line.html#Installation) -[Lots of good Cypress examples](https://example.cypress.io/) diff --git a/OpenRefine/docs/docs/technical-reference/homebrew-cask-process.md b/OpenRefine/docs/docs/technical-reference/homebrew-cask-process.md deleted file mode 100644 index 650af5ccf..000000000 --- a/OpenRefine/docs/docs/technical-reference/homebrew-cask-process.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -id: homebrew-cask-process -title: Maintaining OpenRefine's Homebrew cask -sidebar_label: Maintaining OpenRefine's Homebrew cask ---- diff --git a/OpenRefine/docs/docs/technical-reference/maintainer-guidelines.md b/OpenRefine/docs/docs/technical-reference/maintainer-guidelines.md deleted file mode 100644 index b4ca7b20f..000000000 --- a/OpenRefine/docs/docs/technical-reference/maintainer-guidelines.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -id: maintainer-guidelines -title: Guidelines for maintaining OpenRefine -sidebar_label: Maintainer guidelines ---- - -This page describes our practices to review issues and pull requests in the OpenRefine project. - -## Reviewing issues {#reviewing-issues} - -When people create new issues, they automatically get assigned [the "to be reviewed" tag](https://github.com/OpenRefine/OpenRefine/issues?q=is%3Aissue+is%3Aopen+label%3A%22to+be+reviewed%22). - -Ideally, for each of these issues, someone familiar with OpenRefine (not necessarily a developer!) should read the issue and try to determine if there is a genuine bug to fix, or if the enhancement request is legitimate. In those cases, we can remove the "to be reviewed" tag and leave the issue open. In the others, the issue should be politely closed. - -### Bugs {#bugs} - -For a bug, we should first check if it is a real unexpected behaviour or if just comes from a misunderstanding of the intended behaviour of the tool (which could suggest an improvement to the documentation). Then, if it sounds like a genuine problem, we need to check if it can be reproduced independently on the master branch. If the issue does not give enough details about the bug to reproduce it on master, mark it as "not reproducible" and ask the reporter for more information. After some time without any information from the reporter, we can close the issue. - -### Enhancement requests {#enhancement-requests} - -For an enhancement, we need to make a judgment call of whether the proposed functionality is in the scope of the project. There is no universal rule for this of course, so just use your own intuition: do you think this would improve the tool? Would it be consistent with the spirit of the project? Trust your own opinion - if people disagree, they can have a discussion on the issue. - -### Tagging good first issues {#tagging-good-first-issues} - -Adding [the "good first issue" tag](https://github.com/OpenRefine/OpenRefine/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) is something that requires a bit more familiarity with the development process. This tag is used by GitHub to showcase issues in some project lists and we point interested potential contributors to it. It is therefore important that tackling these issues gives them a nice onboarding experience, with as few hurdles as possible. - -Develepers should add the "good first issue" tag when they are confident that they can provide a good pull request for the issue with at most a few hours of work. Also, solving the issue should not require any difficult design decision. The issue should be uncontentious: it should be clear that the proposed solution should be accepted by the team. - -## Reviewing pull requests {#reviewing-pull-requests} - -### Process {#process} - -1. A committer reviews the PR to check for the requirements below and tests it. Each PR should be linked to one or more corresponding issues and the reviewer should check that those are correctly addressed by the PR. The reviewer should be someone else than the PR author. For PRs with an important impact or contentious issues, it is important to leave enough time for other contributors to give their opinion. - -2. The reviewer merges the pull request by squashing its commits into one (except for Weblate PRs which should be merged without squashing). - -3. The reviewer adds the linked issues to the milestone for the next release (such as [the 3.5 milestone](https://github.com/OpenRefine/OpenRefine/milestone/17)) - -4. If the change is worth noting for users or developers, the reviewer adds an entry in the changelog for the next release (such as [Changes for 3.5](https://github.com/OpenRefine/OpenRefine/wiki/Changes-for-3.5)) - -### Requirements {#requirements} - -#### Code style {#code-style} - -Currently, only our code style for integration tests (using Cypress) is codified and enforced by the CI. -For the rest, we rely on imitating the surrounding code. [We should decide on a code style and check it in the CI for other areas of the tool](https://github.com/OpenRefine/OpenRefine/issues/2338). - -#### Testing {#testing} - -We currently rely have two sorts of tests: -* Backend tests, in Java, written with the TestNG framework. Their granularity varies, but generally speaking they are unit tests which test components in isolation. -* UI tests, in Javascript, written with the Cypress framework. They are integration tests which test both the frontend and the backend at the same time. - -Changes to the backend should generally come with the accompanying TestNG tests. -Functional changes to the UI should ideally come with corresponding Cypress tests as well. - -Those tests should be supplied in the same PR as the one that touches the product code. - -#### Documentation {#documentation} - -Changes to user-facing functionality should be reflected in the docs. Those documentation changes should happen in the same PR as the one that touches the product code. - -#### UI style {#ui-style} - -We do not have formally defined UI style guidelines. Contributors are invited to imitate the existing style. - -#### Licensing and dependencies {#licensing-and-dependencies} - -Dependencies can only be added if they are released under a license that is compatible with our BSD Clause-3 license. -One should pay attention to the size of the dependencies since they inflate the size of the release bundles. - -#### Continuous integration {#continuous-integration} - -The various check statuses reported by our continuous integration suite should be green. - -### Special pull requests {#special-pull-requests} - -#### Weblate PRs {#weblate-prs} - -Weblate PRs should not be squashed as it prevents Weblate from recognizing that the corresponding changes have been made in master. They should be merged without squashing. - -Reviewing Weblate PRs only amonuts to a quick visual sanity check as maintainers are not expected to master the languages involved. If corrections need to be made, they should be done in Weblate itself. - -#### Dependabot PRs {#dependabot-prs} - -When reviewing a Dependabot PR it is generally useful to pay attention to: -* the type of version change: most libraries follow the "semver" versioning convention, which indicates the nature of the change. -* the library's changelog, especially if the version change is more significant than a patch release - diff --git a/OpenRefine/docs/docs/technical-reference/migrating-older-extensions.md b/OpenRefine/docs/docs/technical-reference/migrating-older-extensions.md deleted file mode 100644 index 75f40015e..000000000 --- a/OpenRefine/docs/docs/technical-reference/migrating-older-extensions.md +++ /dev/null @@ -1,169 +0,0 @@ ---- -id: migrating-older-extensions -title: Migrating older Extensions -sidebar_label: Migrating older Extensions ---- - -## Migrating from Ant to Maven {#migrating-from-ant-to-maven} - -### Why are we doing this change? {#why-are-we-doing-this-change} - -Ant is a fairly old (antique?) build system that does not incorporate any dependency management. -By migrating to Maven we are making it easier for developers to extend OpenRefine with new libraries, and stop having to ship dozens of .jar files in the repository. Using the Maven repository also encourages developers to add dependencies to released versions of libraries instead of custom snapshots that are hard to update. - -### When was this change made? {#when-was-this-change-made} - -The migration was done between 3.0 and 3.1-beta with this commit: -https://github.com/OpenRefine/OpenRefine/commit/47323a9e750a3bc9d43af606006b5eb20ca397b8 - -### How to migrate an extension {#how-to-migrate-an-extension} - -You will need to write a `pom.xml` in the root folder of your extension to configure the compilation process with Maven. Sample `pom.xml` files for extensions can be found in the extensions that are shipped with OpenRefine (`gdata`, `database`, `jython`, `pc-axis` and `wikidata`). A sample extension (`sample`) is also provided, with a minimal build file. - -For any library that your extension depends on, you should try to find a matching artifact in the Maven Central repository. If you can find such an artifact, delete the `.jar` file from your extension and add the dependency in your `pom.xml` file. If you cannot find such an artifact, it is still possible to incorporate your own `.jar` file using `maven-install-plugin` that you can configure in your `pom.xml` file as follows: - - - - org.apache.maven.plugins - maven-install-plugin - 2.5.2 - - - install-wdtk-datamodel - process-resources - - ${basedir}/lib/my-proprietary-library.jar - default - com.my.company - my-library - 0.5.3-SNAPSHOT - jar - true - - - install-file - - - - - - -And add the dependency to the `` section as usual: - - - com.my.company - my-library - 0.5.3-SNAPSHOT - - -## Migrating to Wikimedia's i18n jQuery plugin {#migrating-to-wikimedias-i18n-jquery-plugin} - -### Why are we doing this change? {#why-are-we-doing-this-change-1} - -This adds various important localization features, such as the ability to handle plurals or interpolation. This also restores the language fallback (displaying strings in English if they are not available in the target language) which did not work with the previous set up. - -### When was the migration made? {#when-was-the-migration-made} - -The migration was made between 3.1-beta and 3.1, with this commit: https://github.com/OpenRefine/OpenRefine/commit/22322bd0272e99869ab8381b1f28696cc7a26721 - -### How to migrate an extension {#how-to-migrate-an-extension-1} - -You will need to update your translation files, merging nested objets in one global object, concatenating keys. You can do this by running the following Python script on all your JSON translation files: - - import json - import sys - - with open(sys.argv[1], 'r') as f: - j = json.loads(f.read()) - - result = {} - def translate(obj, path): - res = {} - if type(obj) == str: - result['/'.join(path)] = obj - else: - for k, v in obj.items(): - new_path = path + [k] - translate(v, new_path) - - translate(j, []) - - with open(sys.argv[1], 'w') as f: - f.write(json.dumps(result, ensure_ascii=False, indent=4)) - -Then your javascript files which retrieve the translated strings should be updated: `$.i18n._('core-dialogs')['cancel']` becomes `$.i18n('core-dialogs/cancel')`. You can do this with the following `sed` script: - - sed -i "s/\$\.i18n._(['\"]\([A-Za-z0-9/_\\-]*\)['\"])\[['\"]\([A-Za-z0-9\-\_]*\)[\"']\]/$.i18n('\1\/\2')/g" my_javascript_file.js - -You can then chase down the places where you are concatenating translated strings, and replace that with more flexible patterns using [the plugin's features](https://github.com/wikimedia/jquery.i18n#jqueryi18n-plugin). - -## Migrating from org.json to Jackson {#migrating-from-orgjson-to-jackson} - -### Why are we doing this change? {#why-are-we-doing-this-change-2} - -The org.json (or json-java) library has multiple drawbacks. -* First, it has limited functionality - all the serialization and deserialization has to be done explicitly - an important proportion of OpenRefine's code was dedicated to implementing these; -* Second, its implementation is not optimized for speed - multiple projects have reported speedups when migrating to more modern JSON libraries; -* Third, and this was the decisive factor to initiate the migration: [its license](https://json.org/license) is the MIT license with an additional condition which makes it non-free. Getting rid of this dependency was required by the Software Freedom Conservancy as a prerequisite to become a fiscal sponsor for the project. - -### When was the migration made? {#when-was-the-migration-made-1} - -This change was made between 3.1 and 3.2-beta, with this commit: https://github.com/OpenRefine/OpenRefine/commit/5639f1b2f17303b03026629d763dcb6fef98550b - -### How to migrate an extension or fork {#how-to-migrate-an-extension-or-fork} - -You will need to use the Jackson library to serialize the classes that implement interfaces or extend classes exposed by OpenRefine. -The interface `Jsonizable` was deleted. Any class that used to implement this now needs to be serializable by Jackson, producing the same format as the previous serialization code. This applies to any operation, facet, overlay model or GREL function. If you are new to Jackson, have a look at [this tutorial](https://www.baeldung.com/jackson) to learn how to annotate your class for serialization. Once this is done, you can remove the `void write(JSONWriter writer, Properties options)` method from your class. Note that it is important that you do this migration for all classes implementing the `Jsonizable` interface that are exposed to OpenRefine's core. - -We encourage you to migrate out of org.json completely, but this is only required for the classes that interact with OpenRefine's core. - -#### General notes about migrating {#general-notes-about-migrating} - -OpenRefine's ObjectMapper is available at `ParsingUtilities.mapper`. It is configured to only serialize the fields and getters that have been explicitly marked with `@JsonProperty` (to avoid accidental JSON format changes due to refactoring). On deserialization it will ignore any field in the JSON payload that does not correspond to a field in the Java class. It has serializers and deserializers for `OffsetDateTime` and `LocalDateTime`. - -Useful snippets to use in tests: -* deserialize an instance: `MyClass instance = ParsingUtilities.mapper.readValue(jsonString, MyClass.class);` (replaces calls to `Jsonizable.write`); -* serialize an instance: `String json = ParsingUtilities.mapper.writeValueAsString(myInstance);` (replaces calls to static methods such as `load`, `loadStreaming` or `reconstruct`); -* the equivalent of `JSONObject` is `ObjectNode`, the equivalent of `JSONArray` is `ArrayNode`; -* create an empty JSON object: `ParsingUtilities.mapper.createObjectNode()` (replaces `new JSONObject()`); -* create an empty JSON array: `ParsingUtilities.mapper.createArrayNode()` (replaces `new JSONArray()`). - -Before undertaking the migration, we recommend that you write some tests which serialize and deserialize your objects. This will help you make sure that the JSON format is preserved during the migration. One way to do this is to collect some sample JSON representations of your objects, and check in your tests that deserializing these JSON payloads and serializing them back to JSON preserves the JSON payload. Some utilities are available to help you with that in [`TestUtils`](https://github.com/OpenRefine/OpenRefine/blob/master/main/tests/server/src/com/google/refine/tests/util/TestUtils.java) (we had [some to test org.json serialization](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/tests/server/src/com/google/refine/tests/util/TestUtils.java) before we got rid of the dependency, feel free to copy them). - -#### For functions {#for-functions} - -Before the migration, you had to explicitly define JSON serialization of functions with a `write` method. You should now override the getters returning the various documentation fields. - -Example: `Cos` function [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/expr/functions/math/Cos.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/expr/functions/math/Cos.java). - -#### For operations {#for-operations} - -Before the JSON migration we refactored engine-dependent operations so that the engine configuration is represented by an `EngineConfig` object instead of a `JSONObject`. Therefore the constructor for your operation should be updated to use this new class. Your constructor should also be annotated to be used during deserialization. - -Note that you do not need to explicitly serialize the operation type, this is already done for you by `AbstractOperation`. - -Example: `ColumnRemovalOperation` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java). - -#### For changes {#for-changes} - -Changes are serialized in plain text but often relies on JSON serialization for parts of the data. Just use the methods above with `ParsingUtilities.mapper` to maintain this behaviour. - -Example: `ReconChange` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/model/changes/ReconChange.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java). - -#### For importers {#for-importers} - -The importing options have been migrated from `JSONObject` to `ObjectNode`. Your compiler should help you propagate this change. Utility functions in `JSONUtilities` have been migrated to Jackson so you should have minimal changes if you used them. - -Example: `TabularImportingParserBase` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/importers/TabularImportingParserBase.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/importers/TabularImportingParserBase.java). - -#### For overlay models {#for-overlay-models} - -Migrate serialization and deserialization as for other objects. - -Example: `WikibaseSchema` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java#L203) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java#L60) - -#### For preference values {#for-preference-values} - -Any class that is stored in OpenRefine's preference now needs to implement the `com.google.refine.preferences.PreferenceValue` interface. The static `load` method and the `write` method used previously for deserialization should be deleted and regular Jackson serialization and deserialization should be implemented instead. Note that you do not need to explicitly serialize the class name, this is already done for you by the interface. - -Example: `TopList` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/preference/TopList.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/preference/TopList.java) diff --git a/OpenRefine/docs/docs/technical-reference/openrefine-api.md b/OpenRefine/docs/docs/technical-reference/openrefine-api.md deleted file mode 100644 index 7d1a4c402..000000000 --- a/OpenRefine/docs/docs/technical-reference/openrefine-api.md +++ /dev/null @@ -1,285 +0,0 @@ ---- -id: openrefine-api -title: OpenRefine API -sidebar_label: OpenRefine API ---- - -This is a generic API reference for interacting with OpenRefine's HTTP API. - -**NOTE:** This protocol is subject to change without warning at any time (and has in the past) and is not versioned. Use at your own risk! - -For OpenRefine 3.3 and later, all POST requests need to include a CSRF token as described here: https://github.com/OpenRefine/OpenRefine/wiki/Changes-for-3.3#csrf-protection-changes - -## Create project: {#create-project} - -> **Command:** _POST /command/core/create-project-from-upload_ - -When uploading files you will need to send the data as `multipart/form-data`. This is different to all other API calls which use a mixture of query string and POST parameters. - -multipart form-data: - - 'project-file' : file contents - 'project-name' : project name - 'format' : format of data in project-file (e.g. 'text/line-based/*sv') [optional] - 'options' : json object containing options relevant to the file format [optional - however, some importers may have required options, such as `recordPath` for the JSON & XML importers]. - -The formats supported will depend on the version of OpenRefine you are using and any Extensions you have installed. The common formats include: - -* 'text/line-based': Line-based text files -* 'text/line-based/*sv': CSV / TSV / separator-based files [separator to be used in specified in the json submitted to the options parameter] -* 'text/line-based/fixed-width': Fixed-width field text files -* 'binary/text/xml/xls/xlsx': Excel files -* 'text/json': JSON files -* 'text/xml': XML files - -If the format is omitted OpenRefine will try to guess the format based on the file extension and MIME type. -The values which can be specified in the JSON object submitted to the 'options' parameter will vary depending on the format being imported. If not specified the options will either be guessed at by OpenRefine (e.g. separator being used in a separated values file) or a default value used. The import options for each file format are not currently documented, but can be seen in the OpenRefine GUI interface when importing a file of the relevant format. - -If the project creation is successful, you will be redirected to a URL of the form: - - http://127.0.0.1:3333/project?project= - -From the project parameter you can extract the project id for use in future API calls. The content of the response is the HTML for the OpenRefine interface for viewing the project. - -### Get project models: {#get-project-models} - -> **Command:** _GET /command/core/get-models?_ - - 'project' : project id - -Recovers the models for the specific project. This includes columns, records, overlay models, scripting. In the columnModel a list of the columns is displayed, key index and name, and column groupings. - -### Response: {#response} -**On success:** -```JSON -{ - "columnModel":{ - "columns":[ - { - "cellIndex":0, - "originalName":"email", - "name":"email" - }, - { - "cellIndex":1, - "originalName":"name", - "name":"name" - }, - { - "cellIndex":2, - "originalName":"state", - "name":"state" - }, - { - "cellIndex":3, - "originalName":"gender", - "name":"gender" - }, - { - "cellIndex":4, - "originalName":"purchase", - "name":"purchase" - } - ], - "keyCellIndex":0, - "keyColumnName":"email", - "columnGroups":[ - - ] - }, - "recordModel":{ - "hasRecords":false - }, - "overlayModels":{ - - }, - "scripting":{ - "grel":{ - "name":"General Refine Expression Language (GREL)", - "defaultExpression":"value" - }, - "jython":{ - "name":"Python / Jython", - "defaultExpression":"return value" - }, - "clojure":{ - "name":"Clojure", - "defaultExpression":"value" - } - } -} -``` - - - -## Apply operations {#apply-operations} - -> **Command:** _POST /command/core/apply-operations?_ - -In the parameter - - 'project' : project id - -In the form data - - 'operations' : Valid JSON **Array** of OpenRefine operations - -Example of a Valid JSON **Array** -```JSON -'[ - { - "op":"core/column-addition", - "description":"Create column zip type at index 15 based on column Zip Code 2 using expression grel:value.type()", - "engineConfig":{ - "mode":"row-based", - "facets":[] - }, - "newColumnName":"zip type", - "columnInsertIndex":15, - "baseColumnName":"Zip Code 2", - "expression":"grel:value.type()", - "onError":"set-to-blank" - }, - { - "op":"core/column-addition", - "description":"Create column testing at index 15 based on column Zip Code 2 using expression grel:value.toString()0,5]", - "engineConfig":{ - "mode":"row-based", - "facets":[] - }, - "newColumnName":"testing", - "columnInsertIndex":15, - "baseColumnName":"Zip Code 2", - "expression":"grel:value.toString()[0,5]", - "onError":"set-to-blank" - } -] -``` - -On success returns JSON response -`{ "code" : "ok" }` - -## Export rows {#export-rows} - -> **Command:** _POST /command/core/export-rows_ - -In the parameter - - 'project' : project id - 'format' : format... (e.g 'tsv', 'csv') - -In the form data - - 'engine' : JSON string... (e.g. '{"facets":[],"mode":"row-based"}') - -Returns exported row data in the specified format. The formats supported will depend on the version of OpenRefine you are using and any Extensions you have installed. The common formats include: - -* csv -* tsv -* xls -* xlsx -* ods -* html - -## Delete project {#delete-project} - -> **Command:** _POST /command/core/delete-project_ - - 'project' : project id... - -Returns JSON response - -## Check status of async processes {#check-status-of-async-processes} - -> **Command:** _GET /command/core/get-processes_ - - 'project' : project id... - -Returns JSON response - -## Get all projects metadata: {#get-all-projects-metadata} - -> **Command:** _GET /command/core/get-all-project-metadata_ - -Recovers the meta data for all projects. This includes the project's id, name, time of creation and last time of modification. - -### Response: {#response-1} -```json -{ - "projects":{ - "[project_id]":{ - "name":"[project_name]", - "created":"[project_creation_time]", - "modified":"[project_modification_time]" - }, - ...[More projects]... - } -} -``` - -## Expression Preview {#expression-preview} -> **Command:** _POST /command/core/preview-expression_ - -Pass some expression (GREL or otherwise) to the server where it will be executed on selected columns and the result returned. - -### Parameters: {#parameters} -* **cellIndex**: _[column]_ -The cell/column you wish to execute the expression on. -* **rowIndices**: _[rows]_ -The rows to execute the expression on as JSON array. Example: `[0,1]` -* **expression**: _[language]_:_[expression]_ -The expression to execute. The language can either be grel, jython or clojure. Example: grel:value.toLowercase() -* **project**: _[project_id]_ -The project id to execute the expression on. -* **repeat**: _[repeat]_ -A boolean value (true/false) indicating whether or not this command should be repeated multiple times. A repeated command will be executed until the result of the current iteration equals the result of the previous iteration. -* **repeatCount**: _[repeatCount]_ -The maximum amount of times a command will be repeated. - -### Response: {#response-2} -**On success:** -```json -{ - "code": "ok", - "results" : [result_array] -} -``` - -The result array will hold up to ten results, depending on how many rows there are in the project that was specified by the [project_id] parameter. Each result is the string that would be put in the cell if the GREL command was executed on that cell. Note that any expression that would return an array or JSon object will be jsonized, although the output can differ slightly from the jsonize() function. - -**On error:** -```JSON -{ - "code": "error", - "type": "[error_type]", - "message": "[error message]" -} -``` - -## Third-party software libraries {#third-party-software-libraries} -Libraries using the [OpenRefine API](openrefine-api): - -### Python {#python} -* [refine-client-py](https://github.com/PaulMakepeace/refine-client-py/) - * Or this fork of the above with an extended CLI [openrefine-client](https://github.com/felixlohmeier/openrefine-client) -* [refine-python](https://github.com/maxogden/refine-python) - -### Ruby {#ruby} -* [refine-ruby](https://github.com/distillytics/refine-ruby) - * The above is a maintained fork of [google-refine](https://github.com/maxogden/refine-ruby) -* [google_refine](https://github.com/chengguangnan/google_refine) - -### NodeJS {#nodejs} -* [node-openrefine](https://github.com/pm5/node-openrefine) - -### R {#r} -* [rrefine](https://cran.r-project.org/web/packages/rrefine/index.html) - -### PHP {#php} -* [openrefine-php-client](https://github.com/keboola/openrefine-php-client) - -### Java {#java} -* [refine-java](https://github.com/dtap-gmbh/refine-java) - -### Bash {#bash} -* [bash-refine.sh](https://gist.github.com/felixlohmeier/d76bd27fbc4b8ab6d683822cdf61f81d) (templates for shell scripts) diff --git a/OpenRefine/docs/docs/technical-reference/reconciliation-api.md b/OpenRefine/docs/docs/technical-reference/reconciliation-api.md deleted file mode 100644 index 7649a6ebb..000000000 --- a/OpenRefine/docs/docs/technical-reference/reconciliation-api.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -id: reconciliation-api -title: Reconciliation API -sidebar_label: Reconciliation API ---- - -_This is a technical description of the mechanisms behind the reconciliation system in OpenRefine. For usage instructions, see [Reconciliation](/manual/reconciling)._ - -A reconciliation service is a web service that, given some text which is a name or label for something, and optionally some additional details, returns a ranked list of potential entities matching the criteria. The candidate text does not have to match each entity's official name perfectly, and that's the whole point of reconciliation--to get from ambiguous text name to precisely identified entities. For instance, given the text "apple", a reconciliation service probably should return the fruit apple, the Apple Inc. company, and New York city (also known as the Big Apple). - -Entities are identified by strong identifiers in some particular identifier space. In the same identifier space, identifiers follow the same syntax. For example, given the string "apple", a reconciliation service might return entities identified by the strings " [Q89](https://www.wikidata.org/wiki/Q89)", "[Q312](https://www.wikidata.org/wiki/Q312)", and "[Q60](https://www.wikidata.org/wiki/Q60)", in the Wikidata ID space. Each reconciliation service can only reconcile to one single identifier space, but several reconciliation services can reconcile to the same identifier space. - -OpenRefine can connect to any reconciliation service which follows the [reconciliation API v0.1](https://reconciliation-api.github.io/specs/0.1/). This was formerly a specification edited by the OpenRefine project, which has now transitioned to its own -[W3C Entity Reconciliation Community Group](https://www.w3.org/community/reconciliation/). - -Informally, the main function of any reconciliation service is to find good candidates in the underlying database, given the following data: - -* A string, which is normally the name or title of the entity, in some language. -* Optionally, a type which can be used to narrow down the search to entities of this type. OpenRefine does not define a particular set of acceptable types: this choice is left to the reconciliation service (see the suggest API for that). -* Optionally, a list of properties and their values, which can be used to refine the search. For instance, when reconciling a database of books, the author name or the publication date are useful bits of information that can be transferred to the reconciliation service. This information will be sent to the reconciliation service if the user binds columns to properties. Again, the notion of property is not predefined in OpenRefine: its definition depends on the reconciliation service. - -See [the specifications of the protocol](https://reconciliation-api.github.io/specs/0.1) for more details about the protocol. You can suggest changes on its [issues tracker](https://github.com/reconciliation-api/specs/issues) or on the [group mailing -list](https://lists.w3.org/Archives/Public/public-reconciliation/). - diff --git a/OpenRefine/docs/docs/technical-reference/technical-reference-index.md b/OpenRefine/docs/docs/technical-reference/technical-reference-index.md deleted file mode 100644 index 170b60958..000000000 --- a/OpenRefine/docs/docs/technical-reference/technical-reference-index.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -id: technical-reference-index -title: OpenRefine technical reference -sidebar_label: Technical Reference Index ---- - -Technical reference index diff --git a/OpenRefine/docs/docs/technical-reference/translating-docs.md b/OpenRefine/docs/docs/technical-reference/translating-docs.md deleted file mode 100644 index eb8482e5f..000000000 --- a/OpenRefine/docs/docs/technical-reference/translating-docs.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -id: translating-docs -title: Translate OpenRefine's documentation -sidebar_label: Translate OpenRefine's documentation ---- - -Our user manual can be translated using [Crowdin](https://crowdin.com/project/openrefine). - -## Getting access to Crowdin - -You need to request access to Crowdin on [our developers mailing list](https://groups.google.com/forum/#!forum/openrefine-dev). You will then be granted translator access on the platform. - -## Publication of translations - -Only the user manual for the current development version is being translated. When we publish a new stable version, we take a snapshot of this and publish it on the website. - -In the meantime the pages you translate will be visible under https://docs.openrefine.org/next/ after a few days. - - diff --git a/OpenRefine/docs/docs/technical-reference/translating-ui.md b/OpenRefine/docs/docs/technical-reference/translating-ui.md deleted file mode 100644 index 2ca3a811e..000000000 --- a/OpenRefine/docs/docs/technical-reference/translating-ui.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -id: translating-ui -title: Translate OpenRefine's interface -sidebar_label: Translate OpenRefine's interface ---- - -Currently supported languages include English, Spanish, Chinese, French, Hebrew, Italian and Japanese. - -![Translation status](https://hosted.weblate.org/widgets/openrefine/-/287x66-grey.png) - -You can help translate OpenRefine into your language by visiting [Weblate](https://hosted.weblate.org/engage/openrefine/?utm_source=widget) which provides a web based UI to edit and add translations and sends automatic pull requests back to our project. - -Click to help translate --> [Weblate](https://hosted.weblate.org/engage/openrefine/?utm_source=widget) - -## Manual translation process {#manual-translation-process} - -Localized strings are entered in a .json file, one per language. They are located in the folder `main/webapp/modules/core/langs/` in a file named `translation-xx`.json, where xx is the language code (i.e. fr for French). - -### Simple case of localized string {#simple-case-of-localized-string} -This is an example of a simple string, with the start of the JSON file. This example is for French. -``` -{ - "name": "Français", - "core-index/help": "Aide", - (… more lines) -} -``` - -So the key `core-index/help` will render as `"Aide"` in French. - -### Localization with a parameterized value {#localization-with-a-parameterized-value} -In this example, the name of the column (represented by `$1` in this example), will be substituted with the string of the name of the column. - -`"core-facets/edit-facet-title": "Cliquez ici pour éditer le nom de la facette\nColonne : $1",` - -### Localization with a singular/plural value {#localization-with-a-singularplural-value} -In this example, one of the parameter will have a different string depending if the value is 1 or another value. -In this example, the string for page, the second parameter, `$2`, will have an « s » or not depending on the value of `$2`. - -`"core-views/goto-page": "$1 de $2 {{plural:$2|page|pages}}"` - -## Front-end development {#front-end-development} - -The OpenRefine front end has been localized using the [Wikidata jquery.i18n library](https://github.com/OpenRefine/OpenRefine/pull/1285. The localized text is stored in a JSON dictionary on the server and retrieved with a new OpenRefine command. - -### Adding a new string {#adding-a-new-string} - -There should be no hard-coded language strings in the HTML or JSON used for the front end. If you need a new string, first check the existing strings to make sure there isn't an equivalent string, **in an equivalent context**, that you can reuse. Context is important because it can affect how the same literal English text is translated. This cuts down on the amount of text which needs to be translated. - -Strings should be entire sentences or phrases and should include substitution variables for any parameters. Do not concatenate strings in either Java or Javascript (or implicitly by laying them out in a specific order). So, instead of `"You have " + count + " row(s)"` (or worse `count != 1 ? " rows" : " row"`), internationalize everything together so that it can be translated taking into account word ordering and plurals for different languages, ie `"You have $1 {{plural $1: row|rows}}"`, passing the parameter(s) into the `$.i18n` call. - -If there's no string you can reuse, allocate an available key in the appropriate translation dictionary and add the default string, e.g. - -```json -..., -"section/newkey": "new default string for this key", -... -``` - -and then set the text (or HTML) of your HTML element using i18n helper method. So given an HTML fragment like: -```html - -``` -we could set its text using: -``` -$('#new-html-element-id').text($.i18n('section/newkey'])); -``` -or, if you need to embed HTML tags: -``` -$('#new-html-element-id').html($.i18n('section/newkey']); -``` - -### Adding a new language {#adding-a-new-language} - -The language dictionaries are stored in the `langs` subdirectory for the module e.g. - -* https://github.com/OpenRefine/OpenRefine/tree/master/main/webapp/modules/core/langs for the main interface -* https://github.com/OpenRefine/OpenRefine/tree/master/extensions/gdata/module/langs for google spreadsheet connection -* https://github.com/OpenRefine/OpenRefine/tree/master/extensions/database/module/langs for database via JDBC -* https://github.com/OpenRefine/OpenRefine/tree/master/extensions/wikidata/module/langs for Wikidata - -To add support for a new language, the easiest way is to do it directly in Weblate. To do it manually, copy `translation-en.json` to `translation-.json` and have your translator translate all the value strings (ie right hand side). - -#### Main interface {#main-interface} - The translation is best done [with Weblate](https://hosted.weblate.org/engage/openrefine/?utm_source=widget). Files are periodically merged by the developer team. - -Run the latest (hopefully cloned from github) version and check whether translated words fit to the layout. Not all items can be translated word by word, especially into non-Ìndo-European languages. - -If you see any text which remains in English even when you have checked all items, please create bug report in the issue tracker so that the developers can fix it. - -#### Extensions {#extensions} - -Extensions can be translated via Weblate just like the core software. - -The new extension for Wikidata contains lots of domain-specific concepts, with which you may not be familiar. The Wikidata may not have reconciliation service for your language. I recommend checking the glossary(https://www.wikidata.org/wiki/Wikidata:Glossary) to be consistent. - -By default, the system tries to load the language file corresponding to the currently in-use browser language. To override this setting a new menu item ("Language Settings") has been added at the index page. -To support a new language file, the developer should add a corresponding entry to the dropdown menu in this file: `/OpenRefine/main/webapp/modules/core/scripts/index/lang-settings-ui.html`. The entry should look like: -```javascript - -``` - -## Server-side localisation {#server--backend-coding} - -Currently no back end functions are translated, so things like error messages, undo history, etc may appear in English form. Rather than sending raw error text to the front end, it's better to send an error code which is translated into text on the front end. This allows for multiple languages to be supported. diff --git a/OpenRefine/docs/docs/technical-reference/version-release-process.md b/OpenRefine/docs/docs/technical-reference/version-release-process.md deleted file mode 100644 index 3f86ab7d6..000000000 --- a/OpenRefine/docs/docs/technical-reference/version-release-process.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -id: version-release-process -title: How to do an OpenRefine version release -sidebar_label: How to do an OpenRefine version release ---- - -When releasing a new version of Refine, the following steps should be followed: - -1. Make sure the `master` branch is stable and nothing has broken since the previous version. We need developers to stabilize the trunk and some volunteers to try out `master` for a few days. -2. Change the version number in [RefineServlet.java](http://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/RefineServlet.java#L62) and in the POM files using `mvn versions:set -DnewVersion=2.6-beta -DgenerateBackupPoms=false`. Commit the changes. -3. Compose the list of changes in the code and on the wiki. If the issues have been updated with the appropriate milestone, the Github issue tracker should be able to provide a good starting point for this. -4. Set up build machine. This needs to be Mac OS X or Linux. -5. Download Windows and Mac JREs to bundle them in the Windows and Mac packages from [AdoptOpenJDK](https://adoptopenjdk.net/). You only need the JREs, not the JDKs. Use the lowest version of Java supported (Java 11 currently). Configure the location of these JREs in the `settings.xml` file at the root of the repository. It is important to download recent versions of the JREs as this impacts which HTTPS certificates are accepted by the tool. -6. Insert the production Google credentials in https://github.com/OpenRefine/OpenRefine/blob/bc540a880eceb88e54f85ca43eb54769de3bfa4f/extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java#L36-L39 without committing the changes. -7. [Build the release candidate kits using the shell script (not just Maven)](https://github.com/OpenRefine/OpenRefine/wiki/Building-OpenRefine-From-Source). This must be done on Mac OS X or Linux to be able to build all 3 kits. On Linux you will need to install the `genisoimage` program first. -```shell -./refine dist 2.6-beta.2 -``` -To build the Windows version with embedded JRE, use `mvn package -s settings.xml -P embedded-jre -DskipTests=true`. - -8. On a Mac machine, compress the Mac `.dmg` (`genisoimage` does not compress it by default) with the following command on a mac machine: `hdiutil convert openrefine-uncompressed.dmg -format UDZO -imagekey zlib-level=9 -o openrefine-3.1-mac.dmg`. If running OS X in a VM, it's probably quicker and more reliable to transfer the kits to the host machine first and then to Github. Finder -> Go -> Connect -> smb://10.0.2.2/. You can then sign the generated DMG file with `codesign -s "Apple Distribution: Code for Science and Society, Inc." openrefine-3.1-mac.dmg`. This requires that you have installed the appropriate certificate on your Mac, see below. - -9. Tag the release candidate in git and push the tag to Github. For example: -```shell -git tag -a -m "Second beta" 2.6-beta.2 - git push origin --tags -``` -10. Upload the kits to Github releases [https://github.com/OpenRefine/OpenRefine/releases/](https://github.com/OpenRefine/OpenRefine/releases/) Mention the SHA sums of all uploaded artifacts. -11. Announce the beta/release candidate for testing -12. Repeat build/release candidate/testing cycle, if necessary. -13. Tag the release in git. Build the distributions and upload them. -14. [Update the OpenRefine Homebrew cask](https://github.com/OpenRefine/OpenRefine/wiki/Maintaining-OpenRefine's-Homebrew-Cask) or coordinate an update via the [developer list](https://groups.google.com/forum/#!forum/openrefine-dev) -15. Verify that the correct versions are shown in the widget at [http://openrefine.org/download](http://openrefine.org/download) -16. Announce on the [OpenRefine mailing list](https://groups.google.com/forum/#!forum/openrefine). -17. Update the version in master to the next version number with `-SNAPSHOT` (such as `4.3-SNAPSHOT`) -```shell -mvn versions:set -DnewVersion=4.3-SNAPSHOT -``` -18. If releasing a new major or minor version, create a snapshot of the docs, following [Docusaurus' versioning procedure](https://docusaurus.io/docs/versioning). - -Apple code signing -================== - -We have code signing certificates for our iOS distributions. To use them, follow these steps: -* Request advisory.committee@openrefine.org to be added to the Apple team: you need to provide the email address that corresponds to your AppleID account; -* Create a certificate signing request from your Mac: https://help.apple.com/developer-account/#/devbfa00fef7 -* Go to https://developer.apple.com/account/resources/certificates/add and select "Apple Distribution" as certificate type -* Upload the certificate signing request in the form -* Download the generated certificate -* Import this certificate in the "Keychain Access" app on your mac -* You can now sign code on behalf of the team using the `codesign` utility, such as `codesign -s "Apple Distribution: Code for Science and Society, Inc." openrefine-3.1-mac.dmg`. diff --git a/OpenRefine/docs/docs/technical-reference/writing-extensions.md b/OpenRefine/docs/docs/technical-reference/writing-extensions.md deleted file mode 100644 index 3511cc31f..000000000 --- a/OpenRefine/docs/docs/technical-reference/writing-extensions.md +++ /dev/null @@ -1,326 +0,0 @@ ---- -id: writing-extensions -title: Writing Extensions -sidebar_label: Writing Extensions ---- - -## Introduction {#introduction} - -This is a very brief overview of the structure of OpenRefine extensions. For more detailed documentation and step-by-step guides please see the following external documentation/tutorials: - -* Giuliano Tortoreto has [written documentation detailling how to build extension for OpenRefine](https://github.com/giTorto/OpenRefineExtensionDoc/raw/master/main.pdf) -* Owen Stephens has written [a guide to developing an extension which adds new GREL functions to OpenRefine](http://www.meanboyfriend.com/overdue_ideas/2017/05/writing-an-extension-to-add-new-grel-functions-to-openrefine/). - -OpenRefine makes use of a modified version of the [Butterfly framework](https://github.com/OpenRefine/simile-butterfly/tree/openrefine) to provide an extension architecture. OpenRefine extensions are Butterfly modules. You don't really need to know about Butterfly itself, but you might encounter "butterfly" here and there in the code base. - -Extensions that come with the code base are located under [the extensions subdirectory](https://github.com/OpenRefine/OpenRefine/tree/master/extensions), but when you develop your own extension, you can put its code anywhere as long as you point Butterfly to it. That is done by any one of the following methods - -* refer to your extension's directory in [the butterfly.properties file](https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/WEB-INF/butterfly.properties) through a `butterfly.modules.path` setting. -* specify the butterfly.modules.path property on the command line when you run OpenRefine. This overrides the values in the property file, so you need to include the default values first e.g. `-Dbutterfly.modules.path=modules,../../extensions,/path/to/your/extension` - -Please note that you should bundle any dependencies yourself, so you are insulated from OpenRefine packaging changes over time. - -### Directory Layout {#directory-layout} - -A OpenRefine extension sits in a file directory that contains the following files and sub-directories: - -``` -pom.xml - src/ - com/foo/bar/... *.java source files - module/ - *.html, *.vt files - scripts/... *.js files - styles/... *.css and *.less files - images/... image files - MOD-INF/ - lib/*.jar files - classes/... java class files - module.properties - controller.js -``` - -The file named module.properties (see [example](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/sample/module/MOD-INF/module.properties)) contains the extension's metadata. Of importance is the name field, which gives the extension a name that's used in many other places to refer to it. This can be different from the extension's directory name. - -``` -name = my-extension-name -``` - -Your extension's client-side resources (.html, .js, .css files) stored in the module/ subdirectory will be accessible from http://127.0.0.1:3333/extension/my-extension-name/ when OpenRefine is running. - -Also of importance is the dependency - -``` -requires = core -``` - -which makes sure that the core module of OpenRefine is loaded before the extension attempts to hook into it. - -The file named controller.js is responsible for registering the extension's hooks into OpenRefine. Look at the sample-extension extension's [controller.js](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/sample/module/MOD-INF/controller.js) file for an example. It should have a function called init() that does the hook registrations. - -The `pom.xml` file is an [Apache Maven](http://maven.apache.org/) build file. You can make a copy of the sample extension's `pom.xml` file to get started. The important point here is that the Java classes should be built into the `module/MOD-INF/classes` sub-directory. - -Note that your extension's Java code would need to reference some libraries used in OpenRefine and OpenRefine's Java classes themselves. These dependencies are reflected in the Maven configuration for the extension. - -## Sample extension {#sample-extension} - -The sample extension is included in the code base so that you can copy it and get started on writing your own extension. After you copy it, make sure you change its name inside its `module/MOD-INF/controller.js` file. - -### Basic Structure {#basic-structure} - -The sample extension's code is in `refine/extensions/sample/`. In that directory, Java source code is contained under the `src` sub-directory, and webapp code is under the `module` sub-directory. Here is the full directory layout: - -``` -refine/extensions/sample/ - build.xml (ant build script) - src/ - com/google/refine/sampleExtension/ - ... Java source code ... - module/ - MOD-INF/ - module.properties (module settings) - controller.js (module init and routing logic in Javascript) - classes/ - ... compiled Java classes ... - lib/ - ... Java jars ... - ... velocity templates (.vt) ... - ... LESS css files ... - ... client-side files (.html, .css, .js, image files) ... -``` - -The sub-directory `MOD-INF` contains the Butterfly module's metadata and is what Butterfly looks for when it scans directories for modules. `MOD-INF` serves similar functions as `WEB-INF` in other web frameworks. - -Java code is built into the sub-directory `classes` inside `MOD-INF`, and supporting external Java jars are in the `lib` sub-directory. Those will be automatically loaded by Butterfly. (The build.xml script is wired to compile into the `classes` sub-directory.) - -Client-side code is in the inner `module` sub-directory. They can be plain old .html, .css, .js, and image files, or they can be [LESS](http://lesscss.org/) files that get processed into CSS. There are also Velocity .vt files, but they need to be routed inside `MOD-INF/controller.js`. - -`MOD-INF/controller.js` lets you configure the extension's initialization and URL routing in Javascript rather than in Java. For example, when the requested URL path is either `/` or an empty string, we process and return `MOD-INF/index.vt` ( [see http://127.0.0.1:3333/extension/sample/](http://127.0.0.1:3333/extension/sample/) if OpenRefine is running). - -The `init()` function in `controller.js` allows the extension to register various client-side handlers for augmenting pages served by Refine's core. These handlers are feature-specific. For example, [this is where the jython extension adds its parser](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/jython/module/MOD-INF/controller.js#L46). As for the sample extension, it adds its script `project-injection.js` and style `project-injection.less` into the `/project` page. If you [view the source of the /project page](http://127.0.0.1:3333/project), you will see references to those two files. - -### Wiring Up the Extension {#wiring-up-the-extension} - -The Extensions are loaded by the Butterfly framework. Butterfly refers to these as 'modules'. [The location of modules is set in the `main/webapp/butterfly.properties` file](https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/WEB-INF/butterfly.properties#L27). Butterfly simply descends into each of those paths and looks for any `MOD-INF` directories. - -For more information, see [Extension Points](https://github.com/OpenRefine/OpenRefine/wiki/Extension-Points). - -## Extension points {#extension-points} - -### Client-side: Javascript and CSS {#client-side-javascript-and-css} - -The UI in OpenRefine for working with a project is coded in [the /main/webapp/modules/core/project.vt file](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/project.vt). The file is quite small, and that's because almost all of its content is to be expanded dynamically through the Velocity variables $scriptInjection and $styleInjection. So that your own Javascript and CSS files get loaded, you need to register them with the ClientSideResourceManager, which is done in the /module/MOD-INF/controller.js file. See [the controller.js file in this sample extension code](http://github.com/OpenRefine/OpenRefine/blob/master/extensions/sample/module/MOD-INF/controller.js) for an example. - -In the registration call, the variable `module` is already available to your code by default, and it refers to your own extension. - -``` -ClientSideResourceManager.addPaths( - "project/scripts", - module, - [ - "scripts/foo.js", - "scripts/subdir/bar.js" - ] - ); -``` - -You can specify one or more files for registration, and their paths are relative to the `module` subdirectory of your extension. They are included in the order listed. - -Javascript Bundling: Note that `project.vt` belongs to the core module and is thus under the control of the core module's [controller.js file](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js). The Javascript files to be included in `project.vt` are by default bundled together for performance. When debugging, you can prevent this bundling behavior by setting `bundle` to `false` near the top of that `controller.js` file. (If you have commit access to this code base, be sure not to check that change in.) - -### Client-side: Images {#client-side-images} - -We recommend that you always refer to images through your CSS files rather than in your Javascript code. URLs to images will thus be relative to your CSS files, e.g., - -``` -.foo { - background: url(../images/x.png); - } -``` - -If you really really absolutely need to refer to your images in your Javascript code, then look up your extension's URL path in the global Javascript variable `ModuleWirings`: - -``` -ModuleWirings["my-extension"] + "images/x.png" -``` - -### Client-side: HTML Templates {#client-side-html-templates} - -Beside Javascript, CSS, and images, your extension might also include HTML templates that get loaded on the fly by your Javascript code and injected into the page's DOM. For example, here is [the Cluster edit dialog template](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/dialogs/clustering-dialog.html), which gets loaded by code in [the equivalent javascript file 'clustering-dialog.js'](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/dialogs/clustering-dialog.js): - -``` -var dialog = $(DOM.loadHTML("core", "scripts/dialogs/clustering-dialog.html")); -``` - -`DOM.loadHTML` returns the content of the file as a string, and `$(...)` turns it into a DOM fragment. Where `"core"` is, you would want your extension's name. The path of the HTML file is relative to your extension's `module` subdirectory. - -### Client-side: Project UI Extension Points {#client-side-project-ui-extension-points} - -Getting your extension's Javascript code included in `project.vt` doesn't accomplish much by itself unless your code also registers hooks into the UI. For example, you can surely implement an exporter in Javascript, but unless you add a corresponding menu command in the UI, your user can't use your exporter. - -#### Main Menu {#main-menu} - -The main menu can be extended by calling any one of the methods `MenuBar.appendTo`, `MenuBar.insertBefore`, and `MenuBar.insertAfter`. Each method takes 2 arguments: an array of strings that identify a particular existing menu item or submenu, and one new single menu item or submenu or an array of menu items and submenus. For example, to insert 2 menu items and a menu separator before the menu item Project > Export Filtered Rows > Templating..., write this Javascript code wherever that would execute when your Javascript files get loaded: - -``` -MenuBar.insertBefore( - ["core/project", "core/export", "core/export-templating"], - [ - { - "label":"Menu item 1", - "click": function() { ... } - }, - { - "label":"Menu item 2", - "click": function() { ... } - }, - {} // separator - ] - ); -``` - -The array `["core/project", "core/export", "core/export-templating"]` pinpoints the reference menu item. - -See the beginning of [/main/webapp/modules/core/scripts/project/menu-bar.js](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/project/menu-bar.js) for IDs of menu items and submenus. - -#### Column Header Menu {#column-header-menu} - -The drop-down menu of each column can also be extended, but the mechanism is slightly different compared to the main menu. Because the drop-down menu for a particular column is constructed on the fly when the user actually clicks the drop-down menu button, extending the column header menu can't really be done once at start-up time, but must be done every time a column header menu gets created. So, registration in this case involves providing a function that gets called each such time: - -``` -DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { ... do stuff to menu ... }); -``` - -That function takes in the column object (which contains the column's name), the column header UI object (generally not so useful), and the menu to extend. In the previous code line where it says "do stuff to menu", you can write something like this: - -``` -MenuSystem.appendTo(menu, ["core/facet"], [ - { - id: "core/text-facet", - label: "My Facet on " + column.name, - click: function() { - ... use column.name and do something ... - } - }, - ]); -``` - -In addition to `MenuSystem.appendTo`, you can also call `MenuSystem.insertBefore` and `MenuSystem.insertAfter` which the same 3 arguments. To see what IDs you can use, see the function `DataTableColumnHeaderUI.prototype._createMenuForColumnHeader` in [/main/webapp/modules/core/scripts/views/data-table/column-header-ui.js](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/views/data-table/column-header-ui.js). - -### Server-side: Ajax Commands {#server-side-ajax-commands} - -The client-side of OpenRefine gets things done by calling AJAX commands on the server-side. These commands must be registered with the OpenRefine servlet, so that the servlet knows how to route AJAX calls from the client-side. This can be done inside the `init` function in your extension's `controller.js` file, e.g., - -``` -function init() { - var RefineServlet = Packages.com.google.refine.RefineServlet; - RefineServlet.registerCommand(module, "my-command", new Packages.com.foo.bar.MyCommand()); - } -``` - -Your command will then be accessible at [http://127.0.0.1:3333/command/my-extension/my-command](http://127.0.0.1:3333/command/my-extension/my-command). - -### Server-side: Operations {#server-side-operations} - -Most commands change the project's data. Most of them do so by creating abstract operations. See the Changes, History, Processes, and Operations section of the [Server Side Architecture](https://github.com/OpenRefine/OpenRefine/wiki/Server-Side-Architecture) document. - -You can register an operation **class** in the `init` function as follows: - -``` -Packages.com.google.refine.operations.OperationRegistry.registerOperation( - module, - "operation-name", - Packages.com.foo.bar.MyOperation - ); -``` - -Do not call `new` to construct an operation instance. You must register the class itself. The class should have a static function for reconstructing an operation instance from a JSON blob: - -``` -static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - ... - } -``` - -### Server-side: GREL {#server-side-grel} - -GREL can be extended with new functions. This is also done in the `init` function in `controller.js`, e.g., - -``` -Packages.com.google.refine.grel.ControlFunctionRegistry.registerFunction( - "functionName", new Packages.com.foo.bar.TheFunctionClass()); -``` - -You might also want to provide new variables (beyond just `value`, `cells`, `row`, etc.) available to expressions. This is done by registering a binder that implements the interface `com.google.refine.expr.Binder`: - -``` -Packages.com.google.refine.expr.ExpressionUtils.registerBinder( - new Packages.com.foo.bar.MyBinder()); -``` - -### Server-side: Importers {#server-side-importers} - -You can register an importer as follows: - -``` -Packages.com.google.refine.importers.ImporterRegistry.registerImporter( - "importer-name", new Packages.com.foo.bar.MyImporter()); -``` - -The string `"importer-name"` isn't important at all. It's not really related to file extension or mime-type. Just use something unique. Your importer will be explicitly called to test if it can import something. - -### Server-side: Exporters {#server-side-exporters} - -You can register an exporter as follows: - -``` -Packages.com.google.refine.exporters.ExporterRegistry.registerExporter( - "exporter-name", new Packages.com.foo.bar.MyExporter()); -``` - -The string `"exporter-name"` isn't important at all. It's only used by the client-side to tell the server-side which exporter to use. Just use something unique and, of course, relevant. - -### Server-side: Overlay Models {#server-side-overlay-models} - -Overlay models are objects attached onto a core Project object to store and manage additional data for that project. For example, the schema alignment skeleton is managed by the Protograph overlay model. An overlay model implements the interface `com.google.refine.model.OverlayModel` and can be registered like so: - -``` -Packages.com.google.refine.model.Project.registerOverlayModel( - "model-name", - Packages.com.foo.bar.MyOverlayModel); -``` - -Note that you register the **class** , not an instance. The class should implement the following static method for reconstructing an overlay model instance from a JSON blob: - -``` -static public OverlayModel reconstruct(JSONObject o) throws JSONException { - ... - } -``` - -When the project gets saved, the overlay model instance's `write` method will be called: - -``` -public void write(JSONWriter writer, Properties options) throws JSONException { - ... - } -``` - -### Server-side: Scripting Languages {#server-side-scripting-languages} - -A scripting language (such as Jython) can be registered as follows: - -``` -Packages.com.google.refine.expr.MetaParser.registerLanguageParser( - "jython", - "Jython", - Packages.com.google.refine.jython.JythonEvaluable.createParser(), - "return value" - ); -``` - -The first string is the prefix that gets prepended to each expression so that we know which language the expression is in. This should be short, unique, and identifying. The second string is a user-friendly name of the language. The third is an object that implements the interface `com.google.refine.expr.LanguageSpecificParser`. The final string is the default expression in that language that would return the cell's value. - -In 2018 we are making important changes to OpenRefine to modernize it, for the benefit of users and contributors. This page describes the changes that impact developers of extensions or forks and is intended to minimize the effort required on their end to follow the transition. The instructions are written specifically with extension maintainers in mind, but fork maintainers should also find it useful. - -This document describes the migrations in the order they are committed to the master branch. This means that it should be possible to perform each migration in turn, with the ability to run the software between each stage by checking out the appropriate git commit. diff --git a/OpenRefine/docs/docusaurus.config.js b/OpenRefine/docs/docusaurus.config.js deleted file mode 100644 index 778b3cd17..000000000 --- a/OpenRefine/docs/docusaurus.config.js +++ /dev/null @@ -1,129 +0,0 @@ -module.exports = { - onBrokenLinks: 'throw', - onBrokenMarkdownLinks: 'throw', - title: 'OpenRefine', - tagline: 'A power tool for working with messy data.', - url: 'https://docs.openrefine.org/', - baseUrl: '/', - favicon: 'img/openrefine_logo.png', - organizationName: 'OpenRefine', // Usually your GitHub org/user name. - projectName: 'OpenRefine', // Usually your repo name. - i18n: { - defaultLocale: 'en', - locales: ['en', 'jp', 'fr'], - }, - themeConfig: { - navbar: { - title: 'OpenRefine Documentation', - logo: { - alt: 'OpenRefine diamond logo', - src: 'img/openrefine_logo.png', - }, - items: [ - { - to: '/', - activeBasePath: 'docs', - label: 'User Manual', - position: 'left', - }, - { - to: 'technical-reference/technical-reference-index', - label: 'Technical Reference', - position: 'left' - }, - { - type: 'localeDropdown', - position: 'right', - }, - { - href: 'https://github.com/OpenRefine/OpenRefine/edit/master/docs', - 'aria-label': 'GitHub', - className: 'header-github-link', - position: 'right', - }, - ], - }, - algolia: { - apiKey: '591fc612419d2e5b6bee6822cc17064f', - indexName: 'openrefine', - contextualSearch: true, - }, - footer: { - logo: { - alt: 'OpenRefine diamond logo', - src: 'img/openrefine_logo.png', - href: 'https://docs.openrefine.org', - }, - style: 'dark', - links: [ - { - title: 'Community', - items: [ - { - label: 'Mailing List', - href: 'http://groups.google.com/group/openrefine/' - }, - { - label: 'Gitter Chat', - href: 'https://gitter.im/OpenRefine/OpenRefine', - }, - { - label: 'Twitter', - href: 'https://twitter.com/openrefine', - }, - ], - }, - { - title: 'More', - items: [ - { - label: 'Official Website', - href: 'https://openrefine.org', - }, - { - label: 'GitHub', - href: 'https://github.com/OpenRefine/OpenRefine', - }, - ], - }, - ], - copyright: `Copyright © ${new Date().getFullYear()} OpenRefine contributors`, - }, - }, - themes: [], - plugins: [], - presets: [ - [ - '@docusaurus/preset-classic', - { - docs: { - // Docs folder path relative to website dir. Equivalent to `customDocsPath`. - // path: 'docs', - // Sidebars file relative to website dir. - sidebarPath: require.resolve('./sidebars.js'), - // Equivalent to `editUrl` but should point to `website` dir instead of `website/docs`. - editUrl: 'https://github.com/OpenRefine/OpenRefine/edit/master/docs', - // Equivalent to `docsUrl`. - routeBasePath: '/', - // Remark and Rehype plugins passed to MDX. Replaces `markdownOptions` and `markdownPlugins`. - remarkPlugins: [], - rehypePlugins: [], - // Equivalent to `enableUpdateBy`. - showLastUpdateAuthor: true, - // Equivalent to `enableUpdateTime`. - showLastUpdateTime: true, - }, - theme: { - customCss: require.resolve('./src/css/custom.css'), - }, - }, - ], - ], - scripts: [ - { - src: '/js/fix-location.js', - async: false, - defer: false, - }, - ], -}; diff --git a/OpenRefine/docs/netlify.toml b/OpenRefine/docs/netlify.toml deleted file mode 100644 index da425af8b..000000000 --- a/OpenRefine/docs/netlify.toml +++ /dev/null @@ -1,9 +0,0 @@ - -# Note: this file's config overrides the Netlify UI admin config - -[context.production] - command = "yarn netlify:build:production" - -[context.deploy-preview] - command = "yarn netlify:build:deployPreview" - diff --git a/OpenRefine/docs/package.json b/OpenRefine/docs/package.json deleted file mode 100644 index 70a80a852..000000000 --- a/OpenRefine/docs/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "OpenRefine-Documentation", - "version": "1.0.0", - "description": "Documentation for OpenRefine", - "license": "BSD-3-Clause", - "author": "OpenRefine", - "private": true, - "scripts": { - "start": "docusaurus start", - "build": "docusaurus build", - "swizzle": "docusaurus swizzle", - "deploy": "docusaurus deploy", - "crowdin": "crowdin", - "netlify:build:production": "docusaurus write-translations && crowdin upload && crowdin download && yarn build", - "netlify:build:deployPreview": "docusaurus write-translations && yarn build" - }, - "dependencies": { - "@crowdin/cli": "3", - "@docusaurus/core": "^2.0.0-beta.10", - "@docusaurus/preset-classic": "^2.0.0-beta.759298296", - "clsx": "^1.1.1", - "react": "^17.0.2", - "react-dom": "^17.0.2" - } -} diff --git a/OpenRefine/docs/sidebars.js b/OpenRefine/docs/sidebars.js deleted file mode 100644 index b6f16d838..000000000 --- a/OpenRefine/docs/sidebars.js +++ /dev/null @@ -1,53 +0,0 @@ -module.exports = { - docs: { - 'User Manual': [ - 'index', - 'manual/installing', - 'manual/running', - 'manual/starting', - { - type: 'category', - label: 'Exploring data', - items: ['manual/exploring', 'manual/facets', 'manual/sortview'], - }, - { - type: 'category', - label: 'Transforming data', - items: ['manual/transforming', 'manual/cellediting','manual/columnediting','manual/transposing'], - }, - 'manual/reconciling', - { - type: 'category', - label: 'Wikibase', - items: ['manual/wikibase/overview', 'manual/wikibase/configuration', 'manual/wikibase/reconciling', 'manual/wikibase/schema-alignment', - 'manual/wikibase/new-entities', 'manual/wikibase/quality-assurance', 'manual/wikibase/uploading'], - }, - { - type: 'category', - label: 'Expressions', - items: ['manual/expressions', 'manual/grel', 'manual/grelfunctions', 'manual/jythonclojure'], - }, - 'manual/exporting', - 'manual/troubleshooting' - ], - 'GREL Reference': [ - 'manual/grelfunctions'], - 'Technical Reference': [ - 'technical-reference/technical-reference-index', - 'technical-reference/architecture', - 'technical-reference/openrefine-api', - 'technical-reference/reconciliation-api', - 'technical-reference/contributing', - 'technical-reference/build-test-run', - 'technical-reference/development-roadmap', - 'technical-reference/version-release-process', - 'technical-reference/homebrew-cask-process', - 'technical-reference/writing-extensions', - 'technical-reference/migrating-older-extensions', - 'technical-reference/translating-ui', - 'technical-reference/translating-docs', - 'technical-reference/functional-tests', - 'technical-reference/maintainer-guidelines' - ] - }, -}; diff --git a/OpenRefine/docs/src/css/custom.css b/OpenRefine/docs/src/css/custom.css deleted file mode 100644 index 22e632746..000000000 --- a/OpenRefine/docs/src/css/custom.css +++ /dev/null @@ -1,87 +0,0 @@ -/** - * You can override the default Infima variables here. - * Note: this is not a complete list of --ifm- variables. - */ - - /** - * Colors extracted from our OpenRefine logo - */ - :root { - --ifm-color-primary: #00a0ff; - --ifm-color-primary-dark: #0078ff; - --ifm-color-primary-darker: #0050ac; - --ifm-color-primary-darkest: #003b7e; - --ifm-color-primary-light: #00b4ff; - --ifm-color-primary-lighter: #2da6d1; - --ifm-color-primary-lightest: #00dcff; - --ifm-color-secondary: #999999; - --ifm-alert-border-width: 0.375em; - --ifm-alert-background-color: #ffffff; - --ifm-panel-background-color: #ffffff; - --ifm-panel-box-shadow-color: #999999; - --ifm-code-color: var(--ifm-color-primary-darkest); - } - - /******* COMMON *******/ - - html[data-theme='dark'] { - --ifm-panel-background-color: rgb(41, 45, 62); - --ifm-panel-box-shadow-color: var(--ifm-panel-background-color); - --ra-admonition-color: var(--ifm-color-black); - --ra-admonition-icon-color: var(--ifm-color-black); - --ifm-code-color: var(--ifm-color-primary-light); - } - - /******* MARKDOWN STYLES *******/ - - .alert { - border-radius: 0; - border: 0; - box-shadow: 0em 0em 0.625em 0em var(--ifm-panel-box-shadow-color); - border-width: 1em; - border-left: var(--ifm-alert-border-width) solid var(--ifm-alert-border-color); - background-color: var(--ifm-panel-background-color); - color: var(--ifm-font-color-secondary); - } - - .admonition-icon svg { - fill: var(--ifm-font-color-secondary); - } - - .alert a { - color: var(--ifm-font-color-secondary); - } - - .markdown h4{ - margin-top: 1.025em; - font-size: 1.25em; - } - - .markdown h3{ - margin-top: 1.325em; - font-size: 1.8125em; - } - - .markdown h2{ - margin-top: 1.525em; - font-size: 2.1875em; - } - - .markdown .menuItems, .markdown .fieldLabels, .markdown .tabLabels, .markdown .buttonLabels { - font-size: var(--ifm-code-font-size); - display: inline-block; - border: 0.125em solid #4dc6e1; - border-radius: var(--ifm-code-border-radius); - margin: 0.0625em 0em; - padding: 0em 0.25em; - } - - .markdown img { - display: block; - margin-left: auto; - margin-right: auto; - } - - code { - color: var(--ifm-code-color); - } diff --git a/OpenRefine/docs/src/theme/Footer/Footer.js b/OpenRefine/docs/src/theme/Footer/Footer.js deleted file mode 100644 index 11d153e89..000000000 --- a/OpenRefine/docs/src/theme/Footer/Footer.js +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Copyright (c) 2017-present, Facebook, Inc. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -const React = require('react'); - -class Footer extends React.Component { - docUrl(doc, language) { - const baseUrl = this.props.config.baseUrl; - const docsUrl = this.props.config.docsUrl; - const docsPart = `${docsUrl ? `${docsUrl}/` : ''}`; - const langPart = `${language ? `${language}/` : ''}`; - return `${baseUrl}${docsPart}${langPart}${doc}`; - } - - pageUrl(doc, language) { - const baseUrl = this.props.config.baseUrl; - return baseUrl + (language ? `${language}/` : '') + doc; - } - - render() { - return ( - - ); - } -} - -module.exports = Footer; diff --git a/OpenRefine/docs/static/css/custom.css b/OpenRefine/docs/static/css/custom.css deleted file mode 100644 index 55da82683..000000000 --- a/OpenRefine/docs/static/css/custom.css +++ /dev/null @@ -1,2 +0,0 @@ -/* custom CSS can be added here */ - diff --git a/OpenRefine/docs/static/img/cluster.png b/OpenRefine/docs/static/img/cluster.png deleted file mode 100644 index 408c4a256..000000000 Binary files a/OpenRefine/docs/static/img/cluster.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/columnjoin.png b/OpenRefine/docs/static/img/columnjoin.png deleted file mode 100644 index 6b49798cd..000000000 Binary files a/OpenRefine/docs/static/img/columnjoin.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/columnreconciled.png b/OpenRefine/docs/static/img/columnreconciled.png deleted file mode 100644 index de593be27..000000000 Binary files a/OpenRefine/docs/static/img/columnreconciled.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/columnsplit.png b/OpenRefine/docs/static/img/columnsplit.png deleted file mode 100644 index b65d5291f..000000000 Binary files a/OpenRefine/docs/static/img/columnsplit.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/custom-tabular-exporter.png b/OpenRefine/docs/static/img/custom-tabular-exporter.png deleted file mode 100644 index 89099eb46..000000000 Binary files a/OpenRefine/docs/static/img/custom-tabular-exporter.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/custom-tabular-exporter2.png b/OpenRefine/docs/static/img/custom-tabular-exporter2.png deleted file mode 100644 index 2b02f7437..000000000 Binary files a/OpenRefine/docs/static/img/custom-tabular-exporter2.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/dates.png b/OpenRefine/docs/static/img/dates.png deleted file mode 100644 index 7c2e90445..000000000 Binary files a/OpenRefine/docs/static/img/dates.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/eclipse-debug-config.png b/OpenRefine/docs/static/img/eclipse-debug-config.png deleted file mode 100644 index 9463261ea..000000000 Binary files a/OpenRefine/docs/static/img/eclipse-debug-config.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/eclipse-exec-config.png b/OpenRefine/docs/static/img/eclipse-exec-config.png deleted file mode 100644 index d71e032ba..000000000 Binary files a/OpenRefine/docs/static/img/eclipse-exec-config.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/eclipse-import-maven-project-1.png b/OpenRefine/docs/static/img/eclipse-import-maven-project-1.png deleted file mode 100644 index 7c4736825..000000000 Binary files a/OpenRefine/docs/static/img/eclipse-import-maven-project-1.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/eclipse-import-maven-project-2.png b/OpenRefine/docs/static/img/eclipse-import-maven-project-2.png deleted file mode 100644 index a847c7336..000000000 Binary files a/OpenRefine/docs/static/img/eclipse-import-maven-project-2.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/env.png b/OpenRefine/docs/static/img/env.png deleted file mode 100644 index ec8f1b5f5..000000000 Binary files a/OpenRefine/docs/static/img/env.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/error.png b/OpenRefine/docs/static/img/error.png deleted file mode 100644 index 5de4a84bd..000000000 Binary files a/OpenRefine/docs/static/img/error.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/export-menu.png b/OpenRefine/docs/static/img/export-menu.png deleted file mode 100644 index ad02db554..000000000 Binary files a/OpenRefine/docs/static/img/export-menu.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/expression-editor.png b/OpenRefine/docs/static/img/expression-editor.png deleted file mode 100644 index 51a1674c3..000000000 Binary files a/OpenRefine/docs/static/img/expression-editor.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/facetfilter.png b/OpenRefine/docs/static/img/facetfilter.png deleted file mode 100644 index 91f02528b..000000000 Binary files a/OpenRefine/docs/static/img/facetfilter.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/failed-visual-test.png b/OpenRefine/docs/static/img/failed-visual-test.png deleted file mode 100644 index 3792b2b22..000000000 Binary files a/OpenRefine/docs/static/img/failed-visual-test.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/fetchingURLs.png b/OpenRefine/docs/static/img/fetchingURLs.png deleted file mode 100644 index 204b367a9..000000000 Binary files a/OpenRefine/docs/static/img/fetchingURLs.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/goto.png b/OpenRefine/docs/static/img/goto.png deleted file mode 100644 index 256ff9b9e..000000000 Binary files a/OpenRefine/docs/static/img/goto.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/history.png b/OpenRefine/docs/static/img/history.png deleted file mode 100644 index 9c98df3bd..000000000 Binary files a/OpenRefine/docs/static/img/history.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/intellij-maven.png b/OpenRefine/docs/static/img/intellij-maven.png deleted file mode 100644 index 89fb6e95d..000000000 Binary files a/OpenRefine/docs/static/img/intellij-maven.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/intellij-module-settings.png b/OpenRefine/docs/static/img/intellij-module-settings.png deleted file mode 100644 index 1279ae1a1..000000000 Binary files a/OpenRefine/docs/static/img/intellij-module-settings.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/intellij-open-module-settings.png b/OpenRefine/docs/static/img/intellij-open-module-settings.png deleted file mode 100644 index 7b006e02a..000000000 Binary files a/OpenRefine/docs/static/img/intellij-open-module-settings.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/intellij-setup-1.png b/OpenRefine/docs/static/img/intellij-setup-1.png deleted file mode 100644 index 17422c147..000000000 Binary files a/OpenRefine/docs/static/img/intellij-setup-1.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/javahome.png b/OpenRefine/docs/static/img/javahome.png deleted file mode 100644 index e45168715..000000000 Binary files a/OpenRefine/docs/static/img/javahome.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/null.png b/OpenRefine/docs/static/img/null.png deleted file mode 100644 index c19990e77..000000000 Binary files a/OpenRefine/docs/static/img/null.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/numericfacet.png b/OpenRefine/docs/static/img/numericfacet.png deleted file mode 100644 index 5dec9cf87..000000000 Binary files a/OpenRefine/docs/static/img/numericfacet.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/numericlogfacet.png b/OpenRefine/docs/static/img/numericlogfacet.png deleted file mode 100644 index 617326c76..000000000 Binary files a/OpenRefine/docs/static/img/numericlogfacet.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/openrefine_logo.png b/OpenRefine/docs/static/img/openrefine_logo.png deleted file mode 100644 index 15b282386..000000000 Binary files a/OpenRefine/docs/static/img/openrefine_logo.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/projectscreen.png b/OpenRefine/docs/static/img/projectscreen.png deleted file mode 100644 index e4b5e839a..000000000 Binary files a/OpenRefine/docs/static/img/projectscreen.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/reconcile-ambiguous.gif b/OpenRefine/docs/static/img/reconcile-ambiguous.gif deleted file mode 100644 index 3cf09a52b..000000000 Binary files a/OpenRefine/docs/static/img/reconcile-ambiguous.gif and /dev/null differ diff --git a/OpenRefine/docs/static/img/reconcile-by-type.png b/OpenRefine/docs/static/img/reconcile-by-type.png deleted file mode 100644 index 259e339b3..000000000 Binary files a/OpenRefine/docs/static/img/reconcile-by-type.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/reconcile-with-property.png b/OpenRefine/docs/static/img/reconcile-with-property.png deleted file mode 100644 index e25b81a27..000000000 Binary files a/OpenRefine/docs/static/img/reconcile-with-property.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/reconcileGND.png b/OpenRefine/docs/static/img/reconcileGND.png deleted file mode 100644 index 3f37a2ad8..000000000 Binary files a/OpenRefine/docs/static/img/reconcileGND.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/reconcileIDerror.png b/OpenRefine/docs/static/img/reconcileIDerror.png deleted file mode 100644 index e4a6d1646..000000000 Binary files a/OpenRefine/docs/static/img/reconcileIDerror.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/reconcileParis.gif b/OpenRefine/docs/static/img/reconcileParis.gif deleted file mode 100644 index 6d103db94..000000000 Binary files a/OpenRefine/docs/static/img/reconcileParis.gif and /dev/null differ diff --git a/OpenRefine/docs/static/img/reconcileelements.gif b/OpenRefine/docs/static/img/reconcileelements.gif deleted file mode 100644 index 5f0403e39..000000000 Binary files a/OpenRefine/docs/static/img/reconcileelements.gif and /dev/null differ diff --git a/OpenRefine/docs/static/img/reconcilehover.png b/OpenRefine/docs/static/img/reconcilehover.png deleted file mode 100644 index bdc6bd5e6..000000000 Binary files a/OpenRefine/docs/static/img/reconcilehover.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/reconcilewindow.png b/OpenRefine/docs/static/img/reconcilewindow.png deleted file mode 100644 index 91a94ecf0..000000000 Binary files a/OpenRefine/docs/static/img/reconcilewindow.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/reconcilewindow2.png b/OpenRefine/docs/static/img/reconcilewindow2.png deleted file mode 100644 index e0cfc67ee..000000000 Binary files a/OpenRefine/docs/static/img/reconcilewindow2.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/scatterplot.png b/OpenRefine/docs/static/img/scatterplot.png deleted file mode 100644 index 215bcea6a..000000000 Binary files a/OpenRefine/docs/static/img/scatterplot.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/sort.png b/OpenRefine/docs/static/img/sort.png deleted file mode 100644 index a793da81d..000000000 Binary files a/OpenRefine/docs/static/img/sort.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/sort2.png b/OpenRefine/docs/static/img/sort2.png deleted file mode 100644 index 200dc45b6..000000000 Binary files a/OpenRefine/docs/static/img/sort2.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/sortPermanent.png b/OpenRefine/docs/static/img/sortPermanent.png deleted file mode 100644 index 142e96ffb..000000000 Binary files a/OpenRefine/docs/static/img/sortPermanent.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/sql-exporter.png b/OpenRefine/docs/static/img/sql-exporter.png deleted file mode 100644 index f5cb98f3c..000000000 Binary files a/OpenRefine/docs/static/img/sql-exporter.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/sql-exporter2.png b/OpenRefine/docs/static/img/sql-exporter2.png deleted file mode 100644 index 1f23784e8..000000000 Binary files a/OpenRefine/docs/static/img/sql-exporter2.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/templating-exporter.png b/OpenRefine/docs/static/img/templating-exporter.png deleted file mode 100644 index 836e47d68..000000000 Binary files a/OpenRefine/docs/static/img/templating-exporter.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/timelinefacet.png b/OpenRefine/docs/static/img/timelinefacet.png deleted file mode 100644 index 9c0058243..000000000 Binary files a/OpenRefine/docs/static/img/timelinefacet.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/transpose1.png b/OpenRefine/docs/static/img/transpose1.png deleted file mode 100644 index 649c9315c..000000000 Binary files a/OpenRefine/docs/static/img/transpose1.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/transpose2.png b/OpenRefine/docs/static/img/transpose2.png deleted file mode 100644 index ff62166b5..000000000 Binary files a/OpenRefine/docs/static/img/transpose2.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/unicodefacet.png b/OpenRefine/docs/static/img/unicodefacet.png deleted file mode 100644 index 268e0f89b..000000000 Binary files a/OpenRefine/docs/static/img/unicodefacet.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/visual-test-cypress-failure.png b/OpenRefine/docs/static/img/visual-test-cypress-failure.png deleted file mode 100644 index 7f3f90048..000000000 Binary files a/OpenRefine/docs/static/img/visual-test-cypress-failure.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/wikidata-login.png b/OpenRefine/docs/static/img/wikidata-login.png deleted file mode 100644 index 836fdcfb0..000000000 Binary files a/OpenRefine/docs/static/img/wikidata-login.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/wikidata-schema.png b/OpenRefine/docs/static/img/wikidata-schema.png deleted file mode 100644 index d8c7c22b8..000000000 Binary files a/OpenRefine/docs/static/img/wikidata-schema.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/wikidata-terms.png b/OpenRefine/docs/static/img/wikidata-terms.png deleted file mode 100644 index 8327242ae..000000000 Binary files a/OpenRefine/docs/static/img/wikidata-terms.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/wikidata-terms2.png b/OpenRefine/docs/static/img/wikidata-terms2.png deleted file mode 100644 index fbe7d3286..000000000 Binary files a/OpenRefine/docs/static/img/wikidata-terms2.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/wikidata-translated.png b/OpenRefine/docs/static/img/wikidata-translated.png deleted file mode 100644 index 0a96310d3..000000000 Binary files a/OpenRefine/docs/static/img/wikidata-translated.png and /dev/null differ diff --git a/OpenRefine/docs/static/img/yeardata.png b/OpenRefine/docs/static/img/yeardata.png deleted file mode 100644 index 8250d4036..000000000 Binary files a/OpenRefine/docs/static/img/yeardata.png and /dev/null differ diff --git a/OpenRefine/docs/static/js/fix-location.js b/OpenRefine/docs/static/js/fix-location.js deleted file mode 100644 index 0e156b7c2..000000000 --- a/OpenRefine/docs/static/js/fix-location.js +++ /dev/null @@ -1,8 +0,0 @@ -/* - * when trailing slashes are added to URLs, this breaks internal links. - * This is a fix for this issue taken from: - * https://github.com/facebook/docusaurus/issues/2394#issuecomment-630638096 - */ -if (window && window.location && window.location.pathname.endsWith('/') && window.location.pathname !== '/') { - window.history.replaceState('', '', window.location.pathname.substr(0, window.location.pathname.length - 1) + window.location.hash) -} diff --git a/OpenRefine/docs/versioned_docs/version-3.4/index.md b/OpenRefine/docs/versioned_docs/version-3.4/index.md deleted file mode 100644 index 729fd8aa7..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/index.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -slug: / -id: index -title: OpenRefine user manual -sidebar_label: Introduction ---- - - -This manual is designed to comprehensively walk through every aspect of setting up and using OpenRefine 3.4.1, including every interface function and feature. - - - -This user manual starts with instructions for [installing or upgrading OpenRefine on Windows, Mac, and Linux computers](manual/installing). It then walks you through [the interface and how to run OpenRefine](manual/running#jvm-preferences) from a program or command line, with or without setting custom preferences and modifications. - -The manual then teaches you how to [start a project](manual/starting) by importing an existing dataset. We work through how to [view and learn about your data](manual/exploring) using facets, filters, and sorting. - -Then we launch into [transforming that data permanently](manual/transforming) through common and custom transformations, clustering, pulling data from the web, [reconciling](manual/reconciling), and [writing expressions](manual/expressions). - -Finally we discuss what to do with your improved dataset, whether [exporting](manual/exporting) it to a file or uploading statements to Wikidata. - -If you're stuck on any aspect and can't find an answer in the manual, try the [Troubleshooting page](manual/troubleshooting) for links to various places to find help. - -If you are new and want to learn how to use OpenRefine using an example dataset, you may wish to start with a user-contributed tutorial from our [recommendations list](https://github.com/OpenRefine/OpenRefine/wiki/External-Resources). diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/cellediting.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/cellediting.md deleted file mode 100644 index 2c2b4f654..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/cellediting.md +++ /dev/null @@ -1,165 +0,0 @@ ---- -id: cellediting -title: Cell editing -sidebar_label: Cell editing ---- -## Overview {#overview} - -OpenRefine offers a number of features to edit and improve the contents of cells automatically and efficiently. - -One way of doing this is editing through a [text facet](facets#text-facet). Once you have created a facet on a column, hover over the displayed results in the sidebar. Click on the small “edit” button that appears to the right of the facet, and type in a new value. This will apply to all the cells in the facet. - -You can apply a text facet on numbers, boolean values, and dates, but if you edit a value it will be converted into the text [data type](exploring#data-types) (regardless of whether you edit a date into another correctly-formatted date, or a “true” value into “false”, etc.). - -## Transform {#transform} - -Select Edit cellsTransform... to open up an expressions window. From here, you can apply [expressions](expressions) to your data. The simplest examples are GREL functions such as [`toUppercase()`](grelfunctions#touppercases) or [`toLowercase()`](grelfunctions#tolowercases), used in expressions as `toUppercase(value)` or `toLowercase(value)`. When used on a column operation, `value` is the information in each cell in the selected column. - -Use the preview to ensure your data is being transformed correctly. - -You can also switch to the Undo / Redo tab inside the expressions window to reuse expressions you’ve already attempted in this project, whether they have been undone or not. - -OpenRefine offers you some frequently-used transformations in the next menu option, Common transforms. For more custom transforms, read up on [expressions](expressions). - -## Common transforms {#common-transforms} - -### Trim leading and trailing whitespace {#trim-leading-and-trailing-whitespace} - -Often cell contents that should be identical, and look identical, are different because of space or line-break characters that are invisible to users. This function will get rid of any characters that sit before or after visible text characters. - -### Collapse consecutive whitespace {#collapse-consecutive-whitespace} - -You may find that some text cells contain what look like spaces but are actually tabs, or contain multiple spaces in a row. This function will remove all space characters that sit in sequence and replace them with a single space. - -### Unescape HTML {#unescape-html} - -Your data may come from an HTML-formatted source that expresses some characters through references (such as “&nbsp;” for a space, or “%u0107” for a ć) instead of the actual Unicode characters. You can use the “unescape HTML entities” transform to look for these codes and replace them with the characters they represent. For other formatting that needs to be escaped, try a custom transformation with [`escape()`](grelfunctions#escapes-s-mode). - -### Replace smart quotes with ASCII {#replace-smart-quotes-with-ascii} - -Smart quotes (or curly quotes) recognize whether they come at the beginning or end of a string, and will generate an “open” quote (“) and a “close” quote (”). These characters are not ASCII-compliant (though they are UTF8-compliant) so you can use this tranform to replace them with a straight double quote character (") instead. - -### Case transforms {#case-transforms} - -You can transform an entire column of text into UPPERCASE, lowercase, or Title Case using these three options. This can be useful if you are planning to do textual analysis and wish to avoid case-sensitivity (which some functions are) causing problems in your analysis. Consider also using a [custom facet](facets#custom-text-facet) to temporarily modify cases instead of this permanent operation if appropriate. - -### Data-type transforms {#data-type-transforms} - -As detailed in [Data types](exploring#data-types), OpenRefine recognizes different data types: string, number, boolean, and date. When you use these transforms, OpenRefine will check to see if the given values can be converted, then both transform the data in the cells (such as “3” as a text string to “3” as a number) and convert the data type on each successfully transformed cell. Cells that cannot be transformed will output the original value and maintain their original data type. - -:::caution -Be aware that dates may require manual intervention to transform successfully: see the section on [Dates](exploring#dates) for more information. -::: - -Because these common transforms do not offer the ability to output an error instead of the original cell contents, be careful to look for unconverted and untransformed values. You will see a yellow alert at the top of screen that will tell you how many cells were converted - if this number does not match your current row set, you will need to look for and manually correct the remaining cells. Also consider faceting by data type, with the GREL function [`type()`](grelfunctions#typeo). - -You can also convert cells into null values or empty strings. This can be useful if you wish to, for example, erase duplicates that you have identified and are analyzing as a subset. - -## Fill down and blank down {#fill-down-and-blank-down} - -Fill down and blank down are two functions most frequently used when encountering data organized into [records](exploring#row-types-rows-vs-records) - that is, multiple rows associated with one specific entity. - -If you receive information in rows mode and want to convert it to records mode, the easiest way is to sort your first column by the value that you want to use as a unique records key, [make that sorting permanent](transforming#edit-rows), then blank down all the duplicates in that column. OpenRefine will retain the first unique value and erase the rest. Then you can switch from “Show as rows” to “Show as records” and OpenRefine will associate rows to each other based on the remaining values in the first column. - -Be careful that your data is sorted properly before you begin blanking down - not just the first column but other columns you may want to have in a certain order. For example, you may have multiple identical entries in the first column, one with a value in the second column and one with an empty cell in the second column. In this case you want the row with the second-column value to come first, so that you can clean up empty rows later, once you blank down. - -If, conversely, you’ve received data with empty cells because it was already in something akin to records mode, you can fill down information to the rest of the rows. This will duplicate whatever value exists in the topmost cell with a value: if the first row in the record is blank, it will take information from the next cell, or the cell after that, until it finds a value. The blank cells above this will remain blank. - -## Split multi-valued cells {#split-multi-valued-cells} - -Splitting cells with more than one value in them is a common way to get your data from single rows into [multi-row records](exploring#rows-vs-records). Survey data, for example, frequently allows respondents to “Select all that apply,” or an inventory list might have items filed under more than one category. - -You can split a column based on any character or series of characters you input, such as a semi-colon (;) or a slash (/). The default is a comma. Splitting based on a separator will remove the separator characters, so you may wish to include a space with your separator (; ) if it exists in your data. - -You can use [expressions](expressions) to design the point at which a cell should split itself into two or more rows. This can be used to identify special characters or create more advanced evaluations. You can split on a line-break by entering `\n` and checking the “[regular expression](expressions#regular-expressions)” checkbox. - -Regular expressions can be useful if the split is not straightforward: say, if a capital letter (`[A-Z]`) indicates the beginning of a new string, or if you need to _not_ always split on a character that appears in both the strings and as a separator. Remember that this will remove all the matching characters. - -You can also split based on the lengths of the strings you expect to find. This can be useful if you have predictable data in the cells: for example, a 10-digit phone number, followed by a space, followed by another 10-digit phone number. Any characters past the explicit length you’ve specified will be discarded: if you split by “11, 10” any characters that may come after the 21st character will disappear. If some cells only have one phone number, you will end up with blank rows. - -If you have data that should be split into multiple columns instead of multiple rows, see [Split into several columns](columnediting#split-into-several-columns). - -## Join multi-valued cells {#join-multi-valued-cells} - -Joining will reverse the “split multi-valued cells” operation, or join up information from multiple rows into one row. All the strings will be compressed into the topmost cell in the record, in the order they appear. A window will appear where you can set the separator; the default is a comma and a space (, ). This separator is optional. We suggest the separator | as a sufficiently rare character. - -## Cluster and edit {#cluster-and-edit} - -Creating a facet on a column is a great way to look for inconsistencies in your data; clustering is a great way to fix those inconsistencies. Clustering uses a variety of comparison methods to find text entries that are similar but not exact, then shares those results with you so that you can merge the cells that should match. Where editing a single cell or text facet at a time can be time-consuming and difficult, clustering is quick and streamlined. - -Clustering always requires the user to approve each suggested edit - it will display values it thinks are variations on the same thing, and you can select which version to keep and apply across all the matching cells (or type in your own version). - -OpenRefine will do a number of cleanup operations behind the scenes in order to do its analysis, but only the merges you approve will modify your data. Understanding those different behind-the-scenes cleanups can help you choose which clustering method will be more accurate and effective. - -You can start the process in two ways: using the dropdown menu on your column, select Edit cellsCluster and edit…; or create a text facet and then press the “Cluster” button that appears in the facet box. - -![A screenshot of the Clustering window.](/img/cluster.png) - -The clustering pop-up window will take a small amount of time to analyze your column, and then make some suggestions based on the clustering method currently active. - -For each cluster identified, you can pick one of the existing values to apply to all cells, or manually type in a new value in the text box. And, of course, you can choose not to cluster them at all. OpenRefine will keep analyzing every time you make a change, with Merge selected & re-cluster, and you can work through all the methods this way. - -You can also export the currently identified clusters as a JSON file, or close the window with or without applying your changes. You can also use the histograms on the right to narrow down to, for example, clusters with lots of matching rows, or clusters of long or short values. - -### Clustering methods {#clustering-methods} - -You don’t need to understand the details behind each clustering method to apply them successfully to your data. The order in which these methods are presented in the interface and on this page is the order we recommend - starting with the most strict rules and moving to the most lax, which require more human supervision to apply correctly. - -The clustering pop-up window offers you a variety of clustering methods: - -* key collision - * fingerprint - * ngram-fingerprint - * metaphone3 - * cologne-phonetic - * Daitch-Mokotoff - * Beider-Morse -* nearest neighbor - * levenshtein - * ppm - -#### Key collision {#key-collision} - -**Key collisions** are very fast and can process millions of cells in seconds: - -**Fingerprinting** is the least likely to produce false positives, so it’s a good place to start. It does the same kind of data-cleaning behind the scenes that you might think to do manually: fix whitespace into single spaces, put all uppercase letters into lowercase, discard punctuation, remove diacritics (e.g. accents) from characters, split up all strings (words) and sort them alphabetically (so “Zhenyi, Wang” becomes “wang zhenyi”). - -**N-gram fingerprinting** allows you to set the _n_ value to whatever number you’d like, and will create n-grams of _n_ size (after doing some cleaning), alphabetize them, then join them back together into a fingerprint. For example, a 1-gram fingerprint will simply organize all the letters in the cell into alphabetical order - by creating segments one character in length. A 2-gram fingerprint will find all the two-character segments, remove duplicates, alphabetize them, and join them back together (for example, “banana” generates “ba an na an na,” which becomes “anbana”). - -This can help match cells that have typos, or incorrect spaces (such as matching “lookout” and “look out,” which fingerprinting itself won’t identify because it separates words). The higher the _n_ value, the fewer clusters will be identified. With 1-grams, keep an eye out for mismatched values that are near-anagrams of each other (such as “Wellington” and “Elgin Town”). - -##### Phonetic clustering {#phonetic-clustering} - -The next four methods are phonetic algorithms: they identify letters that sound the same when pronounced out loud, and assess text values based on that (such as knowing that a word with an “S” might be a mistype of a word with a “Z”). They are great for spotting mistakes made by not knowing the spelling of a word or name after hearing it spoken aloud. - -**Metaphone3 fingerprinting** is an English-language phonetic algorithm. For example, “Reuben Gevorkiantz” and “Ruben Gevorkyants” share the same phonetic fingerprint in English. - -**Cologne fingerprinting** is another phonetic algorithm, but for German pronunciation. - -**Daitch-Mokotoff** is a phonetic algorithm for Slavic and Yiddish words, especially names. **Baider-Morse** is a version of Daitch-Mokotoff that is slightly more strict. - -Regardless of the language of your data, applying each of them might find different potential matches: for example, Metaphone clusters “Cornwall” and “Corn Hill” and “Green Hill,” while Cologne clusters “Greenvale” and “Granville” and “Cornwall” and “Green Wall.” - -#### Nearest neighbor {#nearest-neighbor} - -**Nearest neighbor** clustering methods are slower than key collision methods. They allow the user to set a radius - a threshold for matching or not matching. OpenRefine uses a “blocking” method first, which sorts values based on whether they have a certain amount of similarity (the default is “6” for a six-character string of identical characters) and then runs the nearest-neighbor operations on those sorted groups. - -We recommend setting the block number to at least 3, and then increasing it if you need to be more strict (for example, if every value with “river” is being matched, you should increase it to 6 or more). Note that bigger block values will take much longer to process, while smaller blocks may miss matches. Increasing the radius will make the matches more lax, as bigger differences will be clustered. - -**Levenshtein distance** counts the number of edits required to make one value perfectly match another. As in the key collision methods above, it will do things like change uppercase to lowercase, fix whitespace, change special characters, etc. Each character that gets changed counts as 1 “distance.” “New York” and “newyork” have an edit distance value of 3 (“N” to “n”; “Y” to “y”; remove the space). It can do relatively advanced edits, such as understand the distance between “M. Makeba” and “Miriam Makeba” (5), but it may create false positives if these distances are greater than other, simpler transformations (such as the one-character distance to “B. Makeba,” another person entirely). - -**PPM (Prediction by Partial Matching)** uses compression to see whether two values are similar or different. In practice, this method is very lax even for small radius values and tends to generate many false positives, but because it operates at a sub-character level it is capable of finding substructures that are not easily identifiable by distances that work at the character level. So it should be used as a “last resort” clustering method. It is also more effective on longer strings than on shorter ones. - -For more of the theory behind clustering, see [Clustering In Depth](https://github.com/OpenRefine/OpenRefine/wiki/Clustering-In-Depth). - -## Replace {#replace} - -OpenRefine provides a find/replace function for you to edit your data. Selecting Edit cellsReplace will bring up a simple window where you can input a string to search and a string to replace it with. You can set case-sensitivity, and set it to only select whole words, defined by a string with spaces or punctuation around it (to prevent, for example, “house” selecting the “house” part of “doghouse”). You can use [regular expressions](expressions#regular-expressions) in this field. You may wish to preview the results of this operation by testing it with a [Text filter](facets#text-filter) first. - -You can also perform a sort of find/replace operation by editing one cell, and selecting “apply to all identical cells.” - -## Edit one cell at a time {#edit-one-cell-at-a-time} - -You can edit individual cells by hovering your mouse over that cell. You should see a tiny blue link labeled “edit.” Click it to edit the cell. That pops up a window with a bigger text field for you to edit. You can change the [data type](exploring#data-types) of that cell, and you can apply these changes to all identical cells (in the same column), using this pop-up window. - -You will likely want to avoid doing this except in rare cases - the more efficient means of improving your data will be through automated and bulk operations. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/columnediting.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/columnediting.md deleted file mode 100644 index 754a61c62..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/columnediting.md +++ /dev/null @@ -1,124 +0,0 @@ ---- -id: columnediting -title: Column editing -sidebar_label: Column editing ---- - -## Overview {#overview} - -Column editing contains some of the most powerful data-improvement methods in OpenRefine. The operations in the Edit column menu involve using one column of data to add entirely new columns and fields to your dataset. - -## Splitting or joining {#splitting-or-joining} - -Many users find that they frequently need to make their data more granular: for example, splitting a “Firstname Lastname” column into two columns, one for first names and one for last names. The reverse is also often true: you may have several columns of category values that you want to join into one “category” column. -. -### Split into several columns {#split-into-several-columns} - -![A screenshot of the settings window for splitting columns.](/img/columnsplit.png) - -You can find this operation at Edit columnSplit into several columns.... Splitting one column into several columns requires you to identify the character, string lengths, or evaluating expression you want to split on. Just like [splitting multi-valued cells into rows](cellediting#split-multi-valued-cells), splitting cells into multiple columns will remove the separator character or string you indicate. Splitting by lengths will discard any information that comes after the specified total length. - -You can also specify a maximum number of new columns to be made: separator characters after this limit will be ignored, and the remaining characters will end up in the last column. - -New columns will be named after the original column, with a number: “Location 1,” “Location 2,” etc. You can choose to remove the original column with this operation, and you can have [data types](exploring#data-types) identified where possible. This function will work best with converting strings to numbers, and may not work with [dates](exploring#dates). - -### Join columns {#join-columns} - -![A screenshot of the settings window for joining columns.](/img/columnjoin.png) - -You can join columns by selecting Edit columnJoin columns.... All the columns currently in your dataset will appear in the pop-up window. You can select or un-select all the columns you want to join, and drag columns to put them in the order you want to join them in. You will define a separator character (optional) and define a string to insert into empty cells (nulls). - -The joined data will appear in the column you originally selected, or you can create a new column for this content and specify a name. You can delete all the columns that were used in this join operation. - -## Add column based on this column {#add-column-based-on-this-column} - -Selecting Edit columnAdd column based on this column... will open up an [expressions](expressions) window where you can transform the data from this column (using `value`), or write a more complex expression that takes information from any number of columns or from external sources. - -Expressions used in this operation will rely on your knowledge of variables. You can learn more in the [Expressions section on variables](expressions#variables). - -The simplest way to use this operation is simply leave the default `value` in the expression field, to create an exact copy of your column. For a column of [reconciled data](reconciling), you can use the variable `cell` instead, to copy both the original string and the existing reconciliation data. This will include matched values, candidates, and new items. - -One useful expression is to create a column based on concatenating (merging) two other columns. Select either of the source columns, choose Edit columnAdd column based on this column..., name your new column, and use the following format in the expression window: - -``` -cells["Column 1"].value + cells["Column 2"].value -``` - -If your column names do not contain spaces, you can use the following format instead: - -``` -cells.Column1.value + cells.Column2.value -``` - -If you are in records mode instead of rows mode, you can concatenate using the following format: - -``` -row.record.cells.Column1.value + row.record.cells.Column2.value -``` - -You may wish to add separators or spaces, or modify your input during this operation with more advanced expressions. - -## Add column by fetching URLs {#add-column-by-fetching-urls} - -Through the Add column by fetching URLs function, OpenRefine supports the ability to fetch HTML or data from web pages or services. In this operation you will be building URL strings based on your column of data, by using `value` to insert a relevant substring. Your chosen column needs to contains parts of paths to valid HTML pages or files online. - -If you have a column of URLs and want to fetch the information that they point to, you can simply run the expression as `value`. If your column has, for example, unique identifiers for Wikidata entities (numerical values starting with Q), you can download the JSON-formatted metadata about each entity with - -``` -"https://www.wikidata.org/wiki/Special:EntityData/" + value + ".json" -``` - -or whatever metadata format you prefer. Information about the format options in Wikidata can be found [here](https://www.wikidata.org/wiki/Wikidata:Data_access). The service you are fetching data from may have similar documentation on its provided options. - -![A screenshot of the settings window for fetching URLs.](/img/fetchingURLs.png) - -This service is more useful when getting metadata files instead of HTML, but you may wish to work with a page’s entire HTML contents and then parse out information from that. - -:::caution -Be aware that the fetching process can take quite some time and that servers may not want to fulfill hundreds or thousands of page requests in seconds. Fetching allows you to set a “throttle delay” which determines the amount of time between requests. The default is 5 seconds per row in your dataset (5000 milliseconds). We recommend leaving this at 1000 or greater. -::: - -Note the following: -* Before pressing “OK,” copy and paste a URL or two from the preview and test them in another browser tab to make sure they work. -* In some situations you may need to set [HTTP request headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers). To set these, click the small “Show” button next to “HTTP headers to be used when fetching URLs” in the settings window. The authorization credentials get logged in your operation history in plain text, which may be a security concern for you. You can set the following request headers: - * [User-Agent](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent) - * [Accept](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept) - * [Authorization](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization) - -### Common errors {#common-errors} - -When OpenRefine attempts to fetch information from a web service, it can fail in a variety of ways. The following information is meant to help troubleshoot and fix problems encountered when using this function. - -First, make sure that your fetching operation is storing errors (check “store error”). Then run the fetch and look at the error messages. - -**“HTTP error 403 : Forbidden”** can be simply down to you not having access to the URL you are trying to use. If you can access the same URL with your browser, the remote site may be blocking OpenRefine because it doesn't recognize its request as valid. Changing the [User-Agent](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent) request header may help. If you believe you should have access to a site but are “forbidden,” you may wish to contract the administrators. - -**“HTTP error 404 : Not Found”** indicates that the information you are requesting does not exist, perhaps due to a problem with your cell values if it only happening in certain rows. - -**“HTTP error 500 : Internal Server Error”** indicates the remote server is having a problem filling your request. You may wish to simply wait and try again later, or double-check the URLs. - -**“error: javax.net.ssl.SSLHandshakeException: Received fatal alert: handshake_failure”** can occur when you are trying to retrieve information over HTTPS but the remote site is using an encryption not supported by the Java virtual machine being used by OpenRefine. - -You can check which encryption methods are supported by your OpenRefine/Java installation by using a service such as **How's my SSL**. Add the URL `https://www.howsmyssl.com/a/check` to an OpenRefine cell and run “Add column by fetching URLs” on it, which will provide a description of the SSL client being used. - -You can try installing additional encryption supports by installing the [Java Cryptography Extension](https://www.oracle.com/java/technologies/javase-jce8-downloads.html). -Note that for Mac users and for Windows users with the OpenRefine installation with bundled JRE, these updated cipher suites need to be dropped into the Java install within the OpenRefine application: - -* On Mac, it will look something like `/Applications/OpenRefine.app/Contents/PlugIns/jdk1.8.0_60.jdk/Contents/Home/jre/lib/security`. -* On Windows: `\server\target\jre\lib\security`. - -**“javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed”** can appear when the remote site is using an HTTPS certificate not trusted by your local Java installation. You will need to make sure that the certificate, or (more likely) the root certificate, is trusted. - -The list of trusted certificates is stored in an encrypted file called `cacerts` in your local Java installation. This can be read and updated by a tool called “keytool.” You can find directions on how to add a security certificate to the list of trusted certificates for a Java installation [here](http://magicmonster.com/kb/prg/java/ssl/pkix_path_building_failed.html) and [here](http://javarevisited.blogspot.co.uk/2012/03/add-list-certficates-java-keystore.html). - -Note that for Mac users and for Windows users with the OpenRefine installation with bundled JRE, the `cacerts` file within the OpenRefine application needs to be updated. - -* On Mac, it will look something like `/Applications/OpenRefine.app/Contents/PlugIns/jdk1.8.0_60.jdk/Contents/Home/jre/lib/security/cacerts`. -* On Windows: `\server\target\jre\lib\security\`. - -## Renaming, removing, and moving {#renaming-removing-and-moving} - -Every column's Edit column dropdown contains options to move it (to the beginning, end, left, or right), rename it, and delete it. -These operations can be undone, but a removed column cannot be restored later if you keep modifying your data. If you wish to temporarily hide a column, go to [View](sortview#view)Collapse this column instead. - -Be cautious about moving columns in [records mode](cellediting#rows-vs-records): if you change the first column in your dataset (the key column), your records may change in unintended ways. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/exploring.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/exploring.md deleted file mode 100644 index 23bafc1a7..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/exploring.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -id: exploring -title: Exploring data -sidebar_label: Overview ---- - -## Overview {#overview} - -OpenRefine offers lots of features to help you learn about your dataset, even if you don’t change a single character. In this section we cover different ways for sorting through, filtering, and viewing your data. - -Unlike spreadsheets, OpenRefine doesn’t store formulas and display the output of those calculations; it only shows the value inside each cell. It doesn’t support cell colors or text formatting. - -## Data types {#data-types} - -Each piece of information (each cell) in OpenRefine is assigned a data type. Some file formats, when imported, can set data types that are recognized by OpenRefine. Cells without an associated data type on import will be considered a “string” at first, but you can have OpenRefine convert cell contents into other data types later. This is set at the cell level, not at the column level. - -You can see data types in action when you preview a new project: check the box next to Attempt to parse cell text into numbers, and cells will be converted to the “number” data type based on their contents. You’ll see numbers change from black text to green if they are recognized. - -The data type will determine what you can do with the value. For example, if you want to add two values together, they must both be recognized as the number type. - -You can check data types at any time by: -* clicking “edit” on a single cell (where you can also edit the type) -* creating a Custom Text Facet on a column, and inserting `type(value)` into the Expression field. This will generate the data type in the preview, and you can facet by data type if you press OK. - -The data types supported are: -* string (one or more text characters) -* number (one or more characters of numbers only) -* boolean (values of “true” or “false”) -* [date](#dates) (ISO-8601-compliant extended format with time in UTC: YYYY-MM-DDTHH:MM:SSZ) - -OpenRefine recognizes two further data types as a result of its own processes: -* error -* null - -An “error” data type is created when the cell is storing an error generated during a transformation in OpenRefine. - -A “null” data type is a special type that means “this cell has no value.” It’s distinct from cells that have values such as “0” or “false”, or cells that look empty but have whitespace in them, or cells that contain empty strings. When you use `type(value)`, it will show you that the cell’s value is “null” and its type is “undefined.” You can opt to [show “null” values](sortview#showhide-null), by going to AllViewShow/Hide ‘null’ values in cells. - -Changing a cell's data type is not the same operation as transforming its contents. For example, using a column-wide transform such as TransformCommon transformsTo date may not convert all values successfully, but going to an individual cell, clicking “edit”, and changing the data type can successfully convert text to a date. These operations use different underlying code. Learn more about date formatting and transformations in the next section. - -To transform data from one type to another, see [Transforming data](cellediting#data-type-transforms) for information on using common tranforms, and see [Expressions](expressions) for information on using [toString()](grelfunctions#tostringo-string-format-optional), [toDate()](grelfunctions#todateo-b-monthfirst-s-format1-s-format2-), and other functions. - - -### Dates {#dates} - -A “date” type is created when a column is [transformed into dates](transforming#to-date), when an expression is used to [convert cells to dates](grelfunctions#todateo-b-monthfirst-s-format1-s-format2-) or when individual cells are set to have the data type “date”. - -Date-formatted data in OpenRefine relies on a number of conversion tools and standards. For something to be considered a date in OpenRefine, it will be converted into the ISO-8601-compliant extended format with time in UTC: YYYY-MM-DDTHH:MM:SSZ. - -When you run Edit cellsCommon transformsTo date, the following column of strings on the left will transform into the values on the right: - -|Input|→|Output| -|---|---|---| -|23/12/2019|→|2019-12-23T00:00:00Z| -|14-10-2015|→|2015-10-14T00:00:00Z| -|2012 02 16|→|2012-02-16T00:00:00Z| -|August 2nd 1964|→|1964-08-02T00:00:00Z| -|today|→|today| -|never|→|never| - -OpenRefine uses a variety of tools to recognize, convert, and format [dates](exploring#dates) and so some of the values above can be reformatted using other methods. In this case, clicking the “today” cell and editing its data type manually will convert “today” into a value such as “2020-08-14T00:00:00Z”. Attempting the same data-type change on “never” will give you an error message and refuse to proceed. - -You can do more precise conversion and formatting using expressions and arguments based on the state of your data: see the GREL functions reference section on [Date functions](grelfunctions#date-functions) for more help. - -You can convert dates into a more human-readable format when you [export your data using the custom tabular exporter](exporting#custom-tabular-exporter). You are given the option to keep your dates in the ISO 8601 format, to output short, medium, long, or full locale formats, or to specify a custom format. This means that you can format your dates into, for example, MM/DD/YY (the US short standard) with or without including the time, after working with ISO-8601-formatted dates in your project. - -The following table shows some example [date and time formatting styles for the U.S. and French locales](https://docs.oracle.com/javase/tutorial/i18n/format/dateFormat.html): - -|Style |U.S. Locale |French Locale| -|---|---|---| -|Default |Jun 30, 2009 7:03:47 AM |30 juin 2009 07:03:47| -|Short |6/30/09 7:03 AM |30/06/09 07:03| -|Medium |Jun 30, 2009 7:03:47 AM |30 juin 2009 07:03:47| -|Long |June 30, 2009 7:03:47 AM PDT |30 juin 2009 07:03:47 PDT| -|Full |Tuesday, June 30, 2009 7:03:47 AM PDT |mardi 30 juin 2009 07 h 03 PDT| - -## Rows vs. records {#rows-vs-records} - -A row is a simple way to organize data: a series of cells, one cell per column. Sometimes there are multiple pieces of information in one cell, such as when a survey respondent can select more than one response. - -In cases where there is more than one value for a single column in one or more rows, you may wish to use OpenRefine’s records mode: this defines a single record as potentially containing more than one row. From there you can transform cells into multiple rows, each cell containing one value you’d like to work with. - -Generally, when you import some data, OpenRefine reads that data in row mode. From the project screen, you can convert the project into records mode. OpenRefine remembers this action and will present you with records mode each time you open the project from then on. - -OpenRefine understands records based on the content of the first column, what we call the “key column.” Splitting a row into a multi-row record will base all association on the first column in your dataset. - -If you have more than one column to split out into multiple rows, OpenRefine will keep your data associated with its original record, and associate subgroups based on the top-most row in each group. - -You can imagine the structure as a tree with many branches, all leading back to the same trunk. - -For example, your key column may be a film or television show, with multiple cast members identified by name, associated to that work. You may have one or more roles listed for each person. The roles are linked to the actors, which are linked to the title. - -|Work|Actor|Role| -|---|---|---| -|The Wizard of Oz|Judy Garland|Dorothy Gale| -||Ray Bolger|"Hunk"| -|||The Scarecrow| -||Jack Haley|"Hickory"| -|||The Tin Man| -||Bert Lahr|"Zeke"| -|||The Cowardly Lion| -||Frank Morgan|Professor Marvel| -|||The Gatekeeper| -|||The Carriage Driver| -|||The Guard| -|||The Wizard of Oz| -||Margaret Hamilton|Miss Almira Gulch| -|||The Wicked Witch of the West| - -Once you are in records mode, you can still move some columns around, but if you move a column to the beginning, you may find your data becomes misaligned. The new key column will sort into records based on empty cells, and values in the old key column will be assigned to the last row in the old record (the key value sitting above those values). - -OpenRefine assigns a unique key behind the scenes, so your records don’t need a unique identifier in the key column. You can keep track of which rows are assigned to each record by the record number that appears under the All column. - -To [split multi-valued cells](transforming#split-multi-valued-cells) and apply other operations that take advantage of records mode, see [Transforming data](transforming). - -Be careful when in records mode that you do not accidentally delete rows based on being blank in one column where there is a value in another. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/exporting.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/exporting.md deleted file mode 100644 index 5dcec2204..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/exporting.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -id: exporting -title: Exporting your work -sidebar_label: Exporting ---- - -## Overview {#overview} - -Once your dataset is ready, you will need to get it out of OpenRefine and into the system of your choice. OpenRefine outputs a number of file formats, can upload your data directly into Google Sheets, and can create or update statements on Wikidata. - -You can also [export your full project data](#export-a-project) so that it can be opened by someone else using OpenRefine (or yourself, on another computer). - -## Export data {#export-data} - -![A screenshot of the Export dropdown.](/img/export-menu.png) - -Many of the options only export data in the current view - that is, with current filters and facets applied. Some will give you the choice to export your entire dataset or just the currently-viewed rows. - -To export data from a project, click the Export dropdown button in the top right corner and pick the format you want. Your options are: - -* Tab-separated value (TSV) or Comma-separated value (CSV) -* HTML-formatted table -* Excel spreadsheet (XLS or XLSX) -* Open Document Format (ODF) spreadsheet (ODS) -* Upload to Google Sheets (requires [Google account authorization](starting#google-sheet-from-drive)) -* [Custom tabular exporter](#custom-tabular-exporter) -* [SQL statement exporter](#sql-statement-exporter) -* [Templating exporter](#templating-exporter), which generates JSON by default - -You can also export reconciled data to Wikidata, or export your Wikidata schema for future use with other OpenRefine projects: - -* [Upload edits to Wikidata](wikidata#upload-edits-to-wikidata) -* [Export to QuickStatements](wikidata#quickstatements-export) (version 1) -* [Export Wikidata schema](wikidata#import-and-export-schema) - -### Custom tabular exporter {#custom-tabular-exporter} - -![A screenshot of the custom tabular content tab.](/img/custom-tabular-exporter.png) - -With the custom tabular exporter, you can choose which of your data to export, the separator you wish to use, and whether you'd like to download the result to your computer or upload it into a Google Sheet. - -On the Content tab, you can drag and drop the columns appearing in the column list to reorder the output. The options for reconciled and date data are applied to each column individually. - -This exporter is especially useful with reconciled data, as you can choose whether you wish to output the cells' original values, the matched values, or the matched IDs. Ouputting “match entity's name”, “matched entity's ID”, or “cell's content” will output, respectively, the contents of `cell.recon.match.name`, `cell.recon.match.id`, and `cell.value`. - -“Output nothing for unmatched cells” will export empty cells for both newly-created matches and cells with no chosen matches. “Link to matched entity's page” will produce hyperlinked text in an HTML table output, but have no effect in other formats. - -At this time, the date-formatting options in this window do not work. You can [keep track of this issue on Github](https://github.com/OpenRefine/OpenRefine/issues/3368). -In the future, you will be able to choose how to [output date-formatted cells](exploring#dates). You can create a custom date output by using [formatting according to the SimpleDateFormat parsing key found here](grelfunctions#todateo-b-monthfirst-s-format1-s-format2-). - -![A screenshot of the custom tabular file download tab.](/img/custom-tabular-exporter2.png) - -On the Download tab, you can generate a preview of how the first ten rows of your dataset will output. If you do not choose one of the file formats on the right, the Download button will generate a text file. On the Upload tab, you can create a new Google Sheet. - -With the Option Code tab, you can copy JSON of your current custom settings to reuse on another export, or you can paste in existing JSON settings to apply to the current project. - -### SQL exporter {#sql-exporter} - -The SQL exporter creates a SQL statement containing the data you’ve exported, which you can use to overwrite or add to an existing database. Choosing ExportSQL exporter will bring up a window with two tabs: one to define what data to output, and another to modify other aspects of the SQL statement, with options to preview and download the statement. - -![A screenshot of the SQL statement content window.](/img/sql-exporter.png) - -The Content tab allows you to craft your dataset into an SQL table. From here, you can choose which columns to export, the data type to export for each (or choose "VARCHAR"), and the maximum character length for each field (if applicable based on the data type). You can set a default value for empty cells after unchecking “Allow null” in one or more columns. - -With this output tool, you can choose whether to output only currently visible rows, or all the rows in your dataset, as well as whether to include empty rows. The option to “Trim column names” will remove their whitespace characters. - -![A screenshot of the SQL statement download window.](/img/sql-exporter2.png) - -The Download tab allows you to finalize your complete SQL statement. - -Include schema means that you will start your statement with the creation of a table. Without that, you will only have an INSERT statement. - -Include content means including the INSERT statement with data from your project. Without that, you will only create empty columns. - -You can include DROP and IF EXISTS if you require them, and set a name for the table to which the statement will refer. - -You can then preview your statement, which will open up a new browser tab/window showing a statement with the first ten rows of your data (if included), or you can save a `.sql` file to your computer. - -### Templating exporter {#templating-exporter} - -If you pick Templating… from the Export dropdown menu, you can “roll your own” exporter. This is useful for formats that we don't support natively yet, or won't support. The Templating exporter generates JSON by default. - -![A screenshot of the Templating exporter generating JSON by default.](/img/templating-exporter.png) - -The Templating Export window allows you to set your own separators, prefix, and suffix to create a complete dataset in the language of your choice. In the Row template section, you can choose which columns to generate from each row by calling them with [variables](expressions#variables). - -This can be used to: -* output [reconciliation data](expressions#reconciliation), such as `cells["ColumnName"].recon.match.name` -* create multiple columns of output from different [member fields](expressions#variables) of a single project column -* employ [expressions](expressions) to modify data for output: for example, `cells["ColumnName"].value.toUppercase()`. - -Anything that appears inside doubled curly braces ({{ }}) is treated as a GREL expression; anything outside is generated as straight text. You can use Jython or Clojure by declaring it at the start: -``` -{{jython:return cells["ColumnName"].value}} -``` - -:::caution -Note that some syntax is different in this tool than elsewhere in OpenRefine: a forward slash must be escaped with a backslash, while other characters do not need escaping. You cannot, at this time, include a closing curly brace (}) anywhere in your expression, or it will cause it to malfunction. -::: - -You can include [regular expressions](expressions#regular-expressions) as usual (inside forward slashes, with any GREL function that accepts them). For example, you could output a version of your cells with punctuation removed, using an expression such as -``` -{{jsonize(cells["ColumnName"].value.replaceChars("/[.!?$&,/]/",""))}} -``` - -You could also simply output a plain-text document inserting data from your project into sentences: for example, "In `{{cells["Year"].value}}` we received `{{cells["RequestCount"].value}}` requests." - -You can use the shorthand `${ColumnName}` (no need for quotes) to insert column values directly. You cannot use this inside an expression, because of the closing curly brace. - -If your projects is in records mode, the Row separator field will insert a separator between records, rather than individual rows. Rows inside a single record will be directly appended to one another as per the content in the Row Template field. - -Once you have created your template, you may wish to save the text you produced in each field, in order to reuse it in the future. Once you click Export OpenRefine will output a simple `.txt` file, and your template will be discarded. - -We have recipes on using the Templating exporter to [produce several different formats](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#12-templating-exporter). - -## Export a project {#export-a-project} - -You can share a project in progress with another computer, a colleague, or with someone who wants to check your history. This can be useful for showing that your data cleanup didn’t distort or manipulate the information in any way. Once you have exported a project, another OpenRefine installation can [import it as a new project](starting#import-a-project). - -You can either save it locally or upload it to Google Drive (which requires you to authorize a Google account). - -:::caution -OpenRefine project archives contain confidential data from previous steps, which will still be accessible to anyone who has the archive. If you are hoping to keep your original dataset hidden for privacy reasons, such as using OpenRefine to anonymize information, do not share your project archive. -::: - -To save your project archive locally: from the Export dropdown, select OpenRefine project archive to file. OpenRefine exports your full project with all of its history. It does not export any current views or applied facets. Existing reconciliation information will be preserved, but the importing computer will need to add the same reconciliation services to keep working with that data. - -OpenRefine exports files in `.tar.gz` format. You can rename the file when you save it; otherwise it will bear the project name. - -To save your project archive to Google Drive: from the Export dropdown, select OpenRefine project archive to Google Drive.... OpenRefine will not share the link with you, only confirm that the file was uploaded. - -## Export operations {#export-operations} - -You can [save and re-apply the history of any project](running#reusing-operations) (all the operations shown in the Undo/Redo tab). This creates JSON that you can save for later reuse on another OpenRefine project. \ No newline at end of file diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/expressions.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/expressions.md deleted file mode 100644 index 3408fbf66..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/expressions.md +++ /dev/null @@ -1,208 +0,0 @@ ---- -id: expressions -title: Expressions -sidebar_label: Overview ---- - -## Overview {#overview} - -You can use expressions in multiple places in OpenRefine to extend data cleanup and transformation. Expressions are available with the following functions: -* Facet: - - * Custom text facet... - * Custom numeric facet… - * Customized facets (click “change” after they have been created to bring up an expressions window) -* Edit cells: - - * Transform… - * Split multi-valued cells… - * Join multi-valued cells… -* Edit column: - - * Split - * Join - * Add column based on this column - * Add column by fetching URLs. - -In the expressions editor window you have the opportunity to select a supported language. The default is [GREL (General Refine Expression Language)](grel); OpenRefine also comes with support for [Clojure](jythonclojure#clojure) and [Jython](jythonclojure#jython). Extensions may offer support for more expressions languages. - -These languages have some syntax differences but support many of the same [variables](#variables). For example, the GREL expression `value.split(" ")[1]` would be written in Jython as `return value.split(" ")[1]`. - -This page is a general reference for available functions, variables, and syntax. For examples that use these expressions for common data tasks, look at the [Recipes section on the wiki](https://github.com/OpenRefine/OpenRefine/wiki/Documentation-For-Users#recipes-and-worked-examples). - -## Expressions {#expressions} - -There are significant differences between OpenRefine's expressions and the spreadsheet formulas you may be used to using for data manipulation. OpenRefine does not store formulas in cells and display output dynamically: OpenRefine’s transformations are one-time operations that can change column contents or generate new columns. These are applied using variables such as `value` or `cell` to perform the same modification to each cell in a column. - -Take the following example: - -|ID|Friend|Age| -|---|---|---| -|1.|John Smith|28| -|2.|Jane Doe|33| - -Were you to apply a transformation to the “friend” column with the expression - -``` - value.split(" ")[1] -``` - -OpenRefine would work through each row, splitting the “friend” values based on a space character. The `value` for row 1 is “John Smith” so the output would be “Smith” (as "[1]" selects the second part of the created output); the `value` for row 2 is “Jane Doe” so the output would be “Doe”. Using variables, a single expression yields different results for different rows. The old information would be discarded; you couldn't get "John" and "Jane" back unless you undid the operation in the [History](running#history-undoredo) tab. - -For another example, if you were to create a new column based on your data using the expression `row.starred`, it would generate a column of true and false values based on whether your rows were starred at that moment. If you were to then star more rows and unstar some rows, that data would not dynamically update - you would need to run the operation again to have current true/false values. - -Note that an expression is typically based on one particular column in the data - the column whose drop-down menu is first selected. Many variables are created to stand for things about the cell in that “base column” of the current row on which the expression is evaluated. There are also variables about rows, which you can use to access cells in other columns. - -## The expressions editor {#the-expressions-editor} - -When you select a function that accepts expressions, you will see a window overlay the screen with what we call the expressions editor. - -![The expressions editor window with a simple expression: value + 10.](/img/expression-editor.png) - -The expressions editor offers you a field for entering your formula and shows you a preview of its transformation on your first few rows of cells. - -There is a dropdown menu from which you can choose an expression language. The default at first is GREL; if you begin working with another language, that selection will persist across OpenRefine. Jython and Clojure are also offered with the installation package, and you may be able to add more language support with third-party extensions and customizations. - -There are also tabs for: -* History, which shows you formulas you’ve recently used from across all your projects -* Starred, which shows you formulas from your History that you’ve starred for reuse -* Help, a quick reference to GREL functions. - -Starring formulas you’ve used in the past can be helpful for repetitive tasks you’re performing in batches. - -You can also choose how formula errors are handled: replicate the original cell value, output an error message into the cell, or ouput a blank cell. - -## Regular expressions {#regular-expressions} - -OpenRefine offers several fields that support the use of regular expressions (regex), such as in a Text filter or a Replace… operation. GREL and other expressions can also use regular expression markup to extend their functionality. - -If this is your first time working with regex, you may wish to read [this tutorial specific to the Java syntax that OpenRefine supports](https://docs.oracle.com/javase/tutorial/essential/regex/). We also recommend this [testing and learning tool](https://regexr.com/). - -### GREL-supported regex {#grel-supported-regex} - -To write a regular expression inside a GREL expression, wrap it between a pair of forward slashes (/) much like the way you would in Javascript. For example, in - -``` -value.replace(/\s+/, " ") -``` - -the regular expression is `\s+`, and the syntax used in the expression wraps it with forward slashes (`/\s+/`). Though the regular expression syntax in OpenRefine follows that of Java (normally in Java, you would write regex as a string and escape it like "\\s+"), a regular expression within a GREL expression is similar to Javascript. - -Do not use slashes to wrap regular expressions outside of a GREL expression. - -On the [GREL functions](grelfunctions) page, functions that support regex will indicate that with a “p” for “pattern.” The GREL functions that support regex are: -* [contains](grelfunctions#containss-sub-or-p) -* [replace](grelfunctions#replaces-s-or-p-find-s-replace) -* [find](grelfunctions#finds-sub-or-p) -* [match](grelfunctions#matchs-p) -* [partition](grelfunctions#partitions-s-or-p-fragment-b-omitfragment-optional) -* [rpartition](grelfunctions#rpartitions-s-or-p-fragment-b-omitfragment-optional) -* [split](grelfunctions#splits-s-or-p-sep) -* [smartSplit](grelfunctions#smartsplits-s-or-p-sep-optional) - -### Jython-supported regex {#jython-supported-regex} - -You can also use [regex with Jython expressions](http://www.jython.org/docs/library/re.html), instead of GREL, for example with a Custom Text Facet: - -``` -python import re g = re.search(ur"\u2014 (.*),\s*BWV", value) return g.group(1) -``` - -### Clojure-supported regex {#clojure-supported-regex} - -[Clojure](https://clojure.org/reference/reader) uses the same regex engine as Java, and can be invoked with [re-find](http://clojure.github.io/clojure/clojure.core-api.html#clojure.core/re-find), [re-matches](http://clojure.github.io/clojure/clojure.core-api.html#clojure.core/re-matches), etc. You can use the #"pattern" reader macro as described [in the Clojure documentation](https://clojure.org/reference/other_functions#regex). For example, to get the nth element of a returned sequence, you can use the nth function: - -``` -clojure (nth (re-find #"\u2014 (.*),\s*BWV" value) 1) -``` - -## Variables {#variables} - -Most OpenRefine variables have attributes: aspects of the variables that can be called separately. We call these attributes “member fields” because they belong to certain variables. For example, you can query a record to find out how many rows it contains with `row.record.rowCount`: `rowCount` is a member field specific to the `record` variable, which is a member field of `row`. Member fields can be called using a dot separator, or with square brackets (`row["record"]`). The square bracket syntax is also used for variables that can call columns by name, for example, `cells["Postal Code"]`. - -|Variable |Meaning | -|-|-| -| `value` | The value of the cell in the current column of the current row (can be null) | -| `row` | The current row | -| `row.record` | One or more rows grouped together to form a record | -| `cells` | The cells of the current row, with fields that correspond to the column names (or row.cells) | -| `cell` | The cell in the current column of the current row, containing value and other attributes | -| `cell.recon` | The cell's reconciliation information returned from a reconciliation service or provider | -| `rowIndex` | The index value of the current row (the first row is 0) | -| `columnName` | The name of the current cell's column, as a string | - -### Row {#row} - -The `row` variable itself is best used to access its member fields, which you can do using either a dot operator or square brackets: `row.index` or `row["index"]`. - -|Field |Meaning | -|-|-| -| `row.index` | The index value of the current row (the first row is 0) | -| `row.cells` | The cells of the row, returned as an array | -| `row.columnNames` | An array of the column names of the project. This will report all columns, even those with null cell values in that particular row. Call a column by number with `row.columnNames[3]` | -| `row.starred` | A boolean indicating if the row is starred | -| `row.flagged` | A boolean indicating if the row is flagged | -| `row.record` | The [record](#record) object containing the current row | - -For array objects such as `row.columnNames` you can preview the array using the expressions window, and output it as a string using `toString(row.columnNames)` or with something like: - -``` -forEach(row.columnNames,v,v).join("; ") -``` - -### Cells {#cells} - -The `cells` object is used to call information from the columns in your project. For example, `cells.Foo` returns a [cell](#cell) object representing the cell in the column named “Foo” of the current row. If the column name has spaces, use square brackets, e.g., `cells["Postal Code"]`. To get the corresponding column's value inside the `cells` variable, use `.value` at the end, for example, `cells["Postal Code"].value`. There is no `cells.value` - it can only be used with member fields. - -### Cell {#cell} - -A `cell` object contains all the data of a cell and is stored as a single object. - -You can use `cell` on its own in the expressions editor to copy all the contents of a column to another column, including reconciliation information. Although the preview in the expressions editor will only show a small representation (“[object Cell]”), it will actually copy all the cell's data. Try this with Edit ColumnAdd Column based on this column .... - -|Field |Meaning |Member fields | -|-|-|-| -| `cell` | An object containing the entire contents of the cell | .value, .recon, .errorMessage | -| `cell.value` | The value in the cell, which can be a string, a number, a boolean, null, or an error | | -| `cell.recon` | An object encapsulating reconciliation results for that cell | See the [reconciliation](expressions#reconciliation) section | -| `cell.errorMessage` | Returns the message of an *EvalError* instead of the error object itself (use value to return the error object) | .value | - -### Reconciliation {#reconciliation} - -Several of the fields here provide the data used in [reconciliation facets](reconciling#reconciliation-facets). You must type `cell.recon`; `recon` on its own will not work. - -|Field|Meaning |Member fields | -|-|-|-| -| `cell.recon.judgment` | A string: either “matched”, "new”, "none” | | -| `cell.recon.judgmentAction` | A string: either "single” or “similar” (or “unknown”) | | -| `cell.recon.judgmentHistory` | A number, the epoch timestamp (in milliseconds) of your judgment | | -| `cell.recon.matched` | A boolean, true if judgment is “matched” | | -| `cell.recon.match` | The recon candidate that has been matched against this cell (or null) | .id, .name, .type | -| `cell.recon.best` | The highest scoring recon candidate from the reconciliation service (or null) | .id, .name, .type, .score | -| `cell.recon.features` | An array of reconciliation features to help you assess the accuracy of your matches | .typeMatch, .nameMatch, .nameLevenshtein, .nameWordDistance | -| `cell.recon.features.typeMatch` | A boolean, true if your chosen type is “matched” and false if not (or “(no type)” if unreconciled) | | -| `cell.recon.features.nameMatch` | A boolean, true if the cell and candidate strings are identical and false if not (or “(unreconciled)”) | | -| `cell.recon.features.nameLevenshtein` | A number representing the [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance): larger if the difference is greater between value and candidate | | -| `cell.recon.features.nameWordDistance` | A number based on the [word similarity](reconciling#reconciliation-facets) | | -| `cell.recon.candidates` | An array of the top 3 candidates (default) | .id, .name, .type, .score | - -The `cell.recon.candidates` and `cell.recon.best` objects have a few deeper fields: `id`, `name`, `type`, and `score`. `type` is an array of type identifiers for a list of candidates, or a single string for the best candidate. - -Arrays such as `cell.recon.candidates` and `cell.recon.candidates.type` can be joined into lists and stored as strings with something like: -``` -forEach(cell.recon.candidates,v,v.name).join("; ") -``` - -### Record {#record} - -A `row.record` object encapsulates one or more rows that are grouped together, when your project is in records mode. You must call it as `row.record`; `record` will not return values. - -|Field|Meaning | -|-|-| -| `row.record.index` | The index of the current record (starting at 0) | -| `row.record.cells` | An array of the [cells](#cells) in the given column of the record | -| `row.record.fromRowIndex` | The row index of the first row in the record | -| `row.record.toRowIndex` | The row index of the last row in the record + 1 (i.e. the next record) | -| `row.record.rowCount` | A count of the number of rows in the record | - -For example, you can facet by number of rows in each record by creating a Custom Numeric Facet (or a Custom Text Facet) and entering `row.record.rowCount`. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/facets.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/facets.md deleted file mode 100644 index b63df21e8..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/facets.md +++ /dev/null @@ -1,325 +0,0 @@ ---- -id: facets -title: Exploring facets -sidebar_label: Facets ---- - -## Overview {#overview} - -Facets are one of OpenRefine’s strongest features - that’s where the diamond logo comes from! - -Faceting allows you to look for patterns and trends. Facets are essentially aspects or angles of data variance in a given column. For example, if you had survey data where respondents indicated one of five responses from “Strongly agree” to “Strongly disagree,” those five responses make up a text facet, showing how many people selected each option. - -Faceted browsing gives you a big-picture look at your data (do they agree or disagree?) and also allows you to filter down to a specific subset to explore it more (what do people who disagree say in other responses?). - -Typically, you create a facet on a particular column. That facet selection appears on the left, in the Facet/Filter tab, and you can click on a displayed facet to view all the records that match. You can also “exclude” the facet, to view every record that does _not_ match, and you can select more than one facet by clicking “include.” - - -### An example {#an-example} - -You can learn about facets and filtering with the following example. You can copy the following table and paste it using the Clipboard method of starting a project if you would like to try it yourself. - -We collected a list of the [10 most populous cities from Wikidata](https://w.wiki/3Em), using an example query of theirs. We removed the GPS coordinates and added the country. - -| cityLabel | population | countryLabel | -|-|-|-| -| Shanghai | 23390000 | People's Republic of China | -| Beijing | 21710000 | People's Republic of China | -| Lagos | 21324000 | Nigeria | -| Dhaka | 16800000 | Bangladesh | -| Mumbai | 15414288 | India | -| Istanbul | 14657434 | Turkey | -| Tokyo | 13942856 | Japan | -| Tianjin | 13245000 | People's Republic of China | -| Guangzhou | 13080500 | People's Republic of China | -| São Paulo | 12106920 | Brazil | - -If we want to see which countries have the most populous cities, we can create a text facet on the “countryLabel” column and OpenRefine will generate a list of all the different strings used in these cells. - -We will see in the sidebar that the countries identified are displayed, along with the number of matches (the “count”). We can sort this list alphabetically or by the count. If you sort by count at the top of the facet window, you’ll learn which countries hold the most populous cities. - -|Facet|Count| -|---|---| -|People's Republic of China|4| -|Bangladesh|1| -|Brazil|1| -|India|1| -|Japan|1| -|Nigeria|1| -|Turkey|1| - -If we want to learn more about a particular country, we can click on its appearance in the facet sidebar. This narrows our dataset down temporarily to only rows matching that facet. - -You’ll see the “10 rows” indicator change to “4 matching rows (10 total)” if you click on “People’s Republic of China”. In the data grid, you’ll see fewer rows: only the ones matching your current filter. Each row will maintain its original numbering, though - in this case, rows #1, 2, and 8. - -If you want to go back to the original dataset, click Reset All or the small “exclude” text next to the facet. If you want to view the most populous cities in both China and India, click “include” next to each facet. Now you’ll see 5 rows - #1, 2, 5, 8, 9. - -We can also explore our data using the population information. In this case, because population is a number, we can create a numeric facet. This will give us the ability to explore by range rather than by exact matching values. - -With the numeric facet, we are given a scale from the smallest to the largest value in the column. We can drag the range minimum and maximum to narrow the results. In this case, if we narrow down to only cities with more than 20 million in population, we get 3 matching rows out of the original 10. - -When you look back at the text facet display of country names, you should see a smaller list with a reduced count: OpenRefine is now displaying the facets of the 3 matching rows, not the total dataset of 10 rows. - -We can combine these facets - say, by narrowing to only the Chinese cities with populations greater than 20 million - simply by clicking in both. You should see 2 matching rows for both these criteria. - -### Things to know about facets {#things-to-know-about-facets} - -When you have facets applied, you will see “matching rows” in the [project grid header](running#project-grid-header). If you click Export and copy your data out of OpenRefine while facets are active, many of the exporting options will only export the matching rows, not all the rows in your project. - -OpenRefine has several default facets, which you’ll learn about below. The most powerful facets are the ones designed by you - custom facets, written using [expressions](expressions) to transform the data behind the scenes and help you narrow down to precisely what you’re looking for. - -Facets are not saved in the project along with the data. But you can save a link to the current state of the application. Find the [Permalink](running#the-project-bar) next to the project’s name. - -You can modify any facet expression by clicking the “change” button to the right of the column name in the facet sidebar. - -Facet boxes that appear in the sidebar can be resized and rearranged. You can drag and drop the title bar of each box to reorder them, and drag on the bottom bar of text facet boxes. - -## Text facet {#text-facet} - -A text facet can be generated on any column with the “text” data type. Select the column dropdown and go to FacetText facet. The created facet will be sorted alphabetically, and can be sorted by count. - -A text facet is very simple: it takes the total contents of the cells of the column in question and matches them up. It does no guessing about typos or near-matches. - -You can edit any entry that appears in the facet display, by hovering over the facet and clicking the “edit” button that appears. You can then type in a new value manually. This will mass-edit every identical cell in the column. This is a great way to fix typos, whitespace, and other issues that may be affecting the way facets appear. You can also automate the cleanup of facets by using [clustering](transforming#cluster-and-edit): a “Cluster” button is displayed within the facet window. It may be most efficient to cluster cells to one value, and then mass-edit that value to your desired string within the clustering operation window. - -Each text facet shows up to 2,000 choices by default. You can [increase this limit on the Preferences screen](running#preferences) if you need to, which may slow down your browser. If your applied facet has more choices than the current limit, you'll be offered the option to increase the limit, which will permanently edit that preference for you. - -The choices and counts displayed in each facet can be copied as tab-separated values. To do so, click on the "X choices" link near the top left corner of the facet. This can be useful to generate small summary tables of your data. - -![A column of years faceted as text and numbers, and with the count ready to be copied.](/img/yeardata.png) - -## Numeric facet {#numeric-facet} - -![A screenshot of an example numeric facet.](/img/numericfacet.png) - -Whereas a text facet groups unique text values into groups, a numeric facet sorts numbers by their range - smallest to biggest. This displays visually as a histogram, and allows you to set a custom facet within that range. You can drag the minimum and maximum range markers to set a range. OpenRefine snaps to some basic equal-sized divisions - 19 in the example set above. - -You will be offered the option to include blank, non-numeric, and error values in your numeric visualization; these will appear in the visual range as “0” values. - -:::info Numbers as text -You can create a text facet on numeric data, which will treat each entry as a string. This can be useful if you wish, for example, to manually include facets instead of selecting a range, or sort by count, or copy that count. -::: - -## Timeline facet {#timeline-facet} - -![A screenshot of an example timeline facet.](/img/timelinefacet.png) - -Much like a numeric facet, a timeline facet will display as a small histogram with the values sorted: in this case, chronologically. A timeline facet only works on cells formatted as the [“date” data type](exploring#dates). - -The facet appears with a count of blank cells and those with errors, which can help you analyze whether your date cells are correctly converted. - -## Scatterplot facet {#scatterplot-facet} - -A scatterplot is a visual representation of two related sets of numeric data. - -You have the option to generate linear scatterplots (where the X and Y axes show continuous increases) or logarithmic scatterplots (where the X and Y axes show exponential or scaled increases). You can also rotate the plot by 45 degrees in either direction, and you can choose the size of the dot indicating a datapoint. You can make these choices in both the preview and in the facet display. - -A scatterplot facet can be generated on any column. You require two or more number columns to generate scatterplots. Selecting FacetScatterplot facet will create a preview of data plotted from every number-formatted column in your dataset, comparing every column against every other column. Each scatterplot will show in its own square, allowing you to choose which data comparison you would like to analyze further. You can control which columns are on the X and Y axes by rearranging the columns in your dataset. - -![A simple scatterplot of two numeric values.](/img/scatterplot.png) - -When you click on your desired square, that two-column comparison will appear in the facets sidebar. From here, you can drag your mouse to draw a rectangle inside the scatterplot, which will narrow down to just the rows matching the points plotted inside that rectangle (as shown by the rectangle inside the square in the image above). This rectangle can be resized by dragging any of the four edges. To draw a new rectangle, simply click and drag your mouse again. To add more scatterplots to the facet sidebar, re-run this process and select a different square. - -If you have multiple facets applied, plotted points in your scatterplot displays will be greyed out if they are not part of the current matching data subset. If the rectangle you have drawn within a scatterplot display only includes grey dots, you will see no matching rows. - -If you would like to export a scatterplot, OpenRefine will open a new tab with a generated PNG file that you can save. - -## Custom text facet {#custom-text-facet} - -You may want to explore your textual data with modifications that aren't permanent. Creating custom text facets will load your column into memory, transform the data temporarily, and store those transformations inside the facet. - -You can also use custom text facets to analyze numerical data, such as by analyzing a number as a string, or by creating a test that will return “true” and “false” as values. - -Clicking on FacetCustom text facet… will bring up an [expressions](expressions) window where you can enter in a GREL, Jython, or Clojure expression to modify how the facet works. - -A custom text facet operates just like a [text facet](#text-facet) by default. Unlike a text facet, however, you cannot click “edit” on the facets that appear in the sidebar and change the matching cells in your dataset - because what they display is modified, not the original entries. - -For example, you may wish to analyze only the first word in a text field - perhaps the first name in a column of “[First Name] [Last Name]” entries. In this case, you can tell OpenRefine to facet only on the information that comes before the first space: - -``` -value.split(" ")[0] -``` - -In this case, `split()` is creating an array of text strings based on every space in the cells ["Firstname", "Lastname"]. Because arrays number their entries starting with 0, we want the first value, so we ask for `[0]`. (Assuming the first name is one word, not something like “Mary Anne.”) We can do the same splitting and ask for the last name with - -``` -value.split(" ")[1] -``` - -You may want to create a facet that references several columns. For example, let’s say you have two columns, “First Name” and “Last Name”, and you want out how many people have the same initial letter for both names (e.g., Marilyn Monroe, Steven Segal). To do so, create a custom text facet on either column and enter the expression - -``` -cells["First Name"].value[0] == cells["Last Name"].value[0] -``` - -That expression will look for the first letter (the character at index 0) of each entry and compare them. Then it will facet your rows into “true” and “false.” - -You can learn more about text-modification functions on the [Expressions page](expressions). - -## Custom numeric facet {#custom-numeric-facet} - -You may want to explore your numerical data with modifications that aren't permanent. You can also use custom numeric facets to analyze textual data, such as by getting the length of text strings (with `value.length()`), or by analyzing it as though it were formatted as numbers (with `toNumber(value)`). - -If you would like to build your own version of a numeric facet, you can use the Custom Numeric Facet option. Clicking on FacetCustom Numeric Facet… will bring up an [expressions](expressions) window where you can enter in a GREL, Jython, or Clojure expression to modify how the facet works. A custom numeric facet operates just like a [numeric facet](#numeric-facet) by default. - -For example, you may wish to create a numeric facet that rounds your value to the nearest integer, enter - -``` -round(value) -``` - -If you have two columns of numbers and for each row you wish to create a numeric facet only on the larger of the two, enter - -``` -max(cells["Column1"].value, cells["Column2"].value) -``` - -If the numeric values in a column are drawn from a power law distribution, then it's better to group them by their logs: - -``` -value.log() -``` - -If the values are periodic you could take the modulus by the period to understand if there's a pattern: - -``` -mod(value, 7) -``` - -You can learn more about numeric-modification functions on the [Expressions page](expressions). - -## Customized facets {#customized-facets} - -Customized facets have been added to expand the number of default facets users can apply with a single click. They represent some common and useful functions you shouldn’t have to work out using an [expression](expressions). - -All facets that display in the Facet/Filter tab can be edited by clicking on the “change” button to the right of the column title. This brings up the expressions window that will allow you to modify and preview the expression being used. - -### Word facet {#word-facet} - -A Word facet is a simple version of a text facet: it splits up the content of the cells based on spaces, and outputs each character string as a facet: - -``` -value.split(" ") -``` - -This can be useful for exploring the language used in a corpus, looking for common first and last names or titles, or seeing what’s in multi-valued cells you don’t wish to split up. - -Word facet is case-sensitive and only splits by spaces, not by line breaks or other natural divisions. - -### Duplicates facet {#duplicates-facet} - -A Duplicates facet will return only rows that have non-unique values in the column you’ve selected. It will create a facet of “true” and “false” values - true being cells that are not unique, and “false” being cells that are. The actual expression being used is - -``` -facetCount(value, 'value', '[Column]') > 1 -``` - -Duplicates facets are case-sensitive and you may wish to filter out things like leading and trailing whitespace or other hard-to-see issues. You can modify the facet expression, for example, with: - -``` -facetCount(trim(toLowercase(value)), 'trim(toLowercase(value))', 'cityLabel') > 1 -``` - -### Numeric log facet {#numeric-log-facet} - -Logarithmic scales reduce wide-ranging quantities to more compact and manageable ranges. A log transformation can be used to make highly skewed distributions less skewed. If your numerical data is unevenly distributed (say, lots of values in one range, and then a long tail extending off into different magnitudes), a Numeric log facet can represent that range better than a simple numeric facet. It will break these values down into more navigable segments than the buckets of a numeric facet. This facet can make patterns in your data more visible. OpenRefine uses a base-10 log, the “common logarithm.” - -For example, we can look at [this data about the body weight of various mammals](http://wiki.stat.ucla.edu/socr/index.php/SOCR_Data_Brain2BodyWeight): - -|Species|BodyWeight (kg)| -|---|---| -| Newborn_Human | 3.2 | -| Adult_Human | 73 | -| Pithecanthropus_Man | 70 | -| Squirrel | 0.8 | -| Hamster | 0.15 | -| Chimpanzee | 50 | -| Rabbit | 1.4 | -| Dog_(Beagle) | 10 | -| Cat | 4.5 | -| Rat | 0.4 | -| Sperm_Whale | 35000 | -| Turtle | 3 | -| Alligator | 270 | - -Most values will be clustered in the 0-100 range, but 35,000 is many magnitudes above that. A numeric facet will create 36 equal buckets of 1,000 each - containing almost all the cells in the first bucket. A numeric log facet will instead display the data more evenly across the visual range. - -![A screenshot of a numeric facet first and a numeric log facet second.](/img/numericlogfacet.png) - -A 1-bounded numeric log facet can be used if you'd like to exclude all the values below 1 (including zero and negative numbers). - -### Text-length facet {#text-length-facet} - -The Text-length facet returns a numerical value for each cell and plots it on a numeric facet chart. The expression used is - -``` -value.length() -``` - -This can be useful to, for example, look for values that did not successfully split on an earlier split operation, or to validate that data is a certain expected length (such as whether a date in YYYY/MM/DD is eight to ten characters). - -You can also employ a Log of text-length facet that allows you to navigate more easily a wide range of string lengths. This can be useful in the case of web-scraping, where lots of textual data is loaded into single cells and needs to be parsed out. - - -### Unicode character-code facet {#unicode-character-code-facet} - -![A screenshot of the Unicode facet.](/img/unicodefacet.png) - -The Unicode facet identifies and returns [Unicode decimal values](https://en.wikipedia.org/wiki/List_of_Unicode_characters). It generates a list of the Unicode numerical values of each character used in each text cell, which allows you to narrow down and search for special characters, punctuation, and other data formatting issues. - -This facet creates a numerical chart, which offers you the ability to narrow down to a range of numbers. For example, lowercase characters are numbers 97-122, uppercase characters are numbers 65-90, and numerical digits are numbers 48-57. - -### Facet by error {#facet-by-error} - -An error is a data type created by OpenRefine in the process of transforming data. For example, say you had converted a column to the number data type. If one cell had text characters in it, OpenRefine could either output the original text string unchanged or output an error. If you allow errors to be created, you can facet by them later to search for them and fix them. - -![A view of the expressions window with an error converting a string to a number.](/img/error.png) - -To store errors in cells, ensure that you have store error selected for the “On error” option in the expressions window. - -### Facet by null, empty, or blank {#facet-by-null-empty-or-blank} - -Any column can be faceted for [null and/or empty cells](#cell-data-types). These can help you find cells where you want to manually enter content. - -“Blank” means both null values and empty values. All three facets will generate “true” and “false” facets, “true” being blank. - -An empty cell is a cell that is set to contain a string, but doesn’t have any characters in it (a zero-length string). This can be left over from an operation that removed characters, or from manually editing a cell and deleting its contents. - -### Facet by star or flag {#facet-by-star-or-flag} - -Stars and flags offer you the opportunity to mark specific rows for yourself for later focus. Stars and flags persist through closing and opening your project, and thus can provide a different function than using a permalink to persist your facets. Stars and flags can be used in any way you want, although they are designed to help you flag errors and star rows of particular importance. - -You can manually star or flag rows simply by clicking on the icons to the left of each row. - -You can also apply stars or flags to all matching rows by using the All dropdown menu (on the first column) and selecting Edit rowsStar rows or Flag rows. This will create “true” and “false” facets in the Facet/Filter. These operations will modify all matching rows in your current subset. You can unstar or unflag them as well. - -You may wish to create a custom subset of your data through a series of separate faceting activities (rather than successively narrowing down with multiple facets applied). For example, you may wish to: -* apply a facet -* star all the matching rows -* remove that facet -* apply another, unrelated facet -* star all the new matching rows (which will not modify already-starred rows) -* remove that facet -* and then work with all of the cumulative starred rows. - -You can also create a text facet on any column with the expression `row.starred` or `row.flagged`. - -## Text filter {#text-filter} - -Filters allow you to narrow down your data based on whether a given column includes a text string. - -When you choose Text filter a box appears in the Facet/Filter tab that allows you to enter in text. Matching rows will narrow dynamically with every character you enter. You can set the search to be case-sensitive or not, and you can use this box to enter in a regular expression. - -For example, you can enter in “side” as a text filter, and it will return all cells in that column containing “side,” “sideways,” “offside,” etc. - -The text filter field supports [regular expressions](expressions#regular-expressions). For example, you can employ a regular expression to view all properly-formatted emails: - -``` -([a-zA-Z0-9_\-\.\+]+)@([a-zA-Z0-9\-\.]+)\.([a-zA-Z0-9\-]{2,15}) -``` - -You can press “invert” on this facet to then see blank cells or invalid email addresses. - -This filter works differently than facets because it is always active as long as it appears in the sidebar. If you “reset” it, you will delete all the text or expression you have entered. - -You can apply multiple text filters in succession, which will successively narrow your data subset. This can be useful if you apply multiple inverted filters, such as to filter out all rows that respond “yes” or “maybe” and only look at the remaining responses. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/grel.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/grel.md deleted file mode 100644 index 841e031b0..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/grel.md +++ /dev/null @@ -1,154 +0,0 @@ ---- -id: grel -title: General Refine Expression Language -sidebar_label: General Refine Expression Language ---- - -## Basics {#basics} - -GREL (General Refine Expression Language) is designed to resemble Javascript. Formulas use variables and depend on data types to do things like string manipulation or mathematical calculations: - -|Example|Output| -|---|---| -| `value + " (approved)"` | Concatenate two strings; whatever is in the cell gets converted to a string first | -| `value + 2.239` | Add 2.239 to the existing value (if a number); append text "2.239" to the end of the string otherwise | -| `value.trim().length()`     | Trim leading and trailing whitespace of the cell value and then output the length of the result | -| `value.substring(7, 10)` | Output the substring of the value from character index 7, 8, and 9 (excluding character index 10) | -| `value.substring(13)` | Output the substring from index 13 to the end of the string | - -Note that the operator for string concatenation is `+` (not “&” as is used in Excel). - -Evaluating conditions uses symbols such as <, >, *, /, etc. To check whether two objects are equal, use two equal signs (`value=="true"`). - -See the [GREL functions page for a thorough reference](grelfunctions) on each function and its inputs and outputs. Read on below for more about the general nature of GREL expressions. - -## Syntax {#syntax} - -In GREL, functions can use either of these two forms: -* functionName(arg0, arg1, ...) -* arg0.functionName(arg1, ...) - -The second form is a shorthand to make expressions easier to read. It simply pulls the first argument out and appends it to the front of the function, with a dot: - -|Dot notation |Full notation | -|-|-| -| `value.trim().length()` | `length(trim(value))` | -| `value.substring(7, 10)` | `substring(value, 7, 10)` | - -So, in the dot shorthand, the functions occur from left to right in the order of calling, rather than in the reverse order with parentheses. This allows you to string together multiple functions in a readable order. - -The dot notation can also be used to access the member fields of [variables](expressions#variables). For referring to column names that contain spaces (anything not a continuous string), use square brackets instead of dot notation: - -|Example |Description | -|-|-| -| `FirstName.cells` | Access the cell in the column named “FirstName” of the current row | -| `cells["First Name"]` | Access the cell in the column called “First Name” of the current row | - -Square brackets can also be used to get substrings and sub-arrays, and single items from arrays: - -|Example |Description | -|-|-| -| `value[1,3]` | A substring of value, starting from character 1 up to but excluding character 3 | -| `"internationalization"[1,-2]` | Will return “nternationalizati” (negative indexes are counted from the end) | -| `row.columnNames[5]` | Will return the name of the fifth column | - -Any function that outputs an array can use square brackets to select only one part of the array to output as a string (remember that the index of the items in an array starts with 0). - -For example, [partition()](grelfunctions#partitions-s-or-p-fragment-b-omitfragment-optional) would normally output an array of three items: the part before your chosen fragment, the fragment you've identified, and the part after. Selecting only the third part with `"internationalization".partition("nation")[2]` will output “alization” (and so will [-1], indicating the final item in the array). - -## Controls {#controls} - -GREL offers controls to support branching and looping (that is, “if” and “for” functions), but unlike functions, their arguments don't all get evaluated before they get run. A control can decide which part of the code to execute and can affect the environment bindings. Functions, on the other hand, can't do either. Each control decides which of their arguments to evaluate to `value`, and how. - -Please note that the GREL control names are case-sensitive: for example, the isError() control can't be called with iserror(). - -#### if(e, eTrue, eFalse) {#ife-etrue-efalse} - -Expression e is evaluated to a value. If that value is true, then expression eTrue is evaluated and the result is the value of the whole if() expression. Otherwise, expression eFalse is evaluated and that result is the value. - -Examples: - -| Example expression | Result | -| ------------------------------------------------------------------------ | ------------ | -| `if("internationalization".length() > 10, "big string", "small string")` | “big string” | -| `if(mod(37, 2) == 0, "even", "odd")` | “odd” | - -Nested if (switch case) example: - - if(value == 'Place', 'http://www.example.com/Location', - - if(value == 'Person', 'http://www.example.com/Agent', - - if(value == 'Book', 'http://www.example.com/Publication', - - null))) - -#### with(e1, variable v, e2) {#withe1-variable-v-e2} - -Evaluates expression e1 and binds its value to variable v. Then evaluates expression e2 and returns that result. - -| Example expression | Result | -| ------------------------------------------------------------------------------------ | ---------- | -| `with("european union".split(" "), a, a.length())` | 2 | -| `with("european union".split(" "), a, forEach(a, v, v.length()))` | [ 8, 5 ] | -| `with("european union".split(" "), a, forEach(a, v, v.length()).sum() / a.length())` | 6.5 | - -#### filter(e1, v, e test) {#filtere1-v-e-test} - -Evaluates expression e1 to an array. Then for each array element, binds its value to variable v, evaluates expression test - which should return a boolean. If the boolean is true, pushes v onto the result array. - -| Expression | Result | -| ---------------------------------------------- | ------------- | -| `filter([ 3, 4, 8, 7, 9 ], v, mod(v, 2) == 1)` | [ 3, 7, 9 ] | - -#### forEach(e1, v, e2) {#foreache1-v-e2} - -Evaluates expression e1 to an array. Then for each array element, binds its value to variable v, evaluates expression e2, and pushes the result onto the result array. - -| Expression | Result | -| ------------------------------------------ | ------------------- | -| `forEach([ 3, 4, 8, 7, 9 ], v, mod(v, 2))` | [ 1, 0, 0, 1, 1 ] | - -#### forEachIndex(e1, i, v, e2) {#foreachindexe1-i-v-e2} - -Evaluates expression e1 to an array. Then for each array element, binds its index to variable i and its value to variable v, evaluates expression e2, and pushes the result onto the result array. - -| Expression | Result | -| ------------------------------------------------------------------------------- | --------------------------- | -| `forEachIndex([ "anne", "ben", "cindy" ], i, v, (i + 1) + ". " + v).join(", ")` | 1. anne, 2. ben, 3. cindy | - -#### forRange(n from, n to, n step, v, e) {#forrangen-from-n-to-n-step-v-e} - -Iterates over the variable v starting at from, incrementing by the value of step each time while less than to. At each iteration, evaluates expression e, and pushes the result onto the result array. - -#### forNonBlank(e, v, eNonBlank, eBlank) {#fornonblanke-v-enonblank-eblank} - -Evaluates expression e. If it is non-blank, forNonBlank() binds its value to variable v, evaluates expression eNonBlank and returns the result. Otherwise (if e evaluates to blank), forNonBlank() evaluates expression eBlank and returns that result instead. - -Unlike other GREL functions beginning with “for,” forNonBlank() is not iterative. forNonBlank() essentially offers a shorter syntax to achieving the same outcome by using the isNonBlank() function within an “if” statement. - -#### isBlank(e), isNonBlank(e), isNull(e), isNotNull(e), isNumeric(e), isError(e) {#isblanke-isnonblanke-isnulle-isnotnulle-isnumerice-iserrore} - -Evaluates the expression e, and returns a boolean based on the named evaluation. - -Examples: - -| Expression | Result | -| ------------------- | ------- | -| `isBlank("abc")` | false | -| `isNonBlank("abc")` | true | -| `isNull("abc")` | false | -| `isNotNull("abc")` | true | -| `isNumeric(2)` | true | -| `isError(1)` | false | -| `isError("abc")` | false | -| `isError(1 / 0)` | true | - -Remember that these are controls and not functions: you can’t use dot notation (for example, the format `e.isX()` will not work). - -## Constants {#constants} -|Name |Meaning | -|-|-| -| true | The boolean constant true | -| false | The boolean constant false | -| PI | From [Java's Math.PI](https://docs.oracle.com/javase/8/docs/api/java/lang/Math.html#PI), the value of pi (that is, 3.1415...) | \ No newline at end of file diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/grelfunctions.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/grelfunctions.md deleted file mode 100644 index e156d4b3b..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/grelfunctions.md +++ /dev/null @@ -1,531 +0,0 @@ ---- -id: grelfunctions -title: GREL functions -sidebar_label: GREL functions ---- - -## Reading this reference {#reading-this-reference} - -For the reference below, the function is given in full-length notation and the in-text examples are written in dot notation. Shorthands are used to indicate the kind of [data type](exploring#data-types) used in each function: s for string, b for boolean, n for number, d for date, a for array, p for a regex pattern, and o for object (meaning any data type), as well as “null” and “error” data types. - -If a function can take more than one kind of data as input or can output more than one kind of data, that is indicated with more than one letter (as with “s or a”) or with o for object, meaning it can take any type of data (string, boolean, date, number, etc.). - -We also use shorthands for substring (“sub”) and separator string (“sep”). -Optional arguments will say “(optional)”. - -In places where OpenRefine will accept a string (s) or a regex pattern (p), you can supply a string by putting it in quotes. If you wish to use any [regex](expressions#regular-expressions) notation, wrap the pattern in forward slashes. - -## Boolean functions {#boolean-functions} - -###### and(b1, b2, ...) {#andb1-b2-} - -Uses the logical operator AND on two or more booleans to output a boolean. Evaluates multiple statements into booleans, then returns true if all of the statements are true. For example, `(1 < 3).and(1 < 0)` returns false because one condition is true and one is false. - -###### or(b1, b2, ...) {#orb1-b2-} - -Uses the logical operator OR on two or more booleans to output a boolean. For example, `(1 < 3).or(1 > 7)` returns true because at least one of the conditions (the first one) is true. - -###### not(b) {#notb} - -Uses the logical operator NOT on a boolean to output a boolean. For example, `not(1 > 7)` returns true because 1 > 7 itself is false. - -###### xor(b1, b2, ...) {#xorb1-b2-} - -Uses the logical operator XOR (exclusive-or) on two or more booleans to output a boolean. Evaluates multiple statements, then returns true if only one of them is true. For example, `(1 < 3).xor(1 < 7)` returns false because more than one of the conditions is true. - -## String functions {#string-functions} - -###### length(s) {#lengths} - -Returns the length of string s as a number. - -###### toString(o, string format (optional)) {#tostringo-string-format-optional} - -Takes any value type (string, number, date, boolean, error, null) and gives a string version of that value. - -You can use toString() to convert numbers to strings with rounding, using an [optional string format](https://docs.oracle.com/javase/8/docs/api/java/util/Formatter.html). For example, if you applied the expression `value.toString("%.0f")` to a column: - -|Input|Output| -|-|-| -|3.2|3| -|0.8|1| -|0.15|0| -|100.0|100| - -You can also convert dates to strings, using date parsing syntax built in to OpenRefine (see [the toDate() function for details](#todateo-boolean-month_first--format1-format2--)). For example, `value.toString("MMM-dd-yyyy")` would convert the date value [2024-10-15T00:00:00Z] to “Oct-15-2024”. - -Note: In OpenRefine, using toString() on a null cell outputs the string “null”. - -### Testing string characteristics {#testing-string-characteristics} - -###### startsWith(s, sub) {#startswiths-sub} - -Returns a boolean indicating whether s starts with sub. For example, `"food".startsWith("foo")` returns true, whereas `"food".startsWith("bar")` returns false. - -###### endsWith(s, sub) {#endswiths-sub} - -Returns a boolean indicating whether s ends with sub. For example, `"food".endsWith("ood")` returns true, whereas `"food".endsWith("odd")` returns false. - -###### contains(s, sub or p) {#containss-sub-or-p} - -Returns a boolean indicating whether s contains sub, which is either a substring or a regex pattern. For example, `"food".contains("oo")` returns true whereas `"food".contains("ee")` returns false. - -You can search for a regular expression by wrapping it in forward slashes rather than quotes: `"rose is a rose".contains(/\s+/)` returns true. startsWith() and endsWith() can only take strings, while contains() can take a regex pattern, so you can use contains() to look for beginning and ending string patterns. - -### Basic string modification {#basic-string-modification} - -#### Case conversion {#case-conversion} - -###### toLowercase(s) {#tolowercases} - -Returns string s converted to all lowercase characters. - -###### toUppercase(s) {#touppercases} - -Returns string s converted to all uppercase characters. - -###### toTitlecase(s) {#totitlecases} - -Returns string s converted into titlecase: a capital letter starting each word, and the rest of the letters lowercase. For example, `"Once upon a midnight DREARY".toTitlecase()` returns the string “Once Upon A Midnight Dreary”. - -#### Trimming {#trimming} - -###### trim(s) {#trims} - -Returns a copy of the string s with leading and trailing whitespace removed. For example, `" island ".trim()` returns the string “island”. Identical to strip(). - -###### strip(s) {#strips} - -Returns a copy of the string s with leading and trailing whitespace removed. For example, `" island ".strip()` returns the string “island”. Identical to trim(). - -###### chomp(s, sep) {#chomps-sep} - -Returns a copy of string s with the string sep removed from the end if s ends with sep; otherwise, just returns s. For example, `"barely".chomp("ly")` and `"bare".chomp("ly")` both return the string “bare”. - -#### Substring {#substring} - -###### substring(s, n from, n to (optional)) {#substrings-n-from-n-to-optional} - -Returns the substring of s starting from character index from, and up to (excluding) character index to. If the to argument is omitted, substring will output to the end of s. For example, `"profound".substring(3)` returns the string “found”, and `"profound".substring(2, 4)` returns the string “of”. - -Remember that character indices start from zero. A negative character index counts from the end of the string. For example, `"profound".substring(0, -1)` returns the string “profoun”. - -###### slice(s, n from, n to (optional)) {#slices-n-from-n-to-optional} - -Identical to substring() in relation to strings. Also works with arrays; see [Array functions section](#slicea-n-from-n-to-optional). - -###### get(s, n from, n to (optional)) {#gets-n-from-n-to-optional} - -Identical to substring() in relation to strings. Also works with named fields. Also works with arrays; see [Array functions section](#geta-n-from-n-to-optional). - -#### Find and replace {#find-and-replace} - -###### indexOf(s, sub) {#indexofs-sub} - -Returns the first character index of sub as it first occurs in s; or, returns -1 if s does not contain sub. For example, `"internationalization".indexOf("nation")` returns 5, whereas `"internationalization".indexOf("world")` returns -1. - -###### lastIndexOf(s, sub) {#lastindexofs-sub} - -Returns the first character index of sub as it last occurs in s; or, returns -1 if s does not contain sub. For example, `"parallel".lastIndexOf("a")` returns 3 (pointing at the second “a”). - -###### replace(s, s or p find, s replace) {#replaces-s-or-p-find-s-replace} - -Returns the string obtained by replacing the find string with the replace string in the inputted string. For example, `"The cow jumps over the moon and moos".replace("oo", "ee")` returns the string “The cow jumps over the meen and mees”. Find can be a regex pattern. For example, `"The cow jumps over the moon and moos".replace(/\s+/, "_")` will return “The_cow_jumps_over_the_moon_and_moos”. - -You cannot find or replace nulls with this, as null is not a string. You can instead: - -1. Facet by null and then bulk-edit them to a string, or -2. Transform the column with an expression such as `if(value==null,"new",value)`. - -###### replaceChars(s, s find, s replace) {#replacecharss-s-find-s-replace} - -Returns the string obtained by replacing a character in s, identified by find, with the corresponding character identified in replace. For example, `"Téxt thát was optícálly recógnízéd".replaceChars("áéíóú", "aeiou")` returns the string “Text that was optically recognized”. You cannot use this to replace a single character with more than one character. - -###### find(s, sub or p) {#finds-sub-or-p} - -Outputs an array of all consecutive substrings inside string s that match the substring or [regex](expressions#grel-supported-regex) pattern p. For example, `"abeadsabmoloei".find(/[aeio]+/)` would result in the array [ "a", "ea", "a", "o", "oei" ]. - -You can supply a substring instead of p, by putting it in quotes, and OpenRefine will compile it into a regex pattern. Anytime you supply quotes, OpenRefine interprets the contents as a string, not regex. If you wish to use any regex notation, wrap the pattern in forward slashes. - -###### match(s, p) {#matchs-p} - -Attempts to match the string s in its entirety against the [regex](expressions#grel-supported-regex) pattern p and, if the pattern is found, outputs an array of all [capturing groups](https://www.regular-expressions.info/brackets.html) (found in order). For example, `"230.22398, 12.3480".match(/.*(\d\d\d\d)/)` returns an array of 1 substring: [ "3480" ]. It does not find 2239 as the first sequence with four digits, because the regex indicates the four digits must come at the end of the string. - -You will need to convert the array to a string to store it in a cell, with a function such as toString(). An empty array [] is returned when there is no match to the desired substrings. A null is output when the entire regex does not match. - -Remember to enclose your regex in forward slashes, and to escape characters and use parentheses as needed. Parentheses denote a desired substring (capturing group); for example, “.*(\d\d\d\d)” would return an array containing a single value, while “(.*)(\d\d\d\d)” would return two. So, if you are looking for a desired substring anywhere within a string, use the syntax `value.match(/.*(desired-substring-regex).*/)`. - -For example, if `value` is “hello 123456 goodbye”, the following would occur: - -|Expression|Result| -|-|-| -|`value.match(/\d{6}/)` |null (does not match the full string)| -|`value.match(/.*\d{6}.*/)` |[ ] (no indicated substring)| -|`value.match(/.*(\d{6}).*/)` |[ "123456" ] (array with one value)| -|`value.match(/(.*)(\d{6})(.*)/)` |[ "hello ", "123456", " goodbye" ] (array with three values)| - -### String parsing and splitting {#string-parsing-and-splitting} - -###### toNumber(s) {#tonumbers} - -Returns a string converted to a number. Will attempt to convert other formats into a string, then into a number. If the value is already a number, it will return the number. - -###### split(s, s or p sep, b preserveTokens (optional)) {#splits-s-or-p-sep-b-preservetokens-optional} - -Returns the array of strings obtained by splitting s by sep. The separator can be either a string or a regex pattern. For example, `"fire, water, earth, air".split(",")` returns an array of 4 strings: [ "fire", " water", " earth", " air" ]. Note that the space characters are retained but the separator is removed. If you include “true” for the preserveTokens boolean, empty segments are preserved. - -###### splitByLengths(s, n1, n2, ...) {#splitbylengthss-n1-n2-} - -Returns the array of strings obtained by splitting s into substrings with the given lengths. For example, `"internationalization".splitByLengths(5, 6, 3)` returns an array of 3 strings: [ "inter", "nation", "ali" ]. Excess characters are discarded. - -###### smartSplit(s, s or p sep (optional)) {#smartsplits-s-or-p-sep-optional} - -Returns the array of strings obtained by splitting s by sep, or by guessing either tab or comma separation if there is no sep given. Handles quotes properly and understands cancelled characters. The separator can be either a string or a regex pattern. For example, `value.smartSplit("\n")` will split at a carriage return or a new-line character. - -Note: [`value.escape('javascript')`](#escapes-s-mode) is useful for previewing unprintable characters prior to using smartSplit(). - -###### splitByCharType(s) {#splitbychartypes} - -Returns an array of strings obtained by splitting s into groups of consecutive characters each time the characters change [Unicode categories](https://en.wikipedia.org/wiki/Unicode_character_property#General_Category). For example, `"HenryCTaylor".splitByCharType()` will result in an array of [ "H", "enry", "CT", "aylor" ]. It is useful for separating letters and numbers: `"BE1A3E".splitByCharType()` will result in [ "BE", "1", "A", "3", "E" ]. - -###### partition(s, s or p fragment, b omitFragment (optional)) {#partitions-s-or-p-fragment-b-omitfragment-optional} - -Returns an array of strings [ a, fragment, z ] where a is the substring within s before the first occurrence of fragment, and z is the substring after fragment. Fragment can be a string or a regex. For example, `"internationalization".partition("nation")` returns 3 strings: [ "inter", "nation", "alization" ]. If s does not contain fragment, it returns an array of [ s, "", "" ] (the original unpartitioned string, and two empty strings). - -If the omitFragment boolean is true, for example with `"internationalization".partition("nation", true)`, the fragment is not returned. The output is [ "inter", "alization" ]. - -You can use regex for your fragment. The expresion `"abcdefgh".partition(/c.e/)` will output [“abc”, "cde", defgh” ]. - -###### rpartition(s, s or p fragment, b omitFragment (optional)) {#rpartitions-s-or-p-fragment-b-omitfragment-optional} - -Returns an array of strings [ a, fragment, z ] where a is the substring within s before the last occurrence of fragment, and z is the substring after the last instance of fragment. (Rpartition means “reverse partition.”) For example, `"parallel".rpartition("a")` returns 3 strings: [ "par", "a", "llel" ]. Otherwise works identically to partition() above. - -### Encoding and hashing {#encoding-and-hashing} - -###### diff(s1, s2, s timeUnit (optional)) {#diffs1-s2-s-timeunit-optional} - -Takes two strings and compares them, returning a string. Returns the remainder of s2 starting with the first character where they differ. For example, `"cacti".diff("cactus")` returns "us". Also works with dates; see [Date functions](#diffd1-d2-s-timeunit). - -###### escape(s, s mode) {#escapes-s-mode} - -Escapes s in the given escaping mode. The mode can be one of: "html", "xml", "csv", "url", "javascript". Note that quotes are required around your mode. See the [recipes](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#question-marks--showing-in-your-data) for examples of escaping and unescaping. - -###### unescape(s, s mode) {#unescapes-s-mode} - -Unescapes s in the given escaping mode. The mode can be one of: "html", "xml", "csv", "url", "javascript". Note that quotes are required around your mode. See the [recipes](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#atampampt----att) for examples of escaping and unescaping. - -###### md5(o) {#md5o} - -Returns the [MD5 hash](https://en.wikipedia.org/wiki/MD5) of an object. If fed something other than a string (array, number, date, etc.), md5() will convert it to a string and deliver the hash of the string. For example, `"internationalization".md5()` will return 2c55a1626e31b4e373ceedaa9adc12a3. - -###### sha1(o) {#sha1o} - -Returns the [SHA-1 hash](https://en.wikipedia.org/wiki/SHA-1) of an object. If fed something other than a string (array, number, date, etc.), sha1() will convert it to a string and deliver the hash of the string. For example, `"internationalization".sha1()` will return cd05286ee0ff8a830dbdc0c24f1cb68b83b0ef36. - -###### phonetic(s, s encoding) {#phonetics-s-encoding} - -Returns a phonetic encoding of a string, based on an available phonetic algorithm. See the [section on phonetic clustering](cellediting#clustering-methods) for more information. Can be one of the following supported phonetic methods: [metaphone, doublemetaphone, metaphone3](https://www.wikipedia.org/wiki/Metaphone), [soundex](https://en.wikipedia.org/wiki/Soundex), [cologne](https://en.wikipedia.org/wiki/Cologne_phonetics). Quotes are required around your encoding method. For example, `"Ruth Prawer Jhabvala".phonetic("metaphone")` outputs the string “R0PRWRJHBFL”. - -###### reinterpret(s, s encoderTarget, s encoderSource) {#reinterprets-s-encodertarget-s-encodersource} - -Returns s reinterpreted through the given character encoders. You must supply one of the [supported encodings](http://java.sun.com/j2se/1.5.0/docs/guide/intl/encoding.doc.html) for each of the original source and the target output. Note that quotes are required around your character encoder. - -When an OpenRefine project is started, data is imported and interpreted. A specific character encoding is identified or manually selected at that time (such as UTF-8). You can reinterpret a column into another specificed encoding using this function. This function may not fix your data; it may be better to use this in conjunction with new projects to test the interpretation, and pre-format your data as needed. - -###### fingerprint(s) {#fingerprints} - -Returns the fingerprint of s, a string that is the first step in [fingerprint clustering methods](cellediting#clustering-methods): it will trim whitespaces, convert all characters to lowercase, remove punctuation, sort words alphabetically, etc. For example, `"Ruth Prawer Jhabvala".fingerprint()` outputs the string “jhabvala prawer ruth”. - -###### ngram(s, n) {#ngrams-n} - -Returns an array of the word n-grams of s. That is, it lists all the possible consecutive combinations of n words in the string. For example, `"Ruth Prawer Jhabvala".ngram(2)` would output the array [ "Ruth Prawer", "Prawer Jhabvala" ]. A word n-gram of 1 simply lists all the words in original order; an n-gram larger than the number of words in the string will only return the original string inside an array (e.g. `"Ruth Prawer Jhabvala".ngram(4)` would simply return ["Ruth Prawer Jhabvala"]). - -###### ngramFingerprint(s, n) {#ngramfingerprints-n} - -Returns the [n-gram fingerprint](cellediting#clustering-methods) of s. For example, `"banana".ngram(2)` would output “anbana”, after first generating the 2-grams “ba an na an na”, removing duplicates, and sorting them alphabetically. - -###### unicode(s) {#unicodes} - -Returns an array of strings describing each character of s in their full unicode notation. For example, `"Bernice Rubens".unicode()` outputs [ 66, 101, 114, 110, 105, 99, 101, 32, 82, 117, 98, 101, 110, 115 ]. - -###### unicodeType(s) {#unicodetypes} - -Returns an array of strings describing each character of s by their unicode type. For example, `"Bernice Rubens".unicodeType()` outputs [ "uppercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "space separator", "uppercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter" ]. - -## Format-based functions (JSON, HTML, XML) {#format-based-functions-json-html-xml} - -###### jsonize(o) {#jsonizeo} - -Quotes a value as a JSON literal value. - -###### parseJson(s) {#parsejsons} - -Parses a string as JSON. get() can then be used with parseJson(): for example, `parseJson(" { 'a' : 1 } ").get("a")` returns 1. - -For example, from the following JSON array in `value`, we want to get all instances of “keywords” having the same object string name of “text”, and combine them, using the forEach() function to iterate over the array. - - { - "status":"OK", - "url":"", - "language":"english", - "keywords":[ - { - "text":"York en route", - "relevance":"0.974363" - }, - { - "text":"Anthony Eden", - "relevance":"0.814394" - }, - { - "text":"President Eisenhower", - "relevance":"0.700189" - } - ] - } - -The GREL expression `forEach(value.parseJson().keywords,v,v.text).join(":::")` will output “York en route:::Anthony Eden:::President Eisenhower”. - -### Jsoup XML and HTML parsing {#jsoup-xml-and-html-parsing} - -###### parseHtml(s) {#parsehtmls} -Given a cell full of HTML-formatted text, parseHtml() simplifies HTML tags (such as by removing “ /” at the end of self-closing tags), closes any unclosed tags, and inserts linebreaks and indents for cleaner code. You cannot pass parseHtml() a URL, but you can pre-fetch HTML with the [Add column by fetching URLs](columnediting#add-column-by-fetching-urls) menu option. - -A cell cannot store the output of parseHtml() unless you convert it with toString(): for example, `value.parseHtml().toString()`. - -When parseHtml() simplifies HTML, it can sometimes introduce errors. When closing tags, it makes its best guesses based on line breaks, indentation, and the presence of other tags. You may need to manually check the results. - -You can then extract or [select()](#selects-element) which portions of the HTML document you need for further splitting, partitioning, etc. An example of extracting all table rows from a div using parseHtml().select() together is described more in depth at [StrippingHTML](https://github.com/OpenRefine/OpenRefine/wiki/StrippingHTML). - -###### parseXml(s) {#parsexmls} -Given a cell full of XML-formatted text, parseXml() returns a full XML document and adds any missing closing tags. You can then extract or [select()](#selects-element) which portions of the XML document you need for further splitting, partitioning, etc. Functions the same way as parseHtml() is described above. - -###### select(s, element) {#selects-element} -Returns an array of all the desired elements from an HTML or XML document, if the element exists. Elements are identified using the [Jsoup selector syntax](https://jsoup.org/apidocs/org/jsoup/select/Selector.html). For example, `value.parseHtml().select("img.portrait")[0]` would return the entirety of the first “img” tag with the “portrait” class found in the parsed HTML inside `value`. Returns an empty array if no matching element is found. Use with toString() to capture the results in a cell. A tutorial of select() is shown in [StrippingHTML](https://github.com/OpenRefine/OpenRefine/wiki/StrippingHTML). - -You can use select() more than once: - -``` -value.parseHtml().select("div#content")[0].select("tr").toString() -``` - -###### htmlAttr(s, element) {#htmlattrs-element} -Returns a string from an attribute on an HTML element. Use it in conjunction with parseHtml() as in the following example: `value.parseHtml().select("a.email")[0].htmlAttr("href")` would retrieve the email address attached to a link with the “email” class. - -###### xmlAttr(s, element) {#xmlattrs-element} -Returns a string from an attribute on an XML element. Functions the same way htmlAttr() is described above. Use it in conjunction with parseXml(). - -###### htmlText(element) {#htmltextelement} -Returns a string of the text from within an HTML element (including all child elements), removing HTML tags and line breaks inside the string. Use it in conjunction with parseHtml() and select() to provide an element, as in the following example: `value.parseHtml().select("div.footer")[0].htmlText()`. - -###### xmlText(element) {#xmltextelement} -Returns a string of the text from within an XML element (including all child elements). Functions the same way htmlText() is described above. Use it in conjunction with parseXml() and select() to provide an element. - -###### innerHtml(element) {#innerhtmlelement} -Returns the [inner HTML](https://developer.mozilla.org/en-US/docs/Web/API/Element/innerHTML) of an HTML element. This will include text and children elements within the element selected. Use it in conjunction with parseHtml() and select() to provide an element. - -###### innerXml(element) {#innerxmlelement} -Returns the inner XML elements of an XML element. Does not return the text directly inside your chosen XML element - only the contents of its children. To select the direct text, use ownText(). To select both, use xmlText(). Use it in conjunction with parseXml() and select() to provide an element. - -###### ownText(element) {#owntextelement} -Returns the text directly inside the selected XML or HTML element only, ignoring text inside children elements (for this, use innerXml()). Use it in conjunction with a parser and select() to provide an element. - -## Array functions {#array-functions} - -###### length(a) {#lengtha} -Returns the size of an array, meaning the number of objects inside it. Arrays can be empty, in which case length() will return 0. - -###### slice(a, n from, n to (optional)) {#slicea-n-from-n-to-optional} -Returns a sub-array of a given array, from the first index provided and up to and excluding the optional last index provided. Remember that array objects are indexed starting at 0. If the to value is omitted, it is understood to be the end of the array. For example, `[0, 1, 2, 3, 4].slice(1, 3)` returns [ 1, 2 ], and `[ 0, 1, 2, 3, 4].slice(2)` returns [ 2, 3, 4 ]. Also works with strings; see [String functions](#slices-n-from-n-to-optional). - -###### get(a, n from, n to (optional)) {#geta-n-from-n-to-optional} -Returns a sub-array of a given array, from the first index provided and up to and excluding the optional last index provided. Remember that array objects are indexed starting at 0. - -If the to value is omitted, only one array item is returned, as a string, instead of a sub-array. To return a sub-array from one index to the end, you can set the to argument to a very high number such as `value.get(2,999)` or you can use something like `with(value,a,a.get(1,a.length()))` to count the length of each array. - -Also works with strings; see [String functions](#gets-n-from-n-to-optional). - -###### inArray(a, s) {#inarraya-s} -Returns true if the array contains the desired string, and false otherwise. Will not convert data types; for example, `[ 1, 2, 3, 4 ].inArray("3")` will return false. - -###### reverse(a) {#reversea} -Reverses the array. For example, `[ 0, 1, 2, 3].reverse()` returns the array [ 3, 2, 1, 0 ]. - -###### sort(a) {#sorta} -Sorts the array in ascending order. Sorting is case-sensitive, uppercase first and lowercase second. For example, `[ "al", "Joe", "Bob", "jim" ].sort()` returns the array [ "Bob", "Joe", "al", "jim" ]. - -###### sum(a) {#suma} -Return the sum of the numbers in the array. For example, `[ 2, 1, 0, 3 ].sum()` returns 6. - -###### join(a, sep) {#joina-sep} -Joins the items in the array with sep, and returns it all as a string. For example, `[ "and", "or", "not" ].join("/")` returns the string “and/or/not”. - -###### uniques(a) {#uniquesa} -Returns the array with duplicates removed. Case-sensitive. For example, `[ "al", "Joe", "Bob", "Joe", "Al", "Bob" ].uniques()` returns the array [ "Joe", "al", "Al", "Bob" ]. - -As of OpenRefine 3.4.1, uniques() reorders the array items it returns; in 3.4 beta 644 and onwards, it preserves the original order (in this case, [ "al", "Joe", "Bob", "Al" ]). - -## Date functions {#date-functions} - -###### now() {#now} - -Returns the current time according to your system clock, in the [ISO 8601 extended format](exploring#data-types) (converted to UTC). For example, 10:53am (and 00 seconds) on November 26th 2020 in EST returns [date 2020-11-26T15:53:00Z]. - -###### toDate(o, b monthFirst, s format1, s format2, ...) {#todateo-b-monthfirst-s-format1-s-format2-} - -Returns the inputted object converted to a date object. Without arguments, it returns the ISO 8601 extended format. With arguments, you can control the output format: -* monthFirst: set false if the date is formatted with the day before the month. -* formatN: attempt to parse the date using an ordered list of possible formats. Supply formats based on the [SimpleDateFormat](https://docs.oracle.com/javase/8/docs/api/java/text/SimpleDateFormat.html) syntax (and see the table below for a handy reference). - -For example, you can parse a column containing dates in different formats, such as cells with “Nov-09” and “11/09”, using `value.toDate('MM/yy','MMM-yy').toString('yyyy-MM')` and both will output “2009-11”. For another example, “1/4/2012 13:30:00” can be parsed into a date using `value.toDate('d/M/y H:m:s')`. - -| Letter | Date or Time Component | Presentation | Examples | -|-|-|-|-| -| G | Era designator | Text | AD | -| y | Year | Year | 1996; 96 | -| Y | [Week year](https://en.wikipedia.org/wiki/ISO_week_date#First_week) | Year | 2009; 09 | -| M | Month in year | Month | July; Jul; 07 | -| w | Week in year | Number | 27 | -| W | Week in month | Number | 2 | -| D | Day in year | Number | 189 | -| d | Day in month | Number | 10 | -| F | Day of week in month | Number | 2 | -| E | Day name in week | Text | Tuesday; Tue | -| u | Day number of week (1 = Monday, ..., 7 = Sunday) | Number | 1 | -| a | AM/PM marker | Text | PM | -| H | Hour in day (0-23) | Number | 0 | -| k | Hour in day (1-24) | Number | 24 | -| K | Hour in AM/PM (0-11) | Number | 0 | -| h | Hour in AM/PM (1-12) | Number | 12 | -| m | Minute in hour | Number | 30 | -| s | Second in minute | Number | 55 | -| S | Millisecond | Number | 978 | -| n | Nanosecond | Number | 789000 | -| z | Time zone | General time zone | Pacific Standard Time; PST; GMT-08:00 | -| Z | Time zone | RFC 822 time zone | \-0800 | -| X | Time zone | ISO 8601 time zone | \-08; -0800; -08:00 | - -###### diff(d1, d2, s timeUnit) {#diffd1-d2-s-timeunit} - -Given two dates, returns a number indicating the difference in a given time unit (see the table below). For example, `diff(("Nov-11".toDate('MMM-yy')), ("Nov-09".toDate('MMM-yy')), "weeks")` will return 104, for 104 weeks, or two years. The later date should go first. If the output is negative, invert d1 and d2. - -Also works with strings; see [diff() in string functions](#diffsd1-sd2-s-timeunit-optional). - -###### inc(d, n, s timeUnit) {#incd-n-s-timeunit} - -Returns a date changed by the given amount in the given unit of time (see the table below). The default unit is “hour”. A positive value increases the date, and a negative value moves it back in time. For example, if you want to move a date backwards by two months, use `value.inc(-2,"month")`. - -###### datePart(d, s timeUnit) {#datepartd-s-timeunit} - -Returns part of a date. The data type returned depends on the unit (see the table below). - -OpenRefine supports the following values for timeUnit: - -| Unit | Date part returned | Returned data type | Example using [date 2014-03-14T05:30:04.000789000Z] as value | -|-|-|-|-| -| years | Year | Number | value.datePart("years") → 2014 | -| year | Year | Number | value.datePart("year") → 2014 | -| months | Month | Number | value.datePart("months") → 2 | -| month | Month | Number | value.datePart("month") → 2 | -| weeks | Week of the month | Number | value.datePart("weeks") → 3 | -| week | Week of the month | Number | value.datePart("week") → 3 | -| w | Week of the month | Number | value.datePart("w") → 3 | -| weekday | Day of the week | String | value.datePart("weekday") → Friday | -| hours | Hour | Number | value.datePart("hours") → 5 | -| hour | Hour | Number | value.datePart("hour") → 5 | -| h | Hour | Number | value.datePart("h") → 5 | -| minutes | Minute | Number | value.datePart("minutes") → 30 | -| minute | Minute | Number | value.datePart("minute") → 30 | -| min | Minute | Number | value.datePart("min") → 30 | -| seconds | Seconds | Number | value.datePart("seconds") → 04 | -| sec | Seconds | Number | value.datePart("sec") → 04 | -| s | Seconds | Number | value.datePart("s") → 04 | -| milliseconds | Millseconds | Number | value.datePart("milliseconds") → 789 | -| ms | Millseconds | Number | value.datePart("ms") → 789 | -| S | Millseconds | Number | value.datePart("S") → 789 | -| n | Nanoseconds | Number | value.datePart("n") → 789000 | -| nano | Nanoseconds | Number | value.datePart("n") → 789000 | -| nanos | Nanoseconds | Number | value.datePart("n") → 789000 | -| time | Milliseconds between input and the [Unix Epoch](https://en.wikipedia.org/wiki/Unix_time) | Number | value.datePart("time") → 1394775004000 | - -## Math functions {#math-functions} - -For integer division and precision, you can use simple evaluations such as `1 / 2`, which is equivalent to `floor(1/2)` - that is, it returns only whole number results. If either operand is a floating point number, they both get promoted to floating point and a floating point result is returned. You can use `1 / 2.0` or `1.0 / 2` or `1.0 * x / y` (if you're working with variables of unknown contents). - -:::caution -Some of these math functions don't recognize integers when supplied as the first argument in dot notation (e.g., `5.cos()` simply returns 5 instead of the expected result). To ensure operations are successful, always wrap the first argument in brackets, such as `(value).cos()`. -::: - -|Function|Use|Example| -|-|-|-| -|`abs(n)`|Returns the absolute value of a number.|`abs(-6)` returns 6.| -|`acos(n)`|Returns the arc cosine of an angle, in the range 0 through [PI](https://docs.oracle.com/javase/8/docs/api/java/lang/Math.html#PI).|`acos(0.345)` returns 1.218557541697832.| -|`asin(n)`|Returns the arc sine of an angle in the range of -PI/2 through PI/2.|`asin(0.345)` returns 0.35223878509706474.| -|`atan(n)`|Returns the arc tangent of an angle in the range of -PI/2 through PI/2.|`atan(0.345)` returns 0.3322135507465967.| -|`atan2(n1, n2)`|Converts rectangular coordinates (n1, n2) to polar (r, theta). Returns number theta.|`atan2(0.345,0.6)` returns 0.5218342798144103.| -|`ceil(n)`|Returns the ceiling of a number.|`3.7.ceil()` returns 4 and `-3.7.ceil()` returns -3.| -|`combin(n1, n2)`|Returns the number of combinations for n2 elements as divided into n1.|`combin(20,2)` returns 190.| -|`cos(n)`|Returns the trigonometric cosine of a value.|`cos(5)` returns 0.28366218546322625.| -|`cosh(n)`|Returns the hyperbolic cosine of a value.|`cosh(5)` returns 74.20994852478785.| -|`degrees(n)`|Converts an angle from radians to degrees.|`degrees(5)` returns 286.4788975654116.| -|`even(n)`|Rounds the number up to the nearest even integer.|`even(5)` returns 6.| -|`exp(n)`|Returns [e](https://en.wikipedia.org/wiki/E_(mathematical_constant)) raised to the power of n.|`exp(5)` returns 148.4131591025766.| -|`fact(n)`|Returns the factorial of a number, starting from 1.|`fact(5)` returns 120.| -|`factn(n1, n2)`|Returns the factorial of n1, starting from n2.|`factn(10,3)` returns 280.| -|`floor(n)`|Returns the floor of a number.|`3.7.floor()` returns 3 and `-3.7.floor()` returns -4.| -|`gcd(n1, n2)`|Returns the greatest common denominator of two numbers.|`gcd(95,135)` returns 5.| -|`lcm(n1, n2)`|Returns the least common multiple of two numbers.|`lcm(95,135)` returns 2565.| -|`ln(n)`|Returns the natural logarithm of n.|`ln(5)` returns 1.6094379124341003.| -|`log(n)`|Returns the base 10 logarithm of n.|`log(5)` returns 0.6989700043360189.| -|`max(n1, n2)`|Returns the larger of two numbers.|`max(3,10)` returns 10.| -|`min(n1, n2)`|Returns the smaller of two numbers.|`min(3,10)` returns 3.| -|`mod(n1, n2)`|Returns n1 modulus n2. Note: `value.mod(9)` will work, whereas `74.mod(9)` will not work.|`mod(74, 9)` returns 2. | -|`multinomial(n1, n2 …(optional))`|Calculates the multinomial of one number or a series of numbers.|`multinomial(2,3)` returns 10.| -|`odd(n)`|Rounds the number up to the nearest odd integer.|`odd(10)` returns 11.| -|`pow(n1, n2)`|Returns n1 raised to the power of n2. Note: value.pow(3)` will work, whereas `2.pow(3)` will not work.|`pow(2, 3)` returns 8 (2 cubed) and `pow(3, 2)` returns 9 (3 squared). The square root of any numeric value can be called with `value.pow(0.5)`.| -|`quotient(n1, n2)`|Returns the integer portion of a division (truncated, not rounded), when supplied with a numerator and denominator.|`quotient(9,2)` returns 4.| -|`radians(n)`|Converts an angle in degrees to radians.|`radians(10)` returns 0.17453292519943295.| -|`randomNumber(n lowerBound, n upperBound)`|Returns a random integer in the interval between the lower and upper bounds (inclusively). Will output a different random number in each cell in a column.| -|`round(n)`|Rounds a number to the nearest integer.|`3.7.round()` returns 4 and `-3.7.round()` returns -4.| -|`sin(n)`|Returns the trigonometric sine of an angle.|`sin(10)` returns -0.5440211108893698.| -|`sinh(n)`|Returns the hyperbolic sine of an angle.|`sinh(10)` returns 11013.232874703393.| -|`sum(a)`|Sums the numbers in an array. Ignores non-number items. Returns 0 if the array does not contain numbers.|`sum([ 10, 2, three ])` returns 12.| -|`tan(n)`|Returns the trigonometric tangent of an angle.|`tan(10)` returns 0.6483608274590866.| -|`tanh(n)`|Returns the hyperbolic tangent of a value.|`tanh(10)` returns 0.9999999958776927.| - -## Other functions {#other-functions} - -###### type(o) {#typeo} -Returns a string with the data type of o, such as undefined, string, number, boolean, etc. For example, a [Transform](cellediting#transform) operation using `value.type()` will convert all cells in a column to strings of their data types. - -###### facetCount(choiceValue, s facetExpression, s columnName) {#facetcountchoicevalue-s-facetexpression-s-columnname} -Returns the facet count corresponding to the given choice value, by looking for the facetExpression in the choiceValue in columnName. For example, to create facet counts for the following table, we could generate a new column based on “Gift” and enter in `value.facetCount("value", "Gift")`. This would add the column we've named “Count”: - -| Gift | Recipient | Price | Count | -|-|-|-|-| -| lamp | Mary | 20 | 1 | -| clock | John | 57 | 2 | -| watch | Amit | 80 | 1 | -| clock | Claire | 62 | 2 | - -The facet expression, wrapped in quotes, can be useful to manipulate the inputted values before counting. For example, you could do a textual cleanup using fingerprint(): `(value.fingerprint()).facetCount(value.fingerprint(),"Gift")`. - -###### hasField(o, s name) {#hasfieldo-s-name} -Returns a boolean indicating whether o has a member field called [name](expressions#variables). For example, `cell.recon.hasField("match")` will return false if a reconciliation match hasn’t been selected yet, or true if it has. You cannot chain your desired fields: for example, `cell.hasField("recon.match")` will return false even if the above expression returns true). - -###### coalesce(o1, o2, o3, ...) {#coalesceo1-o2-o3-} -Returns the first non-null from a series of objects. For example, `coalesce(value, "")` would return an empty string “” if `value` was null, but otherwise return `value`. - -###### cross(cell, s projectName, s columnName) {#crosscell-s-projectname-s-columnname} -Returns an array of zero or more rows in the project projectName for which the cells in their column columnName have the same content as the cell in your chosen column. For example, if two projects contained matching names, and you wanted to pull addresses for people by their names from a project called “People” you would apply the following expression to your column of names: -``` -cell.cross("People","Name").cells["Address"].value[0] -``` - -This would match your current column to the “Name” column in “People” and, using those matches, pull the respective “Address” value into your current project. - -You may need to do some data preparation with cross(), such as using trim() on your key columns or deduplicating values. - -The first argument will be interpreted as `cell.value` if set to `cell`. - -Recipes and more examples for using cross() can be found [on our wiki](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#combining-datasets). diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/installing.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/installing.md deleted file mode 100644 index 0dd2618c2..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/installing.md +++ /dev/null @@ -1,425 +0,0 @@ ---- -id: installing -title: Installing OpenRefine -sidebar_label: Installing ---- - -## System requirements {#system-requirements} - -OpenRefine does not require internet access to run its basic functions. Once you download and install it, it runs as a small web server on your own computer, and you access that local web server by using your browser. It only requires an internet connection to import data from the web, reconcile data using a web service, or export data to the web. - -OpenRefine requires three things on your computer in order to function: - -#### Compatible operating system {#compatible-operating-system} - -OpenRefine is designed to work with **Windows**, **Mac**, and **Linux** operating systems. [Our team releases packages for each](https://openrefine.org/download.html). - -#### Java {#java} - -[Java](https://java.com/en/download/) must be installed and configured on your computer to run OpenRefine. The Mac version of OpenRefine includes Java; new in OpenRefine 3.4, there is also a Windows package with Java included. - -If you install and start OpenRefine on a Windows computer without Java, it will automatically open up a browser window to the [Java downloads page](https://java.com/en/download/), and you can simply follow the instructions there. - -We recommend you [download](https://java.com/en/download/) and install Java before proceeding with the OpenRefine installation. Please note that OpenRefine works with Java 8 to Java 15 but not Java 16 or later versions. - -#### Compatible browser {#compatible-browser} - -OpenRefine works best on browsers based on Webkit, such as: - -* Google Chrome -* Chromium -* Opera -* Microsoft Edge - -We are aware of some minor rendering and performance issues on other browsers such as Firefox. We don't support Internet Explorer. If you are having issues running OpenRefine, see the [section on Running](running.md#troubleshooting). - -### Release versions {#release-versions} - -OpenRefine always has a [latest stable release](https://github.com/OpenRefine/OpenRefine/releases/latest), as well as some more recent developments available in beta, release candidate, or [snapshot releases](https://github.com/OpenRefine/OpenRefine-snapshot-releases/releases). If you are installing for the first time, we recommend [the latest stable release](https://github.com/OpenRefine/OpenRefine/releases/latest). - -If you wish to use an extension that is only compatible with an earlier version of OpenRefine, and do not require the latest features, you may find that [an older stable version is best for you](https://github.com/OpenRefine/OpenRefine/releases) in our list of releases. Look at later releases to see which security vulnerabilities are being fixed, in order to assess your own risk tolerance for using earlier versions. Look for “final release” versions instead of “beta” or “release candidate” versions. - -#### Unstable versions {#unstable-versions} - -If you need a recently developed function, and are willing to risk some untested code, you can look at [the most recent items in the list](https://github.com/OpenRefine/OpenRefine/releases) and see what changes appeal to you. - -“Beta” and “release candidate” versions may both have unreported bugs and are most suitable for people who are willing to help us troubleshoot these versions by [creating bug reports](https://github.com/OpenRefine/OpenRefine/issues). - -For the absolute latest development updates, see the [snapshot releases](https://github.com/OpenRefine/OpenRefine-snapshot-releases/releases). These are created with every commit. - -#### What’s changed {#whats-changed} - -Our [latest version is OpenRefine 3.4.1](https://github.com/OpenRefine/OpenRefine/releases/tag/3.4.1), released September 24th 2020. The major changes in this version are listed on the [3.4.1 release page](https://github.com/OpenRefine/OpenRefine/releases/tag/3.4.1) with the downloadable packages. - -You can find information about all OpenRefine versions on the [Releases page on Github](https://github.com/OpenRefine/OpenRefine/releases). - -:::info Other distributions -OpenRefine may also work in other environments, such as [Chromebooks](https://gist.github.com/organisciak/3e12e5138e44a2fed75240f4a4985b4f) where Linux terminals are available. Look at our list of [Other Distributions on the Downloads page](https://openrefine.org/download.html) for other ways of running OpenRefine, and refer to our contributor community to see new environments in development. -::: - -## Installing or upgrading {#installing-or-upgrading} -### Back up your data {#back-up-your-data} - -If you are upgrading from an older version of OpenRefine and have projects already on your computer, you should create backups of those projects before you install a new version. - -First, [locate your workspace directory](#where-is-data-stored). Then copy everything you find there and paste it into a folder elsewhere on your computer. - -For extra security you can [export your existing OpenRefine projects](exporting#export-a-project). - -:::caution -Take note of the [extensions](#installing-extensions) you have currently installed. They may not be compatible with the upgraded version of OpenRefine. Installations can be installed in two places, so be sure to check both your workspace directory and the existing installation directory. -::: - -### Install or upgrade OpenRefine {#install-or-upgrade-openrefine} - -If you are upgrading an existing OpenRefine installation, you can delete the old program files and install the new files into the same space. Do not overwrite the files as some obsolete files may be left over unnecessarily. - -:::caution -If you have extensions installed, do not delete the `webapp\extensions` folder where you installed them. You may wish to install extensions into the workspace directory instead of the program directory. There is no guarantee that extensions will be forward-compatible with new versions of OpenRefine, and we do not maintain extensions. -::: - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - - -Once you have downloaded the `.zip` file, extract it into a folder where you wish to store program files (such as `D:\Program Files\OpenRefine`). - -You can right-click on `openrefine.exe` or `refine.bat` and pin one of those programs to your Start Menu or create shortcuts for easier access. - - - - - -Once you have downloaded the `.dmg` file, open it and drag the OpenRefine icon onto the Applications folder icon (just like you would normally install Mac applications). - - - - - -The quick version: - -1. Install [Homebrew](http://brew.sh) -2. In Terminal enter `brew install openrefine` -1. Then find OpenRefine in your Applications folder. - -The long version: - -[Homebrew](https://brew.sh) is a popular command-line package manager for Mac. Installing Homebrew is accomplished by pasting the installation command on the Homebrew website into a Terminal window. Once Homebrew is installed, applications like OpenRefine can be installed via a simple command. You can [install Homebrew from their website](https://brew.sh). - -###### Install {#install} - -Install OpenRefine with this command: - -``` -brew install openrefine -``` - -You should see output like this: - -``` -==> Downloading https://github.com/OpenRefine/OpenRefine/releases/download/3.4.1/openrefine-mac-3.4.1.dmg -######################################################################## 100.0% -==> Installing Cask openrefine -==> Moving App 'OpenRefine.app' to '/Applications/OpenRefine.app' -🍺 openrefine was successfully installed! -``` - -Behind the scenes, this command causes Homebrew to download the OpenRefine installer, verify the file’s authenticity (using a SHA-256 checksum), mount the disk image, copy the `OpenRefine.app` application bundle into the Applications folder, unmount the disk image, and save a copy of the installer and metadata about the installation for future use. - -If an existing `OpenRefine.app` is found in the Applications folder, Homebrew will not overwrite it, so installing via Homebrew requires either deleting or renaming previously installed copies. - -###### Uninstall {#uninstall} - -To uninstall OpenRefine, paste this command into the Terminal: - -``` -brew uninstall openrefine -``` - -You should see output like this: - -``` -==> Uninstalling Cask openrefine -==> Backing App 'OpenRefine.app' up to '/usr/local/Caskroom/openrefine/3.4.1/OpenRefine.app' -==> Removing App '/Applications/OpenRefine.app' -==> Purging files for version 3.4.1 of Cask openrefine -``` - -###### Update {#update} - -To update to the latest version of OpenRefine, paste these two commands into the Terminal: - -``` -brew update -brew upgrade -``` - -You should see output like this: - -``` -==> Upgrading 1 outdated package: -openrefine 3.4.0-> 3.4.1 -==> Upgrading openrefine -==> Downloading https://github.com/OpenRefine/OpenRefine/releases/download/3.4.1/openrefine-mac-3.4.1.dmg -######################################################################## 100.0% -==> Backing App 'OpenRefine.app' up to '/usr/local/Caskroom/openrefine/3.4.0/OpenRefine.app' -==> Removing App '/Applications/OpenRefine.app' -==> Installing Cask openrefine -==> Moving App 'OpenRefine.app' to '/Applications/OpenRefine.app' -🍺 openrefine was successfully upgraded! -``` - - - - -Once you have downloaded the `.tar.gz` file, open a shell, navigate to the folder containing the download, and type: - -``` -tar xzf openrefine-linux-3.4.tar.gz -``` - - - - - ---- - - -### Set where data is stored {#set-where-data-is-stored} - -OpenRefine stores data in two places: program files in the program directory, wherever it is you’ve installed it; and project files in what we call the “workspace directory.” You can access this folder easily from OpenRefine by going to the [home screen](running#the-home-screen) (at [http://127.0.0.1:3333/](http://127.0.0.1:3333/)) and clicking Browse workspace directory. - -By default this is: - - - - - -Depending on your version of Windows, the data is in one of these directories: -* `%appdata%\OpenRefine` -* `%localappdata%\OpenRefine` -* `C:\Documents and Settings\(user id)\Local Settings\Application Data\OpenRefine` -* `C:\Users\(user id)\AppData\Roaming\OpenRefine` -* `C:\Users\(user id)\AppData\Local\OpenRefine` -* `C:\Users\(user id)\OpenRefine` - -For older Google Refine releases, replace `OpenRefine` with `Google\Refine`. - -You can change this by adding this line to the file `openrefine.l4j.ini` and specifying your desired drive and folder path: - -``` --Drefine.data_dir=D:\MyDesiredFolder -``` - -If your folder path has spaces, use neutral quotation marks around it: - -``` --Drefine.data_dir="D:\My Desired Folder" -``` - -If the folder does not exist, OpenRefine will create it. - - - - - -``` -~/Library/Application Support/OpenRefine/ -``` - -For older versions, as Google Refine: - -``` -~/Library/Application Support/Google/Refine/ -``` - -Logging is to `/var/log/daemon.log` - grep for `com.google.refine.Refine`. - - - - - -``` -~/.local/share/openrefine/ -``` - -You can change this when you run OpenRefine from the terminal, by pointing to the workspace directory through the `-d` parameter: - -``` - ./refine -p 3333 -i 0.0.0.0 -m 6000M -d /My/Desired/Folder -``` - - - - - ---- - - -### Logs {#logs} - -OpenRefine does not currently output an error log, but because the OpenRefine console window is always open (on Linux and Windows) while OpenRefine runs in your browser, you can copy information from the console if an error occurs. - -Using a Mac, you can [run OpenRefine using the terminal](running#starting-and-exiting) in order to capture errors. - ---- - -## Increasing memory allocation {#increasing-memory-allocation} - -OpenRefine relies on having computer memory available to it to work effectively. If you are planning to work with large datasets, you may wish to set up OpenRefine to handle it at the outset. By “large” we generally mean one of the following indicators: -* more than one million total cells -* an input file size of more than 50 megabytes (MB) -* more than 50 [rows per record in records mode](running#records-mode) - -By default OpenRefine is set to operate with 1 gigabyte (GB) of memory (1024MB). If you feel that OpenRefine is running slowly, or you are getting “out of memory” errors (for example, `java.lang.OutOfMemoryError`), you can try allocating more memory. - -A good practice is to start with no more than 50% of whatever memory is left over after the estimated usage of your operating system, to leave memory for your browser to run. - -All of the settings below use a four-digit number to specify the megabytes (MB) used (actually [mebibytes](https://en.wikipedia.org/wiki/Mebibyte)). The default is usually 1024MB, but the new value doesn't need to be a multiple of 1024. - -:::info Dealing with large datasets -If your project is big enough to need more than the default amount of memory, consider turning off Parse cell text into numbers, dates, ... on import. It's convenient, but less efficient than explicitly converting any columns that you need as a data type other than the default “string” type. -::: - - - - - -#### Using openrefine.exe {#using-openrefineexe} - -If you run `openrefine.exe`, you will need to edit the `openrefine.l4j.ini` file found in the program directory and edit the line - -``` -# max memory memory heap size --Xmx1024M -``` - -The line “-Xmx1024M” defines the amount of memory available in megabytes. Change the number “1024” - for example, edit the line to “-Xmx2048M” to make 2048MB [2GB] of memory available. - -:::caution openrefine.exe not running? -Once you increase the memory allocation, you may find that you cannot run `openrefine.exe`. In this case, your computer needs a 64-bit version of [Java](https://www.java.com/en/download/help/index_installing.xml) (this is different from [Java JDK](#install-or-upgrade-java). Look for the “Windows Offline (64-bit)” download on the Downloads page and install that. Your system must also be set to use the 64-bit version of Java by [changing the Java configuration](https://www.java.com/en/download/help/update_runtime_settings.xml). -::: - -#### Using refine.bat {#using-refinebat} - -On Windows, OpenRefine can also be run by using the file `refine.bat` in the program directory. If you start OpenRefine using `refine.bat`, the memory available to OpenRefine can be specified either through command line options, or through the `refine.ini` file. - -To set the maximum amount of memory on the command line when using `refine.bat`, `cd` to the program directory, then type - -```refine.bat /m 2048m``` - -where “2048” is the maximum amount of MB that you want OpenRefine to use. - -To change the default that `refine.bat` uses, edit the `refine.ini` line that reads - -```REFINE_MEMORY=1024M``` - -Note that this file is only read if you use `refine.bat`, not `openrefine.exe`. - - - - -If you have downloaded the `.dmg` package and you start OpenRefine by double-clicking on it: - -* close OpenRefine -* control-click on the OpenRefine icon (opens the contextual menu) -* click on "show package content” (a finder window opens) -* open the “Contents” folder -* open the `Info.plist` file with any text editor (like Mac's default TextEdit) -* Change “-Xmx1024M” into, for example, “-Xmx2048M” or “-Xmx8G” -* save the file -* restart OpenRefine. - - - - -If you have downloaded the `.tar.gz` package and you start OpenRefine from the command line, add the “-m xxxxM” parameter like this: -`./refine -m 2048m` - -#### Setting a default {#setting-a-default} - -If you don't want to set this option on the command line each time, you can also set it in the `refine.ini` file. Edit the line - -``` -REFINE_MEMORY=1024M -``` - -Make sure it is not commented out (that is, that the line doesn't start with a “#” character), and change “1024” to a higher value. Save the file, and when you next start OpenRefine it will use this value. - - - - - ---- - - -## Installing extensions {#installing-extensions} - -Extensions have been created by our contributor community to add functionality or provide convenient shortcuts for common uses of OpenRefine. [We list extensions we know about on our downloads page](https://openrefine.org/download.html). - -:::info Contributing extensions -If you’d like to create or modify an extension, [see our developer documentation here](https://github.com/OpenRefine/OpenRefine/wiki/Documentation-For-Developers). If you’re having a problem, [use our downloads page](https://openrefine.org/download.html) to go to the extension’s page and report the issue there. -::: - -### Two ways to install extensions {#two-ways-to-install-extensions} - -You can [install extensions in one of two places](#set-where-data-is-stored): - -* Into your OpenRefine program folder, so they will only be available to that version/installation of OpenRefine (meaning the extension will not run if you upgrade OpenRefine), or -* Into your workspace, where your projects are stored, so they will be available no matter which version of OpenRefine you’re using. - -We provide these options because you may wish to reinstall a given extension manually each time you upgrade OpenRefine, in order to be sure it works properly. - -### Find the right place to install {#find-the-right-place-to-install} - -If you want to install the extension into the program folder, go to your program directory and then go to `webapp\extensions` (or create it if not does not exist). - -If you want to install the extension into your workspace, you can: -* launch OpenRefine and click Open Project in the sidebar -* At the bottom of the screen, click Browse workspace directory -* A file-explorer or finder window will open in your workspace -* Create a new folder called “extensions” inside the workspace if it does not exist. - -You can also [find your workspace on each operating system using these instructions](#set-where-data-is-stored). - -### Install the extension {#install-the-extension} - -Some extensions have their own instructions: make sure you read the documentation before you begin installing. - -Some extensions may have multiple versions, to match OpenRefine versions, so be sure to choose the right release for your installation. If you have questions about compatibility or want to request or voice your support for an update, [use our downloads page](https://openrefine.org/download.html) to go to the extension’s page and report the issue there. - -Generally, the installation process will be: - -* Download the extension (usually as a zip file from GitHub) -* Extract the zip contents into the “extensions” directory, making sure all the contents go into one folder with the name of the extension -* Start (or restart) OpenRefine. - -To confirm that installation was a success, follow the instructions provided by the extension. Each extension will appear in its own way inside the OpenRefine interface. Make sure you read its documentation to know where the functionality will appear, such as under specific dropdown menus. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/jythonclojure.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/jythonclojure.md deleted file mode 100644 index a93d18155..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/jythonclojure.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -id: jythonclojure -title: Jython & Clojure -sidebar_label: Jython & Clojure ---- - -## Jython {#jython} - -Jython 2.7.2 comes bundled with the default installation of OpenRefine 3.4.1. You can add libraries and code by following [this tutorial](https://github.com/OpenRefine/OpenRefine/wiki/Extending-Jython-with-pypi-modules). A large number of Python files (`.py` or `.pyc`) are compatible. - -Python code that depends on C bindings will not work in OpenRefine, which uses Java / Jython only. Since Jython is essentially Java, you can also import Java libraries and utilize those. - -You will need to restart OpenRefine, so that new Jython or Python libraries are initialized during startup. - -OpenRefine now has [most of the Jsoup.org library built into GREL functions](grelfunctions#jsoup-xml-and-html-parsing-functions) for parsing and working with HTML and XML elements. - -### Syntax {#syntax} - -Expressions in Jython must have a `return` statement: - -``` - return value[1:-1] -``` - -``` - return rowIndex%2 -``` - -Fields have to be accessed using the bracket operator rather than dot notation: - -``` - return cells["col1"]["value"] -``` - -For example, to access the [edit distance](reconciling#reconciliation-facets) between a reconciled value and an original cell value using [recon variables](#reconciliation): - -``` - return cell["recon"]["features"]["nameLevenshtein"] -``` - -To return the lower case of `value` (if the value is not null): - -``` - if value is not None: - return value.lower() - else: - return None -``` - -### Tutorials {#tutorials} -- [Extending Jython with pypi modules](https://github.com/OpenRefine/OpenRefine/wiki/Extending-Jython-with-pypi-modules) -- [Working with phone numbers using Java libraries inside Python](https://github.com/OpenRefine/OpenRefine/wiki/Jython#tutorial---working-with-phone-numbers-using-java-libraries-inside-python) - -Full documentation on the Jython language can be found on its official site: [http://www.jython.org](http://www.jython.org). - -## Clojure {#clojure} - -Clojure 1.10.1 comes bundled with the default installation of OpenRefine 3.4.1. At this time, not all [variables](expressions#variables) can be used with Clojure expressions: only `value`, `row`, `rowIndex`, `cell`, and `cells` are available. - -For example, functions can take the form -``` -(.. value (toUpperCase) ) -``` - -Or can look like -``` -(-> value (str/split #" ") last ) -``` - -which functions like `value.split(" ")` in GREL. - -For help with syntax, see the [Clojure website's guide to syntax](https://clojure.org/guides/learn/syntax). - -User-contributed Clojure recipes can be found on our wiki at [https://github.com/OpenRefine/OpenRefine/wiki/Recipes#11-clojure](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#11-clojure). - -Full documentation on the Clojure language can be found on its official site: [https://clojure.org/](https://clojure.org/). \ No newline at end of file diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/reconciling.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/reconciling.md deleted file mode 100644 index 9457bf120..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/reconciling.md +++ /dev/null @@ -1,247 +0,0 @@ ---- -id: reconciling -title: Reconciling -sidebar_label: Reconciling ---- - -## Overview {#overview} -:::info -If you are interested in reconciling with Wikidata, see also [our page about Wikidata](wikidata). - -This page covers the general handling in OpenRefine for any reconciliation service. -::: - -Reconciliation is the process of matching your dataset with that of an external source. Datasets for comparison might be produced by libraries, archives, museums, academic organizations, scientific institutions, non-profits, or interest groups. You can also reconcile against user-edited data on [Wikidata](wikidata), or reconcile against [a local dataset that you yourself supply](https://github.com/OpenRefine/OpenRefine/wiki/Reconcilable-Data-Sources#local-services). - -To reconcile your OpenRefine project against an external dataset, that dataset must offer a web service that conforms to the [Reconciliation Service API standards](https://reconciliation-api.github.io/specs/0.1/). - -You may wish to reconcile in order to: -* fix spelling or variations in proper names -* clean up manually-entered subject headings against authorities such as the [Library of Congress Subject Headings](https://id.loc.gov/authorities/subjects.html) (LCSH) -* link your data to an existing dataset -* add to an editable platform such as [Wikidata](https://www.wikidata.org) -* or see whether entities in your project appear in some specific list, such as the [Panama Papers](https://aleph.occrp.org/datasets/734). - -Reconciliation is semi-automated: OpenRefine matches your cell values to the reconciliation information as best it can, but human judgment is required to review and approve the results. Reconciling happens by default through string searching, so typos, whitespace, and extraneous characters will have an effect on the results. You may wish to [clean and cluster](cellediting) your data before reconciliaton. - -:::info Working iteratively -We recommend planning your reconciliation operations as iterative: reconcile multiple times with different settings, and with different subgroups of your data. -::: - -## Sources {#sources} - -Start with [this current list of reconcilable authorities](https://reconciliation-api.github.io/testbench/), which includes instructions for adding new services via Wikidata editing if you have one to add. - -OpenRefine maintains a [further list of sources on the wiki](https://github.com/OpenRefine/OpenRefine/wiki/Reconcilable-Data-Sources), which can be edited by anyone. This list includes ways that you can reconcile against a [local dataset](https://github.com/OpenRefine/OpenRefine/wiki/Reconcilable-Data-Sources#local-services). - -Other services may exist that are not yet listed in these two places: for example, the [310 datasets hosted by the Organized Crime and Corruption Reporting Project (OCCRP)](https://aleph.occrp.org/datasets/) each have their own reconciliation URL, or you can reconcile against their entire database with the URL [shared on the reconciliation API list](https://reconciliation-api.github.io/testbench/). For another example, you can reconcile against the entire Virtual International Authority File (VIAF) dataset, or [only the contributions from certain institutions](http://refine.codefork.com/). Search online to see if the authority you wish to reconcile against has an available service, or whether you can download a copy to reconcile against locally. - -OpenRefine includes Wikidata reconciliation in the installation package - see the [Wikidata](wikidata) page for more information particular to that service. Extensions can add reconciliation services, and can also add enhanced reconciliation capacities. Check the list of extensions on the [Downloads page](https://openrefine.org/download.html) for more information. - -Each source will have its own documentation on how it provides reconciliation. The table on [the reconciliation API list](https://reconciliation-api.github.io/testbench/) indicates whether your chosen service supports the features described below. Refer to the service's documentation if you have questions about its behaviors and which OpenRefine features it supports. - -In addition to the reconciliation services mentioned above, you may also choose to build your own service. You can either start from scratch using the [API specification](https://reconciliation-api.github.io/specs/latest/) or use one of the frameworks mentioned in the [Reconciliation census](https://reconciliation-api.github.io/census/services/). - -Of particular note is [reconcile-csv](http://okfnlabs.org/reconcile-csv/) which allows you to build a reconciliation service from a simple CSV file. Thus if you wanted to reconcile one OpenRefine project against another, you'd simply need to export the target project as a CSV, point `reconcile-csv` at it and you're good to go. A somewhat newer port of this project written in Python can be found at [csv-reconcile](https://github.com/gitonthescene/csv-reconcile) which is more configurable and defaults to parsing tab separated files for convenience. - -Similiarly, you may choose to export some SPARQL output to a TSV to limit the scope of values you're reconciling against and/or for better peformance. - -## Getting started {#getting-started} - -Choose a column to reconcile and use its dropdown menu to select ReconcileStart reconciling. If you want to reconcile only some cells in that column, first use filters and facets to isolate them. - -In the reconciliation window, you will see Wikidata offered as a default service. To add another service, click Add Standard Service... and paste in the URL of a [service](#sources). You should see the name of the service appear in the list of Services if the URL is correct. - -![The reconciliation window.](/img/reconcilewindow.png) - -Once you select a service, your selected column may be sampled in order to suggest [“types” (categories)](#reconciling-by-type) to reconcile against. Other services will suggest their available types without sampling, and some services have no types. - -For example, if you had a list of artists represented in a gallery collection, you could reconcile their names against the Getty Research Institute’s [Union List of Artist Names (ULAN)](https://www.getty.edu/research/tools/vocabularies/ulan/). The same [Getty reconciliation URL](https://services.getty.edu/vocab/reconcile/) will offer you ULAN, AAT (Art and Architecture Thesaurus), and TGN (Thesaurus of Geographic Names). - -![The reconciliation window with types.](/img/reconcilewindow2.png) - -Refer to the [documentation specific to the reconciliation service](https://reconciliation-api.github.io/testbench/) to learn whether types are offered, which types are offered, and which one is most appropriate for your column. You may wish to facet your data and reconcile batches against different types if available. - -Reconciliation can be a time-consuming process, especially with large datasets. We suggest starting with a small test batch. There is no throttle (delay between requests) to set for the reconciliation process. The amount of time will vary for each service, and vary based on the options you select during the process. - -When the process is done, you will see the reconciliation data in the cells. -If the cell was successfully matched, it displays text as a single dark blue link. In this case, the reconciliation is confident that the match is correct, and you should not have to check it manually. -If there is no clear match, one or more candidates are displayed, together with their reconciliation score, with the text in light blue links. You will need to select the correct one. - -For each matching decision you make, you have two options: match this cell only (one checkmark), or also use the same identifier for all other cells containing the same original string (two checkmarks). - -For services that offer the [“preview entities” feature](https://reconciliation-api.github.io/testbench/), you can hover your mouse over the suggestions to see more information about the candidates or matches. Each participating service (and each type) will deliver different structured data that may help you compare the candidates. - -For example, the Getty ULAN shows an artist’s discipline, nationality, and birth and death years: - -![Hovering over matches.](/img/reconcilehover.png) - -Hovering over the suggestion will also offer the two matching options as buttons. - -For matched values (those appearing as dark blue links), the underlying cell value has not been altered - the cell is storing both the original string and the matched entity link at the same time. If you were to copy your column to a new column at this point using `value`, for example, the reconcilation data would not transfer - only the original strings. You can learn more about how OpenRefine stores different pieces of information in each cell in [the Variables section specific to reconciliation data](expressions#reconciliation). - -For each cell, you can manually “Create new item,” which will take the cell’s original value and apply it, as though it is a match. This will not become a dark blue link, because at this time there is nothing to link to: it is a draft entity stored only in your project. You can use this feature to prepare these entries for eventual upload to an editable service such as [Wikidata](wikidata), but most services do not yet support this feature. - -### Reconciliation facets {#reconciliation-facets} - -Under ReconcileFacets there are a number of reconciliation-specific faceting options. OpenRefine automatically creates two facets when you reconcile some cells. - -One is a numeric facet for “best candidate's score,” the range of reconciliation scores of only the best candidate of each cell. Higher scores mean better matches, although each service calculates scores differently and has a different range. You can facet for higher scores using the numeric facet, and then approve them all in bulk, by using Reconcile[Actions](#reconciliation-actions)Match each cell to its best candidate. - -There is also a “judgment” facet created, which lets you filter for the cells that haven't been matched (pick “none” in the facet). As you process each cell, its judgment changes from “none” to “matched” and it disappears from the view. - -You can add other facets by selecting ReconcileFacets on your reconciled column. You can facet by: - -* your judgments (“matched,” or “none” for unreconciled cells, or “new” for entities you've created) -* the action you’ve performed on that cell (chosen a “single” match, or set a “mass” match, or no action, which appears as “unknown”) -* the timestamps on the edits you’ve made so far (these appear as millisecond counts since an arbitrary point: they can be sorted alphabetically to move forward and back in time). - -You can facet only the best candidates for each cell, based on: -* the score (calculated based on each service's own methods) -* the edit distance (using the [Levenshtein distance](cellediting#nearest-neighbor), a number based on how many single-character edits would be required to get your original value to the candidate value, with a larger value being a greater difference) -* the word similarity. - -Word similarity is calculated as a percentage based on how many words (excluding [stop words](https://en.wikipedia.org/wiki/Stop_word)) in the original value match words in the candidate. For example, the value “Maria Luisa Zuloaga de Tovar” matched to the candidate “Palacios, Luisa Zuloaga de” results in a word similarity value of 0.6, or 60%, or 3 out of 5 words. Cells that are not yet matched to one candidate will show as 0.0). - -You can also look at each best candidate’s: -* type (the ones you have selected in successive reconciliation attempts, or other types returned by the service based on the cell values) -* type match (“true” if you selected a type and it succeeded, “false” if you reconciled against no particular type, and “(no type)” if it didn’t reconcile) -* name match (“true” if you’ve matched, “false” if you haven’t yet chosen from the candidates, or “(unreconciled)” if it didn’t reconcile). - -These facets are useful for doing successive reconciliation attempts, against different types, and with different supplementary information. The information represented by these facets are held in the cells themselves and can be called using the [reconciliation variables](expressions#reconciliation) available in expressions. - -### Reconciliation actions {#reconciliation-actions} - -You can use the ReconcileActions menu options to perform bulk changes (which will apply only to your currently viewed set of rows or records): -* Match each cell to its best candidate (by highest score) -* Create a new item for each cell (discard any suggested matches) -* Create one new item for similar cells (a new entity will be created for each unique string) -* Match all filtered cells to... (a specific item from the chosen service, via a search box; only works with services that support the “suggest entities” property) -* Discard all reconciliation judgments (reverts back to multiple candidates per cell, including cells that may have been auto-matched in the original reconciliation process) -* Clear reconciliation data, reverting all cells back to their original values. - -The other options available under Reconcile are: -* Copy reconciliation data... (to an existing column: if the original values in your reconciliation column are identical to those in your chosen column, the matched and new cells will copy over; unmatched values will not change) -* [Use values as identifiers](#reconciling-with-unique-identifiers) (if you are reconciling with unique identifiers instead of by doing string searches) -* [Add entity identifiers column](#add-entity-identifiers-column). - -## Reconciling with unique identifiers {#reconciling-with-unique-identifiers} - -Reconciliation services use unique identifiers for their entities. For example, the 14th Dalai Lama has the VIAF ID [38242123](https://viaf.org/viaf/38242123/) and the Wikidata ID [Q17293](https://www.wikidata.org/wiki/Q37349). You can supply these identifiers directly to your chosen reconciliation service in order to pull more data, but these strings will not be “reconciled” against the external dataset. - -Select the column with unique identifiers and apply the operation ReconcileUse values as identifiers. This will bring up the list of reconciliation services you have already added (to add a new service, open the Start reconciling... window first). If you use this operation on a column of IDs, you will not have access to the usual reconciliation settings. - -Matching identifiers does not validate them. All cells will appear as dark blue “confirmed” matches. You should check before this operation that the identifiers in the column exist on the target service. - -You may get false positives, which you will need to hover over or click on to identify: - -![Hovering over an error.](/img/reconcileIDerror.png) - -## Reconciling by type {#reconciling-by-type} - -Reconciliation services, once added to OpenRefine, may suggest types from their databases. These types will usually be whatever the service specializes in: people, events, places, buildings, tools, plants, animals, organizations, etc. - -Reconciling against a type may be faster and more accurate, but may result in fewer matches. Some services have hierarchical types (such as “mammal” as a subtype of “animal”). When you reconcile against a more specific type, unmatched values may fall back to the broader type; other services will not do this, so you may need to perform successive reconciliation attempts against different types. Refer to the documentation specific to the reconciliation service to learn more. - -When you select a service from the list, OpenRefine will load some or all available types. Some services will sample the first ten rows of your column to suggest types (check the [“Suggest types” column](https://reconciliation-api.github.io/testbench/)). You will see a service’s types in the reconciliation window: - -![Reconciling using a type.](/img/reconcile-by-type.png) - -In this example, “Person” and “Corporate Name” are potential types offered by the reconciliation API for VIAF. You can also use the Reconcile against type: field to enter in another type that the service offers. When you start typing, this field may search and suggest existing types. For VIAF, you could enter “/book/book” if your column contained publications. You may need to enter the service's own strings precisely instead of attempting to search for a match. - -Types are structured to fit their content: the Wikidata “human” type, for example, can include fields for birth and death dates, nationality, etc. The VIAF “person” type can include nationality and gender. You can use this to [include more properties](#reconciling-with-additional-columns) and find better matches. - -If your column doesn’t fit one specific type offered, you can Reconcile against no particular type. This may take longer. - -We recommend working in batches and reconciling against different types, moving from specific to broad. You can create a facet for Best candidate’s types facet to see which types are being represented. Some candidates may return more than one type, depending on the service. Types may appear in facets by their unique IDs, rather than by their semantic labels (for example, Q5 for “human” in Wikidata). - -## Reconciling with additional columns {#reconciling-with-additional-columns} - -Some of your cells may be ambiguous, in the sense that a string can point to more than one entity: there are dozens of places called “Paris” and many characters, people, and pieces of culture, too. Selecting non-geographic or more localized types can help narrow that down, but if your chosen service doesn't provide a useful type, you can include more properties that make it clear whether you're looking for Paris, France. - -![Reconciling sometimes turns up ambiguous matches.](/img/reconcileParis.gif) - -Including supplementary information can be useful, depending on the service (such as including birthdate information about each person you are trying to reconcile). You can re-reconcile unmatched cells with additional properties, in the right side of the Start reconciling window, under “Also use relevant details from other columns.” The column names in your project will appear in the reconciliation window, with an Include? checkbox next to each one. - -Fill in the As Property field with the type of information you are including. When you start typing, potential fields may pop up (depending on the [“suggest properties” feature](https://reconciliation-api.github.io/testbench/)), such as “birthDate” in the case of ULAN or “Geburtsdatum” in the case of Integrated Authority File (GND). Use the documentation for your chosen service to identify the fields in their terms. - -Some services will not be able to search for the exact name of your desired As Property entry, but you can still manually supply the field name. Refer to the service to choose the most appropriate field, and make sure you enter it correctly. - -![Including a birth-date type.](/img/reconcile-with-property.png) - -## Fetching more data {#fetching-more-data} - -One reason to reconcile to some external service is that it allows you to pull data from that service into your OpenRefine project. There are three ways to do this: - -* Add identifiers for your values -* Add columns from reconciled values -* Add column by fetching URLs. - -### Add entity identifiers column {#add-entity-identifiers-column} - -Once you have selected matches for your cells, you can retrieve the unique identifiers for those cells and create a new column for these, with ReconcileAdd entity identifiers column. You will be asked to supply a column name. New items and other unmatched cells will generate null values in this column. - -### Add columns from reconciled values {#add-columns-from-reconciled-values} - -If the reconciliation service supports [data extension](https://reconciliation-api.github.io/testbench/), then you can augment your reconciled data with new columns using Edit columnAdd columns from reconciled values.... - -For example, if you have a column of chemical elements identified by name, you can fetch categorical information about them such as their atomic number and their element symbol: - -![A screenshare of elements fetching related information.](/img/reconcileelements.gif) - -Once you have chosen reconciliation matches for your cells, selecting Add column from reconciled values... will bring up a window to choose which related information you’d like to import into new columns. You can manually enter desired properties, or select from a list of suggestions. - -The quality of the suggested properties will depend on how you have reconciled your data beforehand: reconciling against a specific type will provide you with the associated properties of that type. For example, GND suggests elements about the “people” type after you've reconciled with it, such as their parents, native languages, children, etc. - -![A screenshot of available properties from GND.](/img/reconcileGND.png) - -If you have left any values unreconciled in your column, you will see “<not reconciled>” in the preview. These will generate blank cells if you continue with the column addition process. - -This process may pull more than one property per row in your data (such as multiple occupations), so you may need to switch into records mode after you've added columns. - -### Add columns by fetching URLs {#add-columns-by-fetching-urls} - -If the reconciliation service cannot extend data, look for a generic web API for that data source, or a structured URL that points to their dataset entities via unique IDs (such as “https://viaf.org/viaf/000000”). You can use the Edit column[Add column by fetching URLs](columnediting#add-column-by-fetching-urls) operation to call this API or URL with the IDs obtained from the reconciliation process. This will require using [expressions](expressions). - -You may not want to pull the entire HTML content of the pages at the ends of these URLs, so look to see whether the service offers a metadata endpoint, such as JSON-formatted data. You can either use a column of IDs, or you can pull the ID from each matched cell during the fetching process. - -For example, if you have reconciled artists to the Getty's ULAN, and [have their unique ULAN IDs as a column](#add-entity-identifiers-column), you can generate a new column of JSON-formatted data by using Add column by fetching URLs and entering the GREL expression `"http://vocab.getty.edu/" + value + ".json"`. For this service, the unique IDs are formatted “ulan/000000” and so the generated URLs look like “http://vocab.getty.edu/ulan/000000.json”. - -Alternatively, you can insert the ID directly from the matched column's reconciliation variables, using a GREL expression like `“http://vocab.getty.edu/” + cell.recon.match.id + “.json”` instead. - -Remember to set an appropriate throttle and to refer to the service documentation to ensure your compliance with their terms. See [the section about this operation](columnediting#add-column-by-fetching-urls) to learn more about the fetching process. - -## Keep all the suggestions made {#keep-all-the-suggestions-made} - -To generate a list of each suggestion made, rather than only the best candidate, you can use a [GREL expression](expressions#GREL). Go to Edit columnAdd column based on this column. To create a list of all the possible matches, use something like - -``` -forEach(cell.recon.candidates,c,c.name).join(", ") -``` - -To get the unique identifiers of these matches instead, use - -``` -forEach(cell.recon.candidates,c,c.id).join(", ") -``` - -This information is stored as a string, without any attached reconciliation information. - -## Writing reconciliation expressions {#writing-reconciliation-expressions} - -OpenRefine supplies a number of variables related specifically to reconciled values. These can be used in GREL and Jython expressions. For example, some of the reconciliation variables are: - -* `cell.recon.match.id` or `cell.recon.match.name` for matched values -* `cell.recon.best.name` or `cell.recon.best.id` for best-candidate values -* `cell.recon.candidates` for all listed candidates of each cell -* `cell.recon.judgment` (the values used in the “judgment” facet) -* `cell.recon.judgmentHistory` (the values used in the “judgment action timestamp” facet) -* `cell.recon.matched` (a “true” or “false” value) - -You can find out more in the [reconciliaton variables](expressions#reconciliaton-variables) section. - -## Exporting reconciled data {#exporting-reconciled-data} - -Once you have data that is reconciled to existing entities online, you may wish to export that data to a user-editable service such as Wikidata. See the section on [uploading your edits to Wikidata](wikidata#upload-edits-to-wikidata) for more information, or the section on [exporting](exporting) to see other formats OpenRefine can produce. - -You can share reconciled data in progress through a [project export or import](exporting#export-a-project), with some preparation. The importing user needs to have the appropriate reconciliation services installed on their OpenRefine instance (by going to Start reconciling and clicking on Add Standard Service...) in advance of opening the project, in order to use candidate and match links. Otherwise, the links will be broken and the user will need to add the reconciliation service and re-reconcile the columns in question. [Wikidata](wikidata) reconciliation data can be shared more easily as the service comes bundled with OpenRefine. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/running.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/running.md deleted file mode 100644 index 725d5a1fe..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/running.md +++ /dev/null @@ -1,506 +0,0 @@ ---- -id: running -title: Running OpenRefine -sidebar_label: Running ---- - -## Starting and exiting {#starting-and-exiting} - -OpenRefine does not require internet access to run its basic functions. Once you download and install it, it runs as a small web server on your own computer, and you access that local web server by using your browser. - -You will see a command line window open when you run OpenRefine. Ignore that window while you work on datasets in your browser. - -No matter how you start OpenRefine, it will load its interface in your computer’s default browser. If you would like to use another browser instead, start OpenRefine and then point your chosen browser at the home screen: [http://127.0.0.1:3333/](http://127.0.0.1:3333/). - -OpenRefine works best on browsers based on Webkit, such as: -* Google Chrome -* Chromium -* Opera -* Microsoft Edge - -We are aware of some minor rendering and performance issues on other browsers such as Firefox. We don't support Internet Explorer. - -You can view and work on multiple projects at the same time by simply having multiple tabs or browser windows open. From the Open Project screen, you can right-click on project names and open them in new tabs or windows. - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - - -#### With openrefine.exe {#with-openrefineexe} -You can run OpenRefine by double-clicking `openrefine.exe` or calling it from the command line. - -If you want to [modify the way `openrefine.exe` opens](#starting-with-modifications), you can edit the `openrefine.l4j.ini` file. - -#### With refine.bat {#with-refinebat} -On Windows, OpenRefine can also be run by using the file `refine.bat` in the program directory. If you start OpenRefine using `refine.bat`, you can do so by opening the file itself, or by calling it from the command line. - -If you call `refine.bat` from the command line, you can [start OpenRefine with modifications](#starting-with-modifications). -If you want to modify the way `refine.bat` opens through double-clicking or using a shortcut, you can edit the `refine.ini` file. - -#### Exiting {#exiting} - -To exit OpenRefine, close all the browser tabs or windows, then navigate to the command line window. To close this window and ensure OpenRefine exits properly, hold down `Control` and press `C` on your keyboard. This will save any last changes to your projects. - - - - - -You can find OpenRefine in your Applications folder, or you can open it using Terminal. - -To run OpenRefine using Terminal: -* Find the OpenRefine application / icon in Finder -* Control-click on the icon and select “Show Package Contents” from the context menu -* This should open a new Finder menu: navigate into the “MacOS” folder -* Control-click on “JavaAppLauncher” -* Choose “Open With” from the menu, and select “Terminal.” - -To exit, close all your OpenRefine browser tabs, go back to the terminal window and press `Command` and `Q` to close it down. - -:::caution Problems starting? -If you are using an older version of OpenRefine or are on an older version of MacOS, [check our Wiki for solutions to problems with MacOS](https://github.com/OpenRefine/OpenRefine/wiki/Installation-Instructions#macos). -::: - - - - - -Use a terminal to launch OpenRefine. First, navigate to the installation folder. Then call the program: - -``` -cd openrefine-3.4.1 - ./refine -``` - -This will start OpenRefine and open your browser to the home screen. - -To exit, close all the browser tabs, and then press `control` and `C` in the terminal window. - -:::caution Did you get a JAVA_HOME error? -“Error: Could not find the ‘java’ executable at ‘’, are you sure your JAVA_HOME environment variable is pointing to a proper java installation?” - -If you see this error, you need to [install and configure a JDK package](installing#linux), including setting up `JAVA_HOME`. -::: - - - - - ---- - -### Troubleshooting {#troubleshooting} - -If you are having problems connecting to OpenRefine with your browser, [check our Wiki for information about browser settings and operating-system issues](https://github.com/OpenRefine/OpenRefine/wiki/FAQ#i-am-having-trouble-connecting-to-openrefine-with-my-browser). - -### Starting with modifications {#starting-with-modifications} - -When you run OpenRefine from a command line, you can change a number of default settings. - - - - - -On Windows, use a slash: - -``` -C:>refine /i 127.0.0.2 /p 3334 -``` - -Get a list of all the commands with `refine /?`. - -|Command|Use|Syntax example| -|---|---|---| -|/w|Path to the webapp|refine /w /path/to/openrefine| -|/m|Memory maximum heap|refine /m 6000M| -|/p|Port|refine /p 3334| -|/i|Interface (IP address, or IP and port)|refine /i 127.0.0.2:3334| -|/d|Enable debugging (on port 8000)|refine /d| -|/x|Enable JMX monitoring for Jconsole and JvisualVM|refine /x| - - - - - -You cannot start the Mac version with modifications using Terminal, but you can modify the way the application starts with [settings within files](#modifications-set-within-files). - - - - - -To see the full list of command-line options, run `./refine -h`. - -|Command|Use|Syntax example| -|---|---|---| -|-w|Path to the webapp|./refine -w /path/to/openrefine| -|-d|Path to the workspace|./refine -d /where/you/want/the/workspace| -|-m|Memory maximum heap|./refine -m 6000M| -|-p|Port|./refine -p 3334| -|-i|Interface (IP address, or IP and port)|./refine -i 127.0.0.2:3334| -|-k|Add a Google API key|./refine -k YOUR_API_KEY| -|-v|Verbosity (from low to high: error,warn,info,debug,trace)|./refine -v info| -|-x|Additional Java configuration parameters (see Java documentation)|| -|--debug|Enable debugging (on port 8000)|./refine --debug| -|--jmx|Enable JMX monitoring for Jconsole and JvisualVM|./refine --jmx| - - - - - ---- - -#### Modifications set within files {#modifications-set-within-files} - -On Windows, you can modify the way `openrefine.exe` runs by editing `openrefine.l4j.ini`; you can modify the way `refine.bat` runs by editing `refine.ini`. - -You can modify the Mac application by editing `info.plist`. - -On Linux, you can edit `refine.ini`. - -Some settings, such as changing memory allocations, are already set inside these files, and all you have to do is change the values. Some lines need to be un-commented to work. - -For example, inside `refine.ini`, you should see: -``` -no_proxy="localhost,127.0.0.1" -#REFINE_PORT=3334 -#REFINE_HOST=127.0.0.1 -#REFINE_WEBAPP=main\webapp - -# Memory and max form size allocations -#REFINE_MAX_FORM_CONTENT_SIZE=1048576 -REFINE_MEMORY=1400M - -# Set initial java heap space (default: 256M) for better performance with large datasets -REFINE_MIN_MEMORY=1400M -... -``` - -##### JVM preferences {#jvm-preferences} - -Further modifications can be performed by using JVM preferences. These JVM preferences are different options and have different syntax than the key/value descriptions used on the command line. - -Some of the most common keys (with their defaults) are: -* The project [autosave](starting#autosaving) frequency: `-Drefine.autosave` (5 [minutes]) -* The workspace director: `-Drefine.data_dir` (/) -* Development mode: `-Drefine.development` (false) -* Headless mode: `-Drefine.headless` (false) -* IP: `-Drefine.host` (127.0.0.1) -* Port: `-Drefine.port` (3333) -* The application folder: `-Drefine.webapp` (main/webapp) - -The syntax is as follows: - - - - - -Locate the `refine.l4j.ini` file, and insert lines in this way: - -``` --Drefine.port=3334 --Drefine.host=127.0.0.2 --Drefine.webapp=broker/core -``` - -In `refine.ini`, use a similar syntax, but set multiple parameters within a single line starting with `JAVA_OPTIONS=`: - -``` -JAVA_OPTIONS=-Drefine.data_dir=C:\Users\user\Documents\OpenRefine\ -Drefine.port=3334 - -``` - - - - -Locate the `info.plist`, and find the `array` element that follows the line - -``` -JVMOptions -``` - -Typically this looks something like: - -``` -JVMOptions - --Xms256M --Xmx1024M --Drefine.version=2.6-beta.1 --Drefine.webapp=$APP_ROOT/Contents/Resource/webapp - -``` - -Add in values such as: - -``` -JVMOptions - --Xms256M --Xmx1024M --Drefine.version=2.6-beta.1 --Drefine.webapp=$APP_ROOT/Contents/Resource/webapp --Drefine.autosave=2 --Drefine.port=3334 - - -``` - - - - - -Locate the `refine.ini` file, and add `JAVA_OPTIONS=` before the `-Drefine.preference` declaration. You can un-comment and edit the existing suggested lines, or add lines: - -``` -JAVA_OPTIONS=-Drefine.autosave=2 -JAVA_OPTIONS=-Drefine.port=3334 -JAVA_OPTIONS=-Drefine.data_dir=usr/lib/OpenRefineWorkspace -``` - - - - - - ---- - -Refer to the [official Java documentation](https://docs.oracle.com/javase/8/docs/technotes/tools/windows/java.html) for more preferences that can be set. - -## The home screen {#the-home-screen} - -When you first launch OpenRefine, you will see a screen with a menu on the left hand side that includes Create Project, Open Project, Import Project, and Language Settings. This is called the “home screen,” where you can manage your projects and general settings. - -In the lower left-hand corner of the screen, you'll see Preferences, Help, and About. - -### Language settings {#language-settings} - -From the home screen, look in the options to the left for Language Settings. You can set your preferred interface language here. This language setting will persist until you change it again in the future. Languages are translated as a community effort; some languages are partially complete and default back to English where unfinished. Currently OpenRefine supports the following languages for 75% or more of the interface: - -* Cebuano -* German -* English (UK) -* English (US) -* Spanish -* Filipino -* French -* Hebrew -* Magyar -* Italian -* Japanese (日本語) -* Portuguese (Brazil) -* Tagalog -* Chinese (简体中文) - -To leave the Language Settings screen, click on the diamond “OpenRefine” logo. - -:::info Help us Translate OpenRefine -We use Weblate to provide translations for the interface. You can check [our profile on Weblate](https://hosted.weblate.org/projects/openrefine/translations/) to see which languages are in the process of being supported. See [our technical reference if you are interested in contributing translation work](../technical-reference/translating) to make OpenRefine accessible to people in other languages. -::: - -### Preferences {#preferences} - -In the bottom left corner of the screen, look for Preferences. At this time you can set preferences using a key/value pair: that is, selecting one of the keys below and setting a value for it. - -|Setting|Key|Value syntax|Default|Example|Version| -|---|---|---|---|---|---| -|Interface language|userLang|[ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) two-digit code|en|fr|—| -|Maximum facets|ui.browsing.listFacet.limit|Number|2000|5000|—| -|Timeout for Google Drive import|googleReadTimeOut|Number (microseconds)|180000|500000|—| -|Timeout for Google Drive authorization|googleConnectTimeOut|Number (microseconds)|180000|500000|—| -|Maximum lag for Wikidata edit retries|wikibase.upload.maxLag|Number (seconds)|5|10|—| -|Display of the reconciliation preview on hover|cell-ui.previewMatchedCells|Boolean|true|false|v3.2| -|Width of the panel for facets/history|ui.browsing.facetsHistoryPanelWidth|Number (pixel)|300|500|v3.5| - -To leave the Preferences screen, click on the diamond “OpenRefine” logo. - -If the preference you’re looking for isn’t here, look at the options you can set from the [command line or in an `.ini` file](#starting-with-modifications). - -## The project screen {#the-project-screen} - -The project screen (or work screen) is where you will spend most of your time once you have [begun to work on a project](starting). This is a quick walkthrough of the parts of the interface you should familiarize yourself with. - -![A screenshot of the project screen.](/img/projectscreen.png) - -### The project bar {#the-project-bar} - -The project bar runs across the very top of the project screen. It contains the the OpenRefine logo, the project title, and the project control buttons on the right side. - -At any time you can close your current project and go back to the home screen by clicking on the OpenRefine logo. If you’d like to open another project in a new browser tab or window, you can right-click on the logo and use “Open in a new tab.” You will lose [your current facets and view settings](#facetfilter) if you close your project (but data transformations will be saved in the [History](#history-undoredo) of the project). - -:::caution -Don’t click the “back” button on your browser - it will likely close your current project and you will lose your facets and view settings. -::: - -You can rename a project at any time by clicking inside the project title, which will turn into a text field. Project names don’t have to be unique, as OpenRefine organizes them based on a unique identifier behind the scenes. - -The Permalink allows you to return to a project at a specific view state - that is, with [facets and filters](facets) applied. The Permalink can help you pick up where you left off if you have to close your project while working with facets and filters. It puts view-specific information directly into the URL: clicking on it will load this current-view URL in the existing tab. You can right-click and copy the Permalink URL to copy the current view state to your clipboard, without refreshing the tab you’re using. - -The Open… button will open up a new browser tab showing the Create Project screen. From here you can change settings, start a new project, or open an existing project. - -Export is a dropdown menu that allows you to pick a format for exporting a dataset. Many of the export options will only export rows and records that are currently visible - the currently selected facets and filters, not the total data in the project. - -Help will open up a new browser tab and bring you to this user manual on the web. - -### The grid header {#the-grid-header} - -The grid header sits below the project bar and above the project grid (where the data of your project is displayed). The grid header will tell you the total number of rows or records in your project, and indicate whether you are in [rows or records mode](exploring#rows-vs-records). - -It will also tell you if you’re currently looking at a select number of rows via facets or filtering, rather than the entire dataset, by displaying either, for example, “180 rows” or “67 matching rows (180 total).” - -Directly below the row number, you have the ability to switch between [row mode and records mode](exploring#rows-vs-records). OpenRefine stores projects persistently in one of the two modes, and displays your data as records by default if you are. - -To the right of the rows/records selection is the array of options for how many rows/records to view on screen at one time. At the far right of the screen you can navigate through your entire dataset one page at a time. - -### Extensions {#extensions} - -The Extensions dropdown offers you options for extending your data - most commonly by uploading your edited statements to Wikidata, or by importing or exporting schema. You can learn more about these functions on the [Wikidata page](wikidata). Other extensions may also add functions to this dropdown menu. - -### The grid {#the-grid} - -The area of the project screen that displays your dataset is called the “grid” (or the “data grid,” or the “project grid”). The grid presents data in a tabular format, which may look like a normal spreadsheet program to you. - -Columns widths are automatically set based on their contents; some column headers may be cut off, but can be viewed by mousing over the headers. - -In each column header you will see a small arrow. Clicking on this arrow brings up a dropdown menu containing column-specific data exploration and transformation options. You will learn about each of these options in the [Exploring data](exploring) and [Transforming data](transforming) sections. - -The first column in every project will always be All, which contains options to flag, star, and do non-column-specific operations. The All column is also where rows/records are numbered. Numbering shows the permanent order of rows and records; a temporary sorting or facet may reorder the rows or show a limited set, but numbering will show you the original identifiers unless you make a permanent change. - -The project grid may display with both vertical and horizontal scrolling, depending on the number and width of columns, and the number of rows/records displayed. You can control the display of the project grid by using [Sort and View options](exploring#sort-and-view). - -Mousing over individual cells will allow you to [edit cells individually](cellediting#edit-one-cell-at-a-time). - -### Facet/Filter {#facetfilter} - -The Facet/Filter tab is one of the main ways of exploring your data: displaying the patterns and trends in your data, and helping you narrow your focus and modify that data. [Facets](facets) and [filters](facets#text-filter) are explained more in [Exploring data](exploring). - -![A screenshot of facets and filters in action.](/img/facetfilter.png) - -In the tab, you will see three buttons: Refresh, Reset all, and Remove all. - -Refreshing your facets will ensure you are looking at the latest information about each facet, for example if you have changed the counts or eliminated some options. - -Resetting your facets will remove any inclusion or exclusion you may have set - the facet options will stay in the sidebar, but your view settings will be undone. - -Removing your facets will clear out the sidebar entirely. If you have written custom facets using [expressions](expressions), these will be lost. - -You can preserve your facets and filters for future use by copying a [Permalink](#the-project-bar). - -### History (Undo/Redo) {#history-undoredo} - -In OpenRefine, any activity that changes the data can be undone. Changes are tracked from the very beginning, when a project is first created. The change history of each project is saved with the project's data, so quitting OpenRefine does not erase the steps you've taken. When you restart OpenRefine, you can view and undo changes that you made before you quit OpenRefine. OpenRefine [autosaves](starting#autosaving) your actions every five minutes by default, and when you close OpenRefine properly (using Ctrl + C). You can [change this interval](running#jvm-preferences). - -Project history gets saved when you export a project archive, and restored when you import that archive to a new installation of OpenRefine. - -![A screenshot of the History (Undo/Redo) tab with 13 steps.](/img/history.png "A screenshot of the History (Undo/Redo) tab with 13 steps.") - -When you click on the Undo / Redo tab in the sidebar of any project, that project’s history is shown as a list of changes in order, with the first “change” being the action of creating the project itself. (That first change, indexed as step zero, cannot be undone.) Here is a sample history with 3 changes: - -``` -0. Create project -1. Remove 7 rows -2. Create new column Last Name based on column Name with grel:value.split(" ") -3. Split 230 cell(s) in column Address into several columns by separator -``` - -The current state of the project is highlighted with a dark blue background. If you move back and forth on the timeline you will see the current state become highlighted, while the actions that came after that state will be grayed out. - -To revert your data back to an earlier state, simply click on the last action in the timeline you want to keep. In the example above, if we keep the removal of 7 rows but revert everything we did after that, then click on “Remove 7 rows.” The last 2 changes will be undone, in order to bring the project back to state #1. - -In this example, changes #2 and #3 will now be grayed out. You can redo a change by clicking on it in the history - everything up to and including it will be redone. - -If you have moved back one or more states, and then you perform a new operation on your data, the later actions (everything that’s greyed out) will be erased and cannot be re-applied. - -The Undo/Redo tab will indicate which step you’re on, and if you’re about to risk erasing work - by saying something like “4/5" or “1/7” at the end. - -#### Reusing operations {#reusing-operations} - -Operations that you perform in OpenRefine can be reused. For example, a formula you wrote inside one project can be copied and applied to another project later. - -To reuse one or more operations, first extract it from the project where it was first applied. Click to the Undo/Redo tab and click Extract…. This brings up a box that lists all operations up to the current state (it does not show undone operations). Select the operation or operations you want to extract using the checkboxes on the left, and they will be encoded as JSON on the right. Copy that JSON to the clipboard. - -Move to the second project, go to the Undo/Redo tab, click Apply… and paste in that JSON. - -Not all operations can be extracted. Edits to a single cell, for example, can’t be replicated. - -## Advanced OpenRefine uses {#advanced-openrefine-uses} - -### Running OpenRefine's Linux version on a Mac {#running-openrefines-linux-version-on-a-mac} - -You can run OpenRefine from the command line in Mac by using the Linux installation package. We do not promise support for this method. Follow the instructions in the Linux section. - -### Running as a server {#running-as-a-server} - -:::caution -Please note that if your machine has an external IP (is exposed to the Internet), you should not do this, or should protect it behind a proxy or firewall, such as nginx. Proceed at your own risk. -::: - -By default (and for security reasons), OpenRefine only listens to TCP requests coming from localhost (127.0.0.1) on port 3333. If you want to share your OpenRefine instance with colleagues and respond to TCP requests to any IP address of the machine, start it from the command line like this: -``` -./refine -i 0.0.0.0 -``` - -or set this option in `refine.ini`: -``` -REFINE_HOST=0.0.0.0 -``` - -or set this JVM option: -``` --Drefine.host=0.0.0.0 -``` - -On Mac, you can add a specific entry to the `Info.plist` file located within the app bundle (`/Applications/OpenRefine.app/Contents/Info.plist`): -``` -JVMOptions - - - -Drefine.host=0.0.0.0 - … - -``` - -:::caution -OpenRefine has no built-in security or version control for multi-user scenarios. OpenRefine has a single data model that is not shared, so there is a risk of data operations being overwritten by other users. Care must be taken by users. -::: - -### Automating OpenRefine {#automating-openrefine} - -Some users may wish to employ OpenRefine for batch processing as part of a larger automated pipeline. Not all OpenRefine features can work without human supervision and advancement (such as clustering), but many data transformation tasks can be automated. - -:::caution -The following are all third-party extensions and code; the OpenRefine team does not maintain them and cannot guarantee that any of them work. -::: - -Some examples: - -* This project allows OpenRefine to be run from the command line using [operations saved in a JSON file](#reusing-operations): [OpenRefine batch processing](https://github.com/opencultureconsulting/openrefine-batch) -* A Python project for applying a JSON file of operations to a data file, outputting the new file, and deleting the temporary project, written by David Huynh and Max Ogden: [Python client library for Google Refine](https://github.com/maxogden/refine-python) -* And the same in Ruby: [Refine-Ruby](https://github.com/maxogden/refine-ruby) -* Another Python client library, by Paul Makepeace: [OpenRefine Python Client Library](https://github.com/PaulMakepeace/refine-client-py) - -To look for other instances, search our Google Groups [for users](https://groups.google.com/g/openrefine) and [for developers](https://groups.google.com/g/openrefine-dev), where [these projects were originally posted](https://groups.google.com/g/openrefine/c/GfS1bfCBJow/m/qWYOZo3PKe4J). diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/sortview.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/sortview.md deleted file mode 100644 index a45a4b887..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/sortview.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -id: sortview -title: Sort and view -sidebar_label: Sort and view ---- - -## Sort {#sort} - -You can temporarily sort your rows by one column. You can sort based on [data type](exploring#data-types): -* text alphabetically or reverse -* numbers by largest or smallest -* dates by earliest or latest -* boolean values by false first or true first. - -You can also choose where to place errors and blank cells in the sorting. Text can be case-sensitive or not: if so, cells that start with lowercase characters will appear ahead of those that start with uppercase characters. - -![A screenshot of the Sort window.](/img/sort.png) - -After you apply a sorting method, you can make it permanent, remove it, reverse it, or apply a subsequent sorting. When it is applied, you’ll find Sort in the project grid header to the right of the rows-display setting, which will show all current sorting settings. - -If you have multiple sorting methods applied, they will work in the order you applied them (represented in order in the Sort menu). For example, you can sort an “authors” column alphabetically, and then sort their books by publication date, for those authors that have more than one book. If you apply those in a different order - sort all the publication dates in the dataset first, and then alphabetically by author - your dataset will look different. - -![Temporarily sorted rows.](/img/sort2.png) - -When the sorting method you've applied is temporary, you will see that the rows retain their original numbering. When you make that sorting method permanent, by selecting Reorder rows permanently, the row numbers will change and the Sort menu in the project grid header will disappear. This will apply all current sorting methods. - -## View {#view} - -You can control what data you view in the grid. On each column, you will see a View menu option. From there, you can “collapse” (hide) that specific column, all other columns, all columns to the left, and all columns to the right. Using the View option that appears in the All column’s dropdown menu, you can collapse all columns, and expand all the columns that you previously collapsed. - -### Show/hide “null” {#showhide-null} - -You can find, under AllView, the option to show and hide [“null” values](exploring#data-types). A small grey “null” will appear in each applicable cell. Remember that a null cell is not the same thing as an empty cell. - -![A screenshot of what a null value looks like.](/img/null.png) diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/starting.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/starting.md deleted file mode 100644 index 819a09df4..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/starting.md +++ /dev/null @@ -1,194 +0,0 @@ ---- -id: starting -title: Starting a project -sidebar_label: Starting a project ---- - -## Overview {#overview} - -An OpenRefine project is started by importing in some existing data - OpenRefine doesn’t allow you to create a dataset from nothing. - -No matter where your data comes from, OpenRefine won’t modify your original data source. It copies all the information from your input, creates its own project file, and stores it in your [workspace directory](installing#set-where-data-is-stored). - -The data and all of your edits are [automatically saved](#autosaving) inside the project file. When you’re finished modifying the data, you can [export it back out](exporting) into the file format of your choice. - -You can also receive and open other people’s projects, or send them yours, by [exporting a project archive](exporting#export-a-project) and [importing it](#import-a-project). - -## Create a project by importing data {#create-a-project-by-importing-data} - -When you start OpenRefine, you’ll be taken to the Create Project screen. You’ll see on the left side of the screen that your options are to: - -* import data from one or more files on your computer -* import data from one or more links on the web -* import data by pasting in text from your clipboard -* import data from a database (using SQL), and -* import one or more Sheets from Google Drive. - -From these sources, you can load any of the following file formats: - -* comma-separated values (CSV) or text-separated values (TSV) -* Text files -* Fixed-width columns -* JSON -* XML -* OpenDocument spreadsheet (ODS) -* Excel spreadsheet (XLS or XLSX) -* PC-Axis (PX) -* MARC -* RDF data (JSON-LD, N3, N-Triples, Turtle, RDF/XML) -* Wikitext - -More formats can be imported by [adding extensions to provide that functionality](https://openrefine.org/download.html). - -If you supply two or more files for one project, the files’ rows will be loaded in the order that you specify, and OpenRefine will create a column at the beginning of the dataset with the source URL or file name in it to help you identify where each row came from. If the files have columns with identical names, the data will load in those columns; if not, the successive files will append all of their new columns to the end of the dataset: - -|File|Fruit|Quantity|Berry|Berry source| -|---|---|---|---|---| -|fruits.csv|Orange|4| -|fruits.csv|Apple|6| -|berries.csv||9|Mulberry|Greece| -|berries.csv||2|Blueberry|Canada| - -You cannot combine two datasets into one project by appending data within rows. You can, however, combine two projects later using functions such as [cross()](grelfunctions/#crosscell-s-projectname-s-columnname), or [fetch further data](columnediting) using other methods. - -For whichever method you choose to start your project, when you click Next >> you will be given a preview and a chance to configure the way OpenRefine interprets the data you input. - -### Get data from this computer {#get-data-from-this-computer} - -Click on Browse… and select a file (or several) on your hard drive. All files will be shown, not just compatible ones. - -If you import an archive file (something with the extension `.zip`, `.tar.gz`, `.tgz`, `.tar.bz2`, `.gz`, or `.bz2`), OpenRefine detects the files inside it, shows you a preview screen, and allows you to select which ones to load. This does not work with `.rar` files. - -### Web addresses (URLs) {#web-addresses-urls} - -Type or paste the URL to a data file into the field provided. You can add as many fields as you want. OpenRefine will download the file and preview the project for you. - -If you supply two or more file URLs, OpenRefine will identify each one and ask you to choose which (or all) to load. - -Do not use this form to load a Google Sheet by its link; use [the Google Data form instead](#google-data). - -### Clipboard {#clipboard} - -You can copy and paste in data from anywhere. OpenRefine will recognize comma-separated, tab-separated, or table-formatted information copied from sources such as word-processing documents, spreadsheets, and tables in PDFs. You can also just paste in a list of items that you want to turn into rows. OpenRefine recognizes each new text line as a row. - -This can be useful if you want to pre-select a specific number of rows from your source data, or paste together rows from different places, rather than delete unwanted rows later in the project interace. - -This can also be useful if you would like to paste in a list of URLs, which you can use later to [fetch more data](columnediting). - -### Database (SQL) {#database-sql} - -If you are an administrator or have SQL access to a database of information, you may want to pull the latest dataset directly from there. This could include an online catalogue, a content management system, or a digital repository or collection management system. You can also load a database (`.db`) file saved locally. You will need to use an [SQL query](https://www.w3schools.com/sql/) to import your intended data. - -There are some publicly-accessible databases you can query, such as [one provided by Rfam](https://docs.rfam.org/en/latest/database.html). The instructions provided by Rfam can help you understand how to connect to and query from other databases. - -OpenRefine can connect to PostgreSQL, MySQL, MariaDB, and SQLite database systems. It will automatically populate the Port field based on which of these you choose, but you can manually edit this if needed. - -If you have a `.db` file, you can supply the path to the file on your computer in the Database field at the bottom of the form. You can leave the rest of the fields blank. - -To import data directly from a database, you will need the database type (such as MySQL), database name, the hostname (either an IP address or the domain that hosts the database), and the port on the host. You will need an account authorized for access, and you may need to add OpenRefine's IP address or host to the "allowable hosts" for that account. You can find that information by pressing Test and getting the IP address from the error message that results. - -You can either connect just once to gather data, or save the connection to use it again later. If you press Connect without saving, OpenRefine will forget all the information you just entered. If you’d like to save the connection, name your connection in a way you will recognize later. Click Save and it will appear in the Saved Connections list on the left. From now on, you can click on the ... ellipsis to the right of the connection you’ve saved, and click Connect. - -If your connection is successful, you will see a Query Editor where you can run your SQL query. OpenRefine will give you an error if you write a statement that tries to modify the source database in any way. - -### Google data {#google-data} - -You have two ways to load in data from Google Sheets: -* providing a link to an accessible Google Sheet (that is, one with link-sharing turned on), and -* selecting a Google Sheet in your Google Drive. - -#### Google Sheet by URL {#google-sheet-by-url} - -You can import data from any Google Sheet that has link-sharing turned on. Paste in a URL that looks something like - -``` -https://docs.google.com/spreadsheets/………/edit?usp=sharing -``` - -This will only work with Sheets, not with any other Google Drive file that might have an available link, including `.xls` and other valid files that are hosted in Google Drive. These links will not work when attempting to start a project [by URL](#web-addresses-urls) either, so you need to download those files to your computer. - -#### Google Sheet from Drive {#google-sheet-from-drive} - -You can authorize OpenRefine to access your Google Drive data and import data from any Google Sheet it finds there. This will include Sheets that belong to you and Sheets that are shared with you, as well as Sheets that are in your trash. - -When you select a Google option (either here, or [when exporting project data to Google Drive or Google Sheets](exporting), you will see a pop-up window that asks you to select a Google account to authorize with. You may see an error message when you authorize: if so, try your import or export operation again and it should succeed. - -OpenRefine will not show spreadsheets that are in your email inbox or stored in any other Google property - only in Drive. It also won’t show all compatible file formats, only Sheets files. - -OpenRefine will generate a list of all Sheets it finds, with the most recently modified Sheets at the top. If a file you’ve just added isn’t showing in this list, you can close and restart OpenRefine, or simply navigate to an existing project, open it, then head back to the Create Project window and check again. - -When you click Preview the Sheet will open in a new browser tab. When you click the Sheet title, OpenRefine will begin to process the data. - - -## Project preview {#project-preview} - -Once OpenRefine is ready to import the data, you will see a screen with Configure Parsing Options at the top. You’ll see a preview of the first 100 rows and all identified columns. - -At the bottom of the screen you will find options for telling OpenRefine how to process what it has found. You can tell it which row(s) to parse as column headers, as well as to ignore any number of rows at the top. You can also select a specific range of rows to work with, by discarding some rows at the top (excluding the header) and limiting the total number of rows it loads. - -OpenRefine tries to guess how to parse your data based on the file extension. For example, `.xml` files are going to be parsed as though they are formatted in XML. An unknown file extension (or your clipboard copy-paste) is assumed to be either tab-separated or comma-separated. OpenRefine looks for a tab character, and if one is found, it assumes you have imported tab-separated data. - -If OpenRefine isn’t certain what format you imported, it will provide a list of possibilities under Parse data as and some settings. You can specify a custom separator now, or split columns later while [transforming your data](transforming). - -If you imported a spreadsheet with multiple worksheets, they will be listed along with the number of rows they contain. You can only select data from one worksheet. - -Note that OpenRefine does not preserve any formatting, such as cell or text colour, that my have been in the original data file. Hyperlinked text will be input as plain text, but OpenRefine will recognize links and make them clickable inside the project interface. - -:::info Encoding issues? -Look for character encoding issues at this stage. You may want to manually select an encoding, such as UTF-8, UTF-16, or ASCII, if OpenRefine does not display some characters correctly in the preview. Once your project is created, you can specify another encoding for specific columns using the [reinterpret() function](grelfunctions#reinterprets-s-encoder). -::: - -You should create a project name at this stage. You can also supply tags to keep your projects organized. When you’re happy with the preview, click Create Project. - - -## Import a project {#import-a-project} - -Because OpenRefine only runs locally on your computer, you can’t have a project accessible to more than one person at the same time. - -The best way to collaborate with another person is to export and import projects that save all your changes, so that you can pick up where someone else left off. You can also [export projects](exporting#export-a-project) and import them to other computers, such as for working on the same project from the office and from home. - -An exported project will include all of the [history](running#history-undoredo), so you can see (and undo) all the changes from the previous user. It is essentially a point-in-time snapshot of their work. OpenRefine only exports projects as `.tar.gz` files at this time. -:::caution -If you wish to hide the original state of your data and your history of edits (for example, if you are using OpenRefine to anonymize information), export your cleaned dataset only and do not share your project archive. -::: - -Once someone has sent you a project archive file from their computer, you can save it anywhere. OpenRefine will import it like a new project and save its information to your workspace directory. - -In the left-hand menu of the home screen, click Import Project. Click Browse… and navigate to wherever you saved the file you were sent (for example, your Downloads folder). - -You can rename the project if you’d like - we recommend adding your name, a date, or a version number, if you’re planning to continue collaborating with another person (or working from multiple computers). - -Then, click Import Project. Your project should appear with a step count beside Undo/Redo if steps were saved by the exporter. - -OpenRefine will store the project in its own workspace directory, so you can now delete the original file that was sent to you. - - -## Project management {#project-management} - -You can access all of your created projects by clicking on Open Project. Your project list can be organized by modification date, title, row count, and other metadata you can supply (such as subject, descripton, tags, or creator). To edit the fields you see here, click About to the left of each project. There you can edit a number of available fields. You can also see the project ID that corresponds to the name of the folder in your work directory. - -### Naming projects {#naming-projects} - -You may have multiple projects from the same dataset, or multiple versions from sharing a project with another person. OpenRefine automatically generates a project name from the imported file, or “clipboard” when you use Clipboard importing. Project names don’t have to be unique, and OpenRefine will create many projects with the same name unless you intervene. - -You can edit a project's name when you create it or import it, and you can rename a project later by opening it and clicking on the project name at the top of the screen. - -### Autosaving {#autosaving} - -OpenRefine [saves all of your actions](running#history-undoredo) (everything you can see in the Undo/Redo panel). That includes flagging and starring rows. - -It doesn’t, however, save your facets, filters, or any kind of view you may have in place while you work. This includes the number of rows showing, and any sorting or column collapsing you may have done. A good rule of thumb is: if it’s not showing in Undo/Redo, you will lose it when you leave the project workspace. - -Autosaving happens by default every five minutes. You can [change this preference by following these directions](running#jvm-preferences). - -You can only save and share facets and filters, not any other type of view. To save current facets and filters, click Permalink. The project will reload with a different URL, which you can then copy and save elsewhere. This permalink will save both the facets and filters you’ve set, and the settings for each one (such as sorting by count rather than by name). - -### Deleting projects {#deleting-projects} - -You can delete projects, which will erase the project files from the workspace directory on your computer. This is immediate and cannot be undone. - -Go to Open Project and find the project you want to delete. Click on the X to the left of the project name. There will be a confirmation dialog. - -### Project files {#project-files} - -You can find all of your raw project files in your work directory. They will be named according to the unique “Project ID” that OpenRefine has assigned them, which you can find on the Open Project screen, under the “About” link for each project. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/transforming.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/transforming.md deleted file mode 100644 index 53a436370..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/transforming.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -id: transforming -title: Transforming data -sidebar_label: Overview ---- - -## Overview {#overview} - -OpenRefine gives you powerful ways to clean, correct, codify, and extend your data. Without ever needing to type inside a single cell, you can automatically fix typos, convert things to the right format, and add structured categories from trusted sources. - -This section of ways to improve data are organized by their appearance in the menu options in OpenRefine. You can: - -* change the order of [rows](#edit-rows) or [columns](columnediting#rename-remove-and-move) -* edit [cell contents](cellediting) within a particular column -* [transform](transposing) rows into columns, and columns into rows -* [split or join columns](columnediting#split-or-join) -* [add new columns](columnediting) based on existing data, with fetching new information, or through [reconciliation](reconciling) -* convert your rows of data into [multi-row records](exploring#rows-vs-records). - -## Edit rows {#edit-rows} - -Moving rows around is a permanent change to your data. - -You can [sort your data](sortview#sort) based on the values in one column, but that change is a temporary view setting. With that setting applied, you can make that new order permanent. - -![A screenshot of where to find the Sort menu with a sorting applied.](/img/sortPermanent.png) - -In the project grid header, the word “Sort” will appear when a sort operation is applied. Click on it to show the dropdown menu, and select Reorder rows permanently. You will see the numbering of the rows change under the All column. - -:::info Reordering all rows -Reordering rows permanently will affect all rows in the dataset, not just those currently viewed through [facets and filters](facets). -::: - -You can undo this action using the [History tab](running#history-undoredo). \ No newline at end of file diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/transposing.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/transposing.md deleted file mode 100644 index 6b5ec0dc8..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/transposing.md +++ /dev/null @@ -1,234 +0,0 @@ ---- -id: transposing -title: Transposing -sidebar_label: Transposing ---- - -## Overview {#overview} - -These functions were created to solve common problems with reshaping your data: pivoting cells from a row into a column, or pivoting cells from a column into a row. You can also transpose from a repeated set of values into multiple columns. - -## Transpose cells across columns into rows {#transpose-cells-across-columns-into-rows} - -Imagine personal data with addresses in this format: - -|Name|Street|City|State/Province|Country|Postal code| -|---|---|---|---|---|---| -|Jacques Cousteau|23, quai de Conti|Paris||France|75270| -|Emmy Noether|010 N Merion Avenue|Bryn Mawr|Pennsylvania|USA|19010| - -You can transpose the address information from this format into multiple rows. Go to the “Street” column and select TransposeTranspose cells across columns into rows. From there you can select all of the five columns, starting with “Street” and ending with “Postal code,” that correspond to address information. Once you begin, you should put your project into [records mode](exploring#rows-vs-records) to associate the subsequent rows with “Name” as the key column. - -![A screenshot of the transpose across columns window.](/img/transpose1.png) - -### One column {#one-column} - -You can transpose the multiple address columns into a series of rows: - -|Name|Address| -|---|---| -|Jacques Cousteau|23, quai de Conti| -| |Paris| -| |France| -| |75270| -|Emmy Noether|010 N Merion Avenue| -||Bryn Mawr| -||Pennsylvania| -||USA| -||19010| - -You can choose one column and include the column-name information in each cell by prepending it to the value, with or without a separator: - -|Name|Address| -|---|---| -|Jacques Cousteau|Street: 23, quai de Conti| -| |City: Paris| -| |Country: France| -| |Postal code: 75270| -|Emmy Noether|Street: 010 N Merion Avenue| -||City: Bryn Mawr| -||State/Province: Pennsylvania| -||Country: USA| -||Postal code: 19010| - -### Two columns {#two-columns} - -You can retain the column names as separate cell values, by selecting Two new columns and naming the key and value columns. - -|Name|Address part|Address| -|---|---|---| -|Jacques Cousteau|Street|23, quai de Conti| -| |City|Paris| -| |Country|France| -| |Postal code|75270| -|Emmy Noether|Street|010 N Merion Avenue| -||City|Bryn Mawr| -||State/Province|Pennsylvania| -||Country|USA| -||Postal code|19010| - -## Transpose cells in rows into columns {#transpose-cells-in-rows-into-columns} - -Imagine employee data in this format: - -|Column| -|---| -|Employee: Karen Chiu| -|Job title: Senior analyst| -|Office: New York| -|Employee: Joe Khoury| -|Job title: Junior analyst| -|Office: Beirut| -|Employee: Samantha Martinez| -|Job title: CTO| -|Office: Tokyo| - -The goal is to sort out all of the information contained in one column into separate columns, but keep it organized by the person it represents: - -|Name |Job title |Office| -|---|---|---| -|Karen Chiu |Senior analyst |New York| -|Joe Khoury |Junior analyst |Beirut| -|Samantha Martinez |CTO |Tokyo| - -By selecting TransposeTranspose cells in rows into columns... a window will appear that simply asks how many rows to transpose. In this case, each employee record has three rows, so input “3” (do not subtract one for the original column). The original column will disappear and be replaced with three columns, with the name of the original column plus a number appended. - -|Column 1 |Column 2 |Column 3| -|---|---|---| -|Employee: Karen Chiu |Job title: Senior analyst |Office: New York| -|Employee: Joe Khoury |Job title: Junior analyst |Office: Beirut| -|Employee: Samantha Martinez |Job title: CTO |Office: Tokyo| - -From here you can use Cell editingReplace to remove “Employee: ”, “Job title: ”, and “Office: ” if you wish, or use [expressions](expressions) with Edit cellsTransform... to clean out the extraneous characters: - -``` -value.replace("Employee: ", "") -``` - -If your dataset doesn't have a predictable number of cells per intended row, such that you cannot specify easily how many columns to create, try Columnize by key/value columns. - -## Columnize by key/value columns {#columnize-by-keyvalue-columns} - -This operation can be used to reshape a dataset that contains key and value columns: the repeating strings in the key column become new column names, and the contents of the value column are moved to new columns. This operation can be found at TransposeColumnize by key/value columns. - -![A screenshot of the Columnize window.](/img/transpose2.png) - -Consider the following example, with flowers, their colours, and their International Union for Conservation of Nature (IUCN) identifiers: - -|Field |Data | -|--------|----------------------| -|Name |Galanthus nivalis | -|Color |White | -|IUCN ID |162168 | -|Name |Narcissus cyclamineus | -|Color |Yellow | -|IUCN ID |161899 | - -In this format, each flower species is described by multiple attributes on consecutive rows. The “Field” column contains the keys and the “Data” column contains the values. In the Columnize by key/value columns window you can select each of these from the available columns. It transforms the table as follows: - -| Name | Color | IUCN ID | -|-----------------------|----------|---------| -| Galanthus nivalis | White | 162168 | -| Narcissus cyclamineus | Yellow | 161899 | - -### Entries with multiple values in the same column {#entries-with-multiple-values-in-the-same-column} - -If a new row would have multiple values for a given key, then these values will be grouped on consecutive rows, to form a [record structure](exploring#rows-vs-records). - -For instance, flowers can have multiple colors: - -| Field | Data | -|-------------|-----------------------| -| Name | Galanthus nivalis | -| _Color_ | _White_ | -| _Color_ | _Green_ | -| IUCN ID | 162168 | -| Name | Narcissus cyclamineus | -| Color | Yellow | -| IUCN ID | 161899 | - -This table is transformed by the Columnize operation to: - -| Name | Color | IUCN ID | -|-----------------------|----------|---------| -| Galanthus nivalis | White | 162168 | -| | Green | | -| Narcissus cyclamineus | Yellow | 161899 | - -The first key encountered by the operation serves as the record key, so the “Green” value is attached to the “Galanthus nivalis” name. See the [Row order](#row-order) section for more details about the influence of row order on the results of the operation. - -### Notes column {#notes-column} - -In addition to the key and value columns, you can optionally add a column for notes. This can be used to store extra metadata associated to a key/value pair. - -Consider the following example: - -| Field | Data | Source | -|---------|---------------------|-----------------------| -| Name | Galanthus nivalis | IUCN | -| Color | White | Contributed by Martha | -| IUCN ID | 162168 | | -| Name | Narcissus cyclamineus | Legacy | -| Color | Yellow | 2009 survey | -| IUCN ID | 161899 | | - -If the “Source” column is selected as the notes column, this table is transformed to: - -| Name | Color | IUCN ID | Source: Name | Source: Color | -|-----------------------|----------|---------|---------------|-----------------------| -| Galanthus nivalis | White | 162168 | IUCN | Contributed by Martha | -| Narcissus cyclamineus | Yellow | 161899 | Legacy | 2009 survey | - -Notes columns can therefore be used to preserve provenance or other context about a particular key/value pair. - -### Row order {#row-order} - -The order in which the key/value pairs appear matters. The Columnize operation will use the first key it encounters as the delimiter for entries: every time it encounters this key again, it will produce a new row, and add the following key/value pairs to that row. - -Consider for instance the following table: - -| Field | Data | -|----------|-----------------------| -| _Name_ | Galanthus nivalis | -| Color | White | -| IUCN ID | 162168 | -| _Name_ | Crinum variabile | -| _Name_ | Narcissus cyclamineus | -| Color | Yellow | -| IUCN ID | 161899 | - -The occurrences of the “Name” value in the “Field” column define the boundaries of the entries. Because there is no other row between the “Crinum variabile” and the “Narcissus cyclamineus” rows, the “Color” and “IUCN ID” columns for the “Crinum variabile” entry will be empty: - -| Name | Color | IUCN ID | -|-----------------------|----------|---------| -| Galanthus nivalis | White | 162168 | -| Crinum variabile | | | -| Narcissus cyclamineus | Yellow | 161899 | - -This sensitivity to order is removed if there are extra columns: in that case, the first extra column will serve as the key for the new rows. - -### Extra columns {#extra-columns} - -If your dataset contains extra columns, that are not being used as the key, value, or notes columns, they can be preserved by the operation. For this to work, they must have the same value in all old rows corresponding to a new row. - -In the following example, the “Field” and “Data” columns are used as key and value columns respectively, and the “Wikidata ID” column is not selected: - -| Field | Data | Wikidata ID | -|---------|-----------------------|-------------| -| Name | Galanthus nivalis | Q109995 | -| Color | White | Q109995 | -| IUCN ID | 162168 | Q109995 | -| Name | Narcissus cyclamineus | Q1727024 | -| Color | Yellow | Q1727024 | -| IUCN ID | 161899 | Q1727024 | - -This will be transformed to: - -| Wikidata ID | Name | Color | IUCN ID | -|-------------|-----------------------|----------|---------| -| Q109995 | Galanthus nivalis | White | 162168 | -| Q1727024 | Narcissus cyclamineus | Yellow | 161899 | - -This actually changes the operation: OpenRefine no longer looks for the first key (“Name”) but simply pivots all information based on the first extra column's values. Every old row with the same value gets transposed into one new row. If you have more than one extra column, they are pivoted as well but not used as the new key. - -You can use [Fill down](cellediting#fill-down-and-blank-down) to put identical values in the extra columns if you need to. \ No newline at end of file diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/troubleshooting.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/troubleshooting.md deleted file mode 100644 index 67b4aef6c..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/troubleshooting.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -id: troubleshooting -title: Troubleshooting -sidebar_label: Troubleshooting ---- - -## Frequently asked questions {#frequently-asked-questions} - -We collect and share FAQs and responses on Github at [https://github.com/OpenRefine/OpenRefine/wiki/FAQ](https://github.com/OpenRefine/OpenRefine/wiki/FAQ). - -If you don’t find your problem and solution there, continue on to the resources in the Community section below to see more conversations and look for solutions. - -## Community {#community} - -### If you’re having a problem: {#if-youre-having-a-problem} -* Search the [User forum](https://groups.google.com/g/openrefine) to see if the problem is already reported -* Search [Github issues](https://github.com/OpenRefine/OpenRefine/issues) to see if the problem is already reported -* Read [Stack Overflow](https://stackoverflow.com/questions/tagged/openrefine) to see if others had a similar problem -* Check [Twitter](https://twitter.com/search?f=tweets&vertical=default&q=OpenRefine%20OR%20%22Open%20Refine%22%20OR%20%23OpenRefine&src=typd) to see if others are discussing the problem -* Report an issue: - * First as a new thread (conversation) in the [User forum](https://groups.google.com/g/openrefine). - * Then, if you wish, you can create a Github issue. - -### If you want to contribute: {#if-you-want-to-contribute} -* [Help us translate the tool into more languages](../technical-reference/translating), using Weblate -* [We have a guide to contributing](../technical-reference/contributing) in the [Technical Reference](../technical-reference/technical-reference-index) section -* Contribute your feature requests in the [User forum](https://groups.google.com/g/openrefine) or as [Github issues](https://github.com/OpenRefine/OpenRefine/issues/new/choose) -* Join the User Forum and/or the [Developer Forum](https://groups.google.com/g/openrefine-dev) -* Share your successes and use cases with us, in the User forum -* Add your [blog posts, guides, tips, tricks, tutorials to our list](https://github.com/OpenRefine/OpenRefine/wiki/External-Resources) -* Keep an eye out for and respond to our biennial user survey. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/manual/wikidata.md b/OpenRefine/docs/versioned_docs/version-3.4/manual/wikidata.md deleted file mode 100644 index 6f462f1af..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/manual/wikidata.md +++ /dev/null @@ -1,190 +0,0 @@ ---- -id: wikidata -title: Wikidata -sidebar_label: Wikidata ---- - -## Overview {#overview} - -OpenRefine provides powerful ways to both pull data from Wikidata and add data to it. - -You do not need a Wikidata account to reconcile your local OpenRefine project to Wikidata. If you wish to [upload your cleaned dataset to Wikidata](#editing-wikidata-with-openrefine), you will need an [autoconfirmed](https://www.wikidata.org/wiki/Wikidata:Autoconfirmed_users) account, and you must [authorize OpenRefine with that account](#manage-wikidata-account). - -:::info A better resource -The best source for information about how OpenRefine works with Wikidata is [on Wikidata itself, under Tools](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine). That page has tutorials, guidelines on editing, and spaces for discussion and help. The following text on this page reviews the basics and can help you get set up, but the Wikidata help page is more regularly updated when technology or policies change. Links to the Wikidata help page are included throughout this page. -::: - -OpenRefine’s connections to Wikidata were formerly an optional extension, but are now included automatically with installation. The Wikidata extension can be removed manually by navigating to your OpenRefine installation folder, and then looking inside `webapp/extensions/` and deleting the `wikidata` folder found there. - -## Reconciling with Wikidata {#reconciling-with-wikidata} - -The Wikidata [reconciliation service](reconciling) for OpenRefine [supports](https://reconciliation-api.github.io/testbench/): -* A large number of potential types to reconcile against -* Previewing and viewing entities -* Suggesting entities, types, and properties -* Augmenting your project with more information pulled from Wikidata. - -You can find documentation and further resources on the reconciliation API [here](https://wikidata.reconci.link/). - -For the most part, Wikidata reconciliation behaves the same way other reconciliation services do, but there are a few processes and features specific to Wikidata. - -### Language settings {#language-settings} - -You can install a version of the Wikidata reconciliation service that uses your language. First, you need the language code: this is the [two-letter code found on this list](https://en.wikipedia.org/wiki/List_of_Wikipedias), or in the domain name of the desired Wikipedia/Wikidata (for instance, “fr” if your Wikipedia is https://fr.wikipedia.org/wiki/). - -Then, open the reconciliation window (under ReconcileStart reconciling...) and click Add Standard Service. The URL to enter is `https://wikidata.reconci.link/fr/api`, where “fr” is your desired language code. - -When reconciling using this interface, items and properties will be displayed in your chosen language if the label is available. The matching score of the reconciliation is not influenced by your choice of language for the service: items are matched by considering all labels and returning the best possible match. The language of your dataset is also irrelevant to your choice of language for the reconciliation service; it simply determines which language labels to return based on the entity chosen. - -### Restricting matches by type {#restricting-matches-by-type} - -In Wikidata, types are items themselves. For instance, the [university of Ljubljana (Q1377)](https://www.wikidata.org/wiki/Q1377) has the type [public university (Q875538)](https://www.wikidata.org/wiki/Q875538), using the [instance of (P31)](https://www.wikidata.org/wiki/Property:P31) property. Types can be subclasses of other types, using the [subclass of (P279)](https://www.wikidata.org/wiki/Property:P279) property. For instance, [public university (Q875538)](https://www.wikidata.org/wiki/Q875538) is a subclass of [university (Q3918)](https://www.wikidata.org/wiki/Q3918). You can visualize these structures with the [Wikidata Graph Builder](https://angryloki.github.io/wikidata-graph-builder/). - -When you select or enter a type for reconciliation, OpenRefine will include that type and all of its subtypes. For instance, if you select [university (Q3918)](https://www.wikidata.org/wiki/Q3918), then [university of Ljubljana (Q1377)](https://www.wikidata.org/wiki/Q1377) will be a possible match, though that item isn't directly linked to Q3918 - because it is directly linked to Q875538, the subclass of Q3918. - -Some items and types may not yet be set as an instance or subclass of anything (because Wikidata is crowdsourced). If you restrict reconciliation to a type, items without the chosen type will not appear in the results, except as a fallback, and will have a lower score. - -### Reconciling via unique identifiers {#reconciling-via-unique-identifiers} - -You can supply a column of unique identifiers (in the form "Q###" for entities) directly to Wikidata in order to pull more data, but [these strings will not be “reconciled” against the external dataset](reconciling#reconciling-with-unique-identifiers). Apply the operation ReconcileUse values as identifiers on your column of QIDs. All cells will appear as dark blue “confirmed” matches. Some of the “matches” may be errors, which you will need to hover over or click on to identify. You cannot use this to reconcile properties (in the form "P###"). - -If the identifier you submit is assigned to multiple Wikidata items (because Wikidata is crowdsourced), all of the items are returned as candidates, with none automatically matched. - -### Property paths, special properties, and subfields {#property-paths-special-properties-and-subfields} - -Wikidata's hierarchical property structure can be called by using property paths (using |, /, and . symbols). Labels, aliases, descriptions, and sitelinks can also be accessed. You can also match values against subfields, such as latitude and longitude subfields of a geographical coordinate. - -For information on how to do this, read the [documentation and further resources here](https://wikidata.reconci.link/#documentation). - -## Editing Wikidata with OpenRefine {#editing-wikidata-with-openrefine} - -The best resource is the [Editing section](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing) on Wikidata. - -As a user-maintained data source, Wikidata can be edited by anyone. OpenRefine makes it simple to upload information in bulk. You simply need to get your information into the correct format, and ensure that it is new (not redundant to information already on Wikidata) and does not conflict with existing Wikidata information. - -Wikidata is built by creating entities (such as people, organizations, or places, identified with unique numbers starting with Q), defining properties (unique numbers starting with P), and using properties to define relationships between entities (a Q has a property P, with a value of another Q). - -For example, you may wish to create entities for local authors and the books they've set in your community. Each writer will be an entity with the occupation [author (Q482980)](https://www.wikidata.org/wiki/Q482980), each book will be an entity with the property “instance of” ([P31](https://www.wikidata.org/wiki/Property:P31)) linking it to a class such as [literary work (Q7725634)](https://www.wikidata.org/wiki/Q7725634), and books will be related to authors through a property [author (P50)](https://www.wikidata.org/wiki/Property:P50). Books can have places where they are set, with the property [narrative location (P840)](https://www.wikidata.org/wiki/Property:P840). - -To do this with OpenRefine, you'll need a column of publication titles that you have reconciled (and create new items where needed); each publication will have one or more locations in a “setting” column, which is also reconciled to municipalities or regions where they exist (and create new items where needed). Then you can add those new relationships, and create new entities for authors, books, and places where needed. You do not need columns for properties; those are defined later, in the creation of your [schema](#edit-wikidata-schema). - -There is a list of [tutorials and walkthroughs on Wikidata](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing) that will allow you to see the full process. You can save your schemas and drafts in OpenRefine, and your progress stays in draft until you are ready to upload it to Wikidata. You can also find information on [how to design a schema](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Schema_alignment) and [how OpenRefine evaluates your proposed edits for issues](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Quality_assurance). - -Batches of edits to Wikidata that are created with OpenRefine can be undone. You can test out the uploading process by reconciling to several “sandbox” entities created specifically for drafting edits and learning about Wikidata: -* https://www.wikidata.org/wiki/Q4115189 -* https://www.wikidata.org/wiki/Q13406268 -* https://www.wikidata.org/wiki/Q15397819 -* https://www.wikidata.org/wiki/Q64768399 - -If you upload edits that are redundant (that is, all the statements you want to make have already been made), nothing will happen. If you upload edits that conflict with existing information (such as a different birthdate than one already in Wikidata), it will be added as a second statement. OpenRefine produces no warnings as to whether your data replicates or conflicts with existing Wikidata elements. - -You can use OpenRefine's reconciliation preview to look at the target Wikidata elements and see what information they already have, and whether the elements' histories have had similar edits reverted in the past. - -### Edit Wikidata schema {#edit-wikidata-schema} - -The best resource is the [Schema alignment page](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Schema_alignment) on Wikidata. - -A [schema](https://en.wikipedia.org/wiki/Database_schema) is a plan for how to structure information in a database. In OpenRefine, the schema operates as a template for how Wikidata edits should be applied: how to translate your tabular data into statements. With a schema, you can: -* preview the Wikidata edits and inspect them manually; -* analyze and fix any issues highlighted by OpenRefine; -* upload your changes to Wikidata by logging in with your own account; -* export the changes to the QuickStatements v1 format. - -For example, if your dataset has columns for authors, publication titles, and publication years, your schema can be conceptualized as: [publication title] has the author [author], and was published in [publication year]. To establish these facts, you need to establish one or more columns as “items,” for which you will make “statements” that relate them to other columns. - -You can export any schema you create, and import an existing schema for use with a new dataset. This can help you work in batches on a large amount of data while minimizing redundant labor. - -Once you select Edit Wikidata schema under the Extensions dropdown menu, your project interface will change. You’ll see new tabs added to the right of “X rows/records" in the grid header: “Schema,” “Issues,” and “Preview.” You can now switch between the tabular grid format of your dataset and the screens that allow you to prepare data for uploading. - -OpenRefine presents you with an easy visual way to map out the relationships in your dataset. Each of the columns of your project will appear at the top of the sceren, and you can simply drag and drop them into the appropriate slots. To get start, select one column as an item. - -![A screenshot of the schema construction window in OpenRefine.](/img/wikidata-schema.png) - -You may wish to refer to [this Wikidata tutorial on how OpenRefine handles Wikidata schema](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Tutorials/Basic_editing). - -#### Editing terms with your schema {#editing-terms-with-your-schema} - -With OpenRefine, you can edit the terms (labels, aliases, descriptions, or sitelinks) of Wikidata entities as well as establish relationships between entities. For example, you may wish to upload pseudonyms, pen names, maiden names, or married names for authors. - -![An author with a number of aliases indicating pseudonyms.](/img/wikidata-terms.png) - -You can do so by putting the preferred names in one column of your dataset and alternative names in another column. In the schema interface, add an item for the preferred values, then click “Add term” on the right-hand side of the screen. Select “Alias” from the dropdown, enter in “English” in the language field, and drop your alternative names column into the space. For this example, you should also consider adding those alternative names to the authors' entries using the property [pseudonym (P742)](https://www.wikidata.org/wiki/Property:P742). The "description" and "label" terms can only contain one value, so there is an option to override existing values if needed. Aliases can be potentially infinite. - -![The schema window showing a term being edited.](/img/wikidata-terms2.png) - -Terms must always have an associated language. You can select the term's language by typing in the “lang” field, which will auto-complete for you. You cannot edit multiple languages at once, unless you supply a suitable column instead. For example, suppose you had translated publication titles, with data in the following format: - -|English title|Translated title|Translation language| -|---|---|---| -|Possession|Besessen|German| -||Обладать|Russian| -|Disgrace|Disgrâce|French| -||Vergogna|Italian| -|Wolf Hall|En la corte del lobo|Spanish| -||ウルフ・ホール|Japanese| - -You could upload the “Translated titles” to “Label” with the language specified by “Translation language.” You may wish to fetch the two-letter language code and use that instead for better language matches. - -![Constructing a schema with aliases and languages.](/img/wikidata-translated.png) - -### Manage Wikidata account {#manage-wikidata-account} - -To edit Wikidata directly from OpenRefine, you must log in with a Wikidata account. OpenRefine can only upload edits with Wikidata user accounts that are “[autoconfirmed](https://www.wikidata.org/wiki/Wikidata:Autoconfirmed_users)” - at this time, that means accounts that have more than 50 edits and have existed for longer than four days. - -Use the Extensions menu to select Manage Wikidata account and you will be presented with the following window: - -![The Wikidata authorization window in OpenRefine.](/img/wikidata-login.png) - -For security reasons, you should not use your main account authorization with OpenRefine. Wikidata allows you to set special passwords to access your account through software. You can find [this setting for your account here](https://www.wikidata.org/wiki/Special:BotPasswords) once logged in. Creating bot access will prompt you for a unique name. You should then enable the following required settings: -* High-volume editing -* Edit existing pages -* Create, edit, and move pages - -It will then generate a username (in the form of “yourwikidatausername@yourbotname”) and password for you to use with OpenRefine. - -If your account or your bot is not properly authorized, OpenRefine will not display a warning or error when you try to upload your edits. - -You can store your unencrypted username and password in OpenRefine, saved locally to your computer and available for future use. For security reasons, you may wish to leave this box unchecked. You can also save your OpenRefine-specific bot password in your browser or with a password management tool. - -### Import and export schema {#import-and-export-schema} - -You can save time on repetitive processes by defining a schema on one project, then exporting it and importing for use on new datasets in the future. Or you and your colleagues can share a schema with each other to coordinate your work. - -You can export a schema from a project using ExportWikidata schema, or by using ExtensionsExport schema. OpenRefine will generate a JSON file for you to save and share. You may experience issues with pop-up windows in your browser: consider allowing pop-ups from the OpenRefine URL (`127.0.0.1`) from now on. - -You can import a schema using ExtensionsImport schema. You can upload a JSON file, or paste JSON statements directly into a field in the window. An imported schema will look for columns with the same names, and you will see an error message if your project doesn't contain matching columns. - -### Upload edits to Wikidata {#upload-edits-to-wikidata} - -The best resource is the [Uploading page](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Uploading) on Wikidata. - -There are two menu options in OpenRefine for applying your edits to Wikidata. Under Export you will see Wikidata edits... and under Extensions you will see Upload edits to Wikidata. Both will bring up the same window for you to [log in with a Wikidata account](#manage-wikidata-account). - -Once you are authorized, you will see a window with any outstanding issues. You can ignore these issues, but we recommend you resolve them. - -If you are ready to upload your edits, you can provide an “Edit summary” - a short message describing the batch of edits you are making. It can be helpful to leave notes for yourself, such as “batch 1: authors A-G” or other indicators of your workflow progress. OpenRefine will show the progress of the upload as it is happening, but does not show a confirmaton window. - -If your edits have been successful, you will see them listed on [your Wikidata user contributions page](https://www.wikidata.org/wiki/Special:Contributions/), and on the [Edit groups page](https://editgroups.toolforge.org/). All edits can be undone from this second interface. - -### QuickStatements export {#quickstatements-export} - -Your OpenRefine data can be exported in a format recognized by [QuickStatements](https://www.wikidata.org/wiki/Help:QuickStatements), a tool that creates Wikidata edits using text commands. OpenRefine generates “version 1” QuickStatements commands. - -There are advantages to using QuickStatements rather than uploading your edits directly to Wikidata, including the way QuickStatements resolves duplicates and redundancies. You can learn more on QuickStatements' [Help page](https://www.wikidata.org/wiki/Help:QuickStatements), and on OpenRefine's [Uploading page](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Uploading). - -In order to use QuickStatements, you must authorize it with a Wikidata account that is [autoconfirmed](https://www.wikidata.org/wiki/Wikidata:Autoconfirmed_users) (it may appear as “MediaWiki” when you authorize). - -Follow the [steps listed on this page](https://www.wikidata.org/wiki/Help:QuickStatements#Running_QuickStatements). -To prepare your OpenRefine data into QuickStatements, select ExportQuickStatements file, or ExtensionsExport to QuickStatements. Exporting your schema from OpenRefine will generate a text file called `statements.txt` by default. Paste the contents of the text file into a new QuickStatements batch using version 1. You can find [version 1 of the tool (no longer maintained) here](https://wikidata-todo.toolforge.org/quick_statements.php). The text commands will be processed into Wikidata edits and previewed for you to review before submitting. - -### Schema alignment {#schema-alignment} - -The best resource is the [Schema alignment page](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Schema_alignment) on Wikidata. - -### Issue detection {#issue-detection} - -The best resource is the [Quality assurance page](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Quality_assurance) on Wikidata. - -OpenRefine will analyze your schema and make suggestions. It does not check for conflicts in your proposed edits, or tell you about redundancies. - -One of the most common suggestions is to attach [a reference to your edits](https://www.wikidata.org/wiki/Help:Sources) - a citation for where the information can be found. This can be a book or newspaper citation, a URL to an online page, a reference to a physical source in an archival or special collection, or another source. If the source is itself an item on Wikidata, use the relationship [stated in (P248)](https://www.wikidata.org/wiki/Property:P248); otherwise, use [reference URL (P854)](https://www.wikidata.org/wiki/Property:P854) to identify an external source. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/architecture.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/architecture.md deleted file mode 100644 index e80d7015a..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/architecture.md +++ /dev/null @@ -1,283 +0,0 @@ ---- -id: architecture -title: Architecture -sidebar_label: Architecture ---- - -OpenRefine is a web application, but is designed to be run locally on your own machine. The server-side maintains states of the data (undo/redo history, long-running processes, etc.) while the client-side maintains states of the user interface (facets and their selections, view pagination, etc.). The client-side makes GET and POST ajax calls to cause changes to the data and to fetch data and data-related states from the server-side. - -This architecture provides a good separation of concerns (data vs. UI); allows the use of familiar web technologies (HTML, CSS, Javascript) to implement user interface features; and enables the server side to be called by third-party software through standard GET and POST operations. - -## Technology stack {#technology-stack} - -The server-side part of OpenRefine is implemented in Java as one single servlet which is executed by the [Jetty](http://jetty.codehaus.org/jetty/) web server + servlet container. The use of Java strikes a balance between performance and portability across operating systems (there is very little OS-specific code and has mostly to do with starting the application). - -OpenRefine has no database. It uses its own in-memory data-store that is built up-front to be optimized for the operations required by faceted browsing and infinite undo. - -The client-side part of OpenRefine is implemented in HTML, CSS and Javascript and uses the following libraries: -* [jQuery](http://jquery.com/) -* [jQueryUI](http:jqueryui.com/) -* [Recurser jquery-i18n](https://github.com/recurser/jquery-i18n) - -The functional extensibility of OpenRefine is provided by a fork of the [SIMILE Butterfly](https://github.com/OpenRefine/simile-butterfly) modular web application framework. - -Several projects provide the functionality to read and write custom format files (POI, opencsv, JENA, marc4j). - -String clustering is provided by the [SIMILE Vicino](http://code.google.com/p/simile-vicino/) project. - -OAuth functionality is provided by the [Signpost](https://github.com/mttkay/signpost) project. - -## Server-side architecture {#server-side-architecture} - -OpenRefine's server-side is written entirely in Java (`main/src/`) and its entry point is the Java servlet `com.google.refine.RefineServlet`. By default, the servlet is hosted in the lightweight Jetty web server instantiated by `server/src/com.google.refine.Refine`. Note that the server class itself is under `server/src/`, not `main/src/`; this separation leaves the possibility of hosting `RefineServlet` in a different servlet container. - -The web server configuration is in `main/webapp/WEB-INF/web.xml`; that's where `RefineServlet` is hooked up. `RefineServlet` itself is simple: it just reacts to requests from the client-side by routing them to the right `Command` class in the packages `com.google.refine.commands.**`. - -As mentioned before, the server-side maintains states of the data, and the primary class involved is `com.google.refine.ProjectManager`. - -### Projects {#projects} - -In OpenRefine there's the concept of a workspace similar to that in Eclipse. When you run OpenRefine it manages projects within a single workspace, and the workspace is embodied in a file directory with sub-directories. The default workspace directories are listed in the [FAQs](https://github.com/OpenRefine/OpenRefine/wiki/FAQ-Where-Is-Data-Stored). You can get OpenRefine to use a different directory by specifying a -d parameter at the command line. - -The class `ProjectManager` is what manages the workspace. It keeps in memory the metadata of every project (in the class `ProjectMetadata`). This metadata includes the project's name and last modified date, and any other information necessary to present and let the user interact with the project as a whole. Only when the user decides to look at the project's data would `ProjectManager` load the project's actual data. The separation of project metadata and data is to minimize the amount of stuff loaded into memory. - -A project's _actual_ data includes the columns, rows, cells, reconciliation records, and history entries. - -A project is loaded into memory when it needs to be displayed or modified, and it remains in memory until 1 hour after the last time it gets modified. Periodically the project manager tries to save modified projects, and it saves as many modified projects as possible within 30 seconds. - -### Data Model {#data-model} - -A project's data consists of - -- _raw data_: a list of rows, each row consisting of a list of cells -- _models_ on top of that raw data that give high-level presentation or interpretation of that data. This design lets the same raw data be viewed in different ways by different models, and let the models be changed without costly changes to the raw data. - -#### Column Model {#column-model} - -Cells in rows are not named and can only be addressed by their list position indices. So, a _column model_ is needed to give a name to each list position. The column model also stores other metadata for each column, including the type that cells in the column have been reconciled to and the overall reconciliation statistics of those cells. - -Each column also acts as a cache for data computed from the raw data related to that column. - -Columns in the column model can be removed and re-ordered without changing the raw data--the cells in the rows. This makes column removal and ordering operations really quick. - -##### Column Groups {#column-groups} - -Consider the following data: - -![Illustration of row groups in OpenRefine](https://raw.github.com/OpenRefine/OpenRefine/2.0/graphics/row-groups.png) - -Although the data is in a grid, we humans can understand that it is a tree. First of all, all rows contain data ultimately linked to the movie Austin Powers, although only one row contains the text "Austin Powers" in the "movie title" column. We also know that "USA" and "Germany" are not related to Elizabeth Hurley and Mike Myers respectively (say, as their nationality), but rather, "USA" and "Germany" are related to the movie (where it was released). We know that Mike Myers played both the character "Austin Powers" and the character "Dr. Evil"; and for the latter he received 2 awards. We humans can understand how to interpret the grid as a tree based on its visual layout as well as some knowledge we have about the movie domain but is not encoded in the table. - -OpenRefine can capture our knowledge of this transformation from grid to tree using _column groups_, also stored in the column model. Each column group illustrated as a blue bracket above specifies which columns are grouped together, as well as which of those columns is the key column in that group (blue triangle). One column group can span over columns grouped by another column group, and in this way, column groups form a hierarchy determined by which column group envelopes another. This hierarchy of column groups allows the 2-dimensional (grid-shaped) table of rows and cells to be interpreted as a list of hierarchical (tree-shaped) data records. - -Blank cells play a very important role. The blank cell in a key column of a row (e.g., cell "character" on row 4) makes that row (row 4) _depend_ on the first preceding row with that column filled in (row 3). This means that "Best Comedy Perf" on row 4 applies to "Dr. Evil" on row 3. Row 3 is said to be a _context row_ for row 4. Similarly, since rows 2 - 6 all have blank cells in the first column, they all depend on row 1, and all their data ultimately applies to the movie Austin Powers. Row 1 depends on no other row and is said to be a _record row_. Rows 1 - 6 together form one _record_. - -Currently (as of 12th December 2017) only the XML and JSON importers create column groups, and while the data table view does display column groups but it doesn't support modifying them. - -### Changes, History, Processes, and Operations {#changes-history-processes-and-operations} - -All changes to the project's data are tracked (N.B. this does not include changes to a project's metadata - such as the project name.) - -Changes are stored as `com.google.refine.history.Change` objects. `com.google.refine.history.Change` is an interface, and implementing classes are in `com.google.refine.model.changes.**`. Each change object stores enough data to modify the project's data when its `apply()` method is called, and enough data to revert its effect when its `revert()` method is called. It's only supposed to _store_ data, not to actually _compute_ the change. In this way, it's like a .diff patch file for a code base. - -Some change objects can be huge, as huge as the project itself. So change objects are not kept in memory except when they are to be applied or reverted. However, since we still need to show the user some information about changes (as displayed in the History panel in the UI), we keep metadata of changes separate from the change objects. For each change object there is one corresponding `com.google.refine.history.HistoryEntry` for storing its metadata, such as the change's human-friendly description and timestamp. - -Each project has a `com.google.refine.history.History` object that contains an ordered list of all `HistoryEntry` objects storing metadata for all changes that have been done since after the project was created. Actually, there are 2 ordered lists: one for done changes that can be reverted (undone), an done for undone changes that can be re-applied (redone). Changes must be done or redone in their exact orders in these lists because each change makes certain assumptions about the state of the project before and after it is applied. As changes cannot be undone/redone out of order, when one change fails to revert, it blocks the whole history from being reverted to any state preceding that change (as happened in [Issue #2](https://github.com/OpenRefine/OpenRefine/issues/2)). - -As mentioned before, a change contains only the diff and does not actually compute that diff. The computation is performed by a `com.google.refine.process.Process` object--every change object is created by a process object. A process can be immediate, producing its change object synchronously within a very short period of time (e.g., starring one row); or a process can be long-running, producing its change object after a long time and a lot of computation, including network calls (e.g., reconciling a column). - -As the user interacts with the UI on the client-side, their interactions trigger ajax calls to the server-side. Some calls are meant to modify the project. Those are handled by commands that instantiates processes. Processes are queued in a first-in-first-out basis. The first-in process gets run and until it is done all the other processes are stuck in the queue. - -A process can effect a change in one thing in the project (e.g., edit one particular cell, star one particular row), or a process can effect changes in _potentially_ many things in the project (e.g., edit zero or more cells sharing the same content, starring all rows filtered by some facets). The latter kind of process is generalizable: it is meaningful to apply them on another similar project. Such a process is associated with an _abstract operation_ `com.google.refine.model.AbstractOperation` that encodes the information necessary to create another instance of that process, but potentially for a different project. When you click "extract" in the History panel, these abstract operations are called to serialize their information to JSON; and when you click "apply" in the History panel, the JSON you paste in is used to re-construct these abstract operations, which in turn create processes, which get run sequentially in a queue to generate change object and history entry pairs. - -In summary, - -- change objects store diffs -- history entries store metadata of change objects -- processes compute diffs and create change object and history entry pairs -- some processes are long-running and some are immediate; processes are run sequentially in a queue -- generalizable processes can be re-constructed from abstract operations - -## Client-side architecture {#client-side-architecture} -The client-side part of OpenRefine is implemented in HTML, CSS and Javascript and uses the following Javascript libraries: -* [jQuery](http://jquery.com/) -* [jQueryUI](http:jqueryui.com/) -* [Recurser jquery-i18n](https://github.com/recurser/jquery-i18n) - -### Importing architecture {#importing-architecture} - -OpenRefine has a sophisticated architecture for accommodating a diverse and extensible set of importable file formats and work flows. The formats range from simple CSV, TSV to fixed-width fields to line-based records to hierarchical XML and JSON. The work flows allow the user to preview and tweak many different import settings before creating the project. In some cases, such as XML and JSON, the user also has to select which elements in the data file to import. Additionally, a data file can also be an archive file (e.g., .zip) that contains many files inside; the user can select which of those files to import. Finally, extensions to OpenRefine can inject functionalities into any part of this architecture. - -### The Index Page and Action Areas {#the-index-page-and-action-areas} - -The opening screen of OpenRefine is implemented by the file refine/main/webapp/modules/core/index.vt and will be referred to here as the index page. Its default implementation contains 3 finger tabs labeled Create Project, Open Project, and Import Project. Each tab selects an "action area". The 3 default action areas are for, obviously, creating a new project, opening an existing project, and importing a project .tar file. - -Extensions can add more action areas in Javascript. For example, this is how the Create Project action area is added (refine/main/webapp/modules/core/scripts/index/create-project-ui.js): - -```javascript -Refine.actionAreas.push({ - id: "create-project", - label: "Create Project", - uiClass: Refine.CreateProjectUI -}); -``` - -The UI class is a constructor function that takes one argument, a jQuery-wrapped HTML element where the tab body of the action area should be rendered. - -If your extension requires a very unique importing work flow, or a very novel feature that should be exposed on the index page, then add a new action area. Otherwise, try to use the existing work flows as much as possible. - -### The Create Project Action Area {#the-create-project-action-area} - -The Create Project action area is itself extensible. Initially, it embeds a set of finger tabs corresponding to a variety of "source selection UIs": you can select a source of data by specifying a file on your computer, or you can specify the URL to a publicly accessible data file or data feed, or you can paste in from the clipboard a chunk of data. - -There are actually 3 points of extension in the Create Project action area, and the first is invisible. - -#### Importing Controllers {#importing-controllers} - -The Create Project action area manages a list of "importing controllers". Each controller follows a particular work flow (in UI terms, think "wizard"). Refine comes with a "default importing controller" (refine/main/webapp/modules/core/scripts/index/default-importing-controller/controller.js) and its work flow assumes that the data can be retrieved and cached in whole before getting processed in order to generate a preview for the user to inspect. (If the data cannot be retrieved and cached in whole before previewing, then another importing controller is needed.) - -An importing controller is just programming logic, but it can manifest itself visually by registering one or more data source UIs and one or more custom panels in the Create Project action area. The default importing controller registers 3 such custom panels, which act like pages of a wizard. - -An extension can register any number of importing controller. Each controller has a client-side part and a server-side part. Its client-side part is just a constructor function that takes an object representing the Create Project action area (usually named `createProjectUI`). The controller (client-side) is expected to use that object to register data source UIs and/or create custom panels. The controller is not expected to have any particular interface method. The default importing controller's client-side code looks like this (refine/main/webapp/modules/core/scripts/index/default-importing-controller/controller.js): - -```javascript -Refine.DefaultImportingController = function(createProjectUI) { - this._createProjectUI = createProjectUI; // save a reference to the create project action area - - this._progressPanel = createProjectUI.addCustomPanel(); // create a custom panel - this._progressPanel.html('...'); // render the custom panel - ... do other stuff ... -}; -Refine.CreateProjectUI.controllers.push(Refine.DefaultImportingController); // register the controller -``` - -We will cover the server-side code below. - -#### Data Source Selection UIs {#data-source-selection-uis} - -Data source selection UIs are another point of extensibility in the Create Project action area. As mentioned previously, by default there are 3 data source UIs. Those are added by the default importing controller. - -Extensions can also add their own data source UIs. A data source selection UI object can be registered like so - -```javascript -createProjectUI.addSourceSelectionUI({ - label: "This Computer", - id: "local-computer-source", - ui: theDataSourceSelectionUIObject -}); -``` - -`theDataSourceSelectionUIObject` is an object that has the following member methods: - -- `attachUI(bodyDiv)` -- `focus()` - -If you want to install a data source selection UI that is managed by the default importing controller, then register its UI class with the default importing controller, like so (refine/main/webapp/modules/core/scripts/index/default-importing-sources/sources.js): - -```javascript -Refine.DefaultImportingController.sources.push({ - "label": "This Computer", - "id": "upload", - "uiClass": ThisComputerImportingSourceUI -}); -``` - -The default importing controller will assume that the `uiClass` field is a constructor function and call it with one argument--the controller object itself. That constructor function should save the controller object for later use. More specifically, for data source UIs that use the default importing controller, they can call the controller to kickstart the process that retrieves and caches the data to import: - -```javascript -controller.startImportJob(form, "... status message ..."); -``` - -The argument `form` is a jQuery-wrapped FORM element that will get submitted to the server side at the command /command/core/create-importing-job. That command and the default importing controller will take care of uploading or downloading the data, caching it, updating the client side's progress display, and then showing the next importing step when the data is fully cached. - -See refine/main/webapp/modules/core/scripts/index/default-importing-sources/sources.js for examples of such source selection UIs. While we write about source selection UIs managed by the default importing controller here, chances are your own extension will not be adding such a new source selection UI. Your extension probably adds with a new importing controller as well as a new source selection UI that work together. - -#### File Selection Panel {#file-selection-panel} -Documentation not currently available - -#### Parsing UI Panel {#parsing-ui-panel} -Documentation not currently available - -### Server-side Components {#server-side-components} - -#### ImportingController {#importingcontroller} -Documentation not currently available - -#### UrlRewriter {#urlrewriter} -Documentation not currently available - -#### FormatGuesser {#formatguesser} -Documentation not currently available - -#### ImportingParser {#importingparser} -Documentation not currently available - - -## Faceted browsing architecture {#faceted-browsing-architecture} - -Faceted browsing support is core to OpenRefine as it is the primary and only mechanism for filtering to a subset of rows on which to do something _en masse_ (ie in bulk). Without faceted browsing or an equivalent querying/browsing mechanism, you can only change one thing at a time (one cell or row) or else change everything all at once; both kinds of editing are practically useless when dealing with large data sets. - -In OpenRefine, different components of the code need to know which rows to process from the faceted browsing state (how the facets are constrained). For example, when the user applies some facet selections and then exports the data, the exporter serializes only the matching rows, not all rows in the project. Thus, faceted browsing isn't only hooked up to the data view for displaying data to the user, but it is also hooked up to almost all other parts of the system. - -### Engine Configuration {#engine-configuration} - -As OpenRefine is a web app, there might be several browser windows opened on the same project, each in a different faceted browsing state. It is best to maintain the faceted browsing state in each browser window while keeping the server side completely stateless with regard to faceted browsing. Whenever the client-side needs something done by the server, it transfers the entire faceted browsing state over to the server-side. The faceted browsing state behaves much like the `WHERE` clause in a SQL query, telling the server-side how to select the rows to process. - -In fact, it is best to think of the faceted browsing state as just a database query much like a SQL query. It can be passed around the whole system, to any component needing to know which rows to process. It is serialized into JSON to pass between the client-side and the server side, or to save in an abstract operation's specification. The job of the faceted browsing subsystem on the client-side is to let the user interactively modify this "faceted browsing query", and the job of the faceted browsing subsystem on the server side is to resolve that query. - -In the code, the faceted browsing state, or faceted browsing query, is actually called the *engine configuration* or *engine config* for short. It consists mostly of an array facet configurations. For each facet, it stores the name of the column on which the facet is based (or an empty string if there is no base column). Each type of facet has different configuration. Text search facets have queries and flags for case-sensitivity mode and regular expression mode. Text facets (aka list facets) and numeric range facets have expressions. Each list facet also has an array of selected choices, an invert flag, and flags for whether blank and error cells are selected. Each numeric range facet has, among other things, a "from" and a "to" values. If you trace the AJAX calls, you'd see the engine configs being shuttled, e.g., - -```json -{ - "facets" : [ - { - "type": "text", - "name": "Shrt_Desc", - "columnName": "Shrt_Desc", - "mode": "text", - "caseSensitive": false, - "query": "cheese" - }, - { - "type": "list", - "name": "Shrt_Desc", - "columnName": "Shrt_Desc", - "expression": "grel:value.toLowercase().split(\",\")", - "omitBlank": false, - "omitError": false, - "selection": [], - "selectBlank":false, - "selectError":false, - "invert":false - }, - { - "type": "range", - "name": "Water", - "expression": "value", - "columnName": "Water", - "selectNumeric": true, - "selectNonNumeric": true, - "selectBlank": true, - "selectError": true, - "from": 0, - "to": 53 - } - ], - "includeDependent": false - } -``` - -### Server-Side Subsystem {#server-side-subsystem} - -From an engine configuration like the one above, the server-side faceted browsing subsystem is capable of producing: - -- an iteration over the rows matching the facets' constraints -- information on how to render the facets (e.g., choice and count pairs for a list facet, histogram for a numeric range facet) - -When the engine config JSON arrives in an HTTP request on the server-side, a `com.google.refine.browsing.Engine` object is constructed and initialized with that JSON. It in turns constructs zero or more `com.google.refine.browsing.facets.Facet` objects. Then for each facet, the engine calls its `getRowFilter()` method, which returns `null` if the facet isn't constrained in anyway, or a `com.google.refine.browsing.filters.RowFilter` object. Then, to when iterating over a project's rows, the engine calls on all row filters' `filterRow()` method. If and only if all row filters return `true` the row is considered to match the facets' constraints. How each row filter works depends on the corresponding type of facet. - -To produce information on how to render a particular facet in the UI, the engine follows the same procedure described in the previous except it skips over the facet in question. In other words, it produces an iteration over all rows constrained by the other facets. Then it feeds that iteration to the facet in question by calling the facet's `computeChoices()` method. This gives the method a chance to compute the rendering information for its UI counterpart on the client-side. When all facets have been given a chance to compute their rendering information, the engine calls all facets to serialize their information as JSON and returns the JSON to the client-side. Only one HTTP call is needed to compute all facets. - -### Client-side subsystem {#client-side-subsystem} - -On the client-side there is also an engine object (implemented in Javascript rather than Java) and zero or more facet objects (also in Javascript, obviously). The engine is responsible for distributing the rendering information computed on the server-side to the right facets, and when the user interacts with a facet, the facet tells the engine to update the whole UI. To do so, the engine gathers the configuration of each facet and composes the whole engine config as a single JSON object. Two separate AJAX calls are made with that engine config, one to retrieve the rows to render, and one to re-compute the rendering information for the facets because changing one facet does affect all the other facets. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/build-test-run.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/build-test-run.md deleted file mode 100644 index 085070e64..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/build-test-run.md +++ /dev/null @@ -1,270 +0,0 @@ ---- -id: build-test-run -title: How to build, test and run -sidebar_label: How to build, test and run ---- - -import useBaseUrl from '@docusaurus/useBaseUrl'; - - -You will need: -* [OpenRefine source code](https://github.com/OpenRefine/OpenRefine) -* [Java JDK](http://java.sun.com/javase/downloads/index.jsp) (Get [OpenJDK from here](https://jdk.java.net/archive/). At the moment you cannot use JDK > 15, see [Issue on Github](https://github.com/OpenRefine/OpenRefine/issues/4106).) -* [Apache Maven](https://maven.apache.org) (OPTIONAL) -* A Unix/Linux shell environment OR the Windows command line - -From the top level directory in the OpenRefine application you can build, test and run OpenRefine using the `./refine` shell script (if you are working in a \*nix shell), or using the `refine.bat` script from the Windows command line. Note that the `refine.bat` on Windows only supports a subset of the functionality, supported by the `refine` shell script. The example commands below are using the `./refine` shell script, and you will need to use `refine.bat` if you are working from the Windows command line. - -### Set up JDK {#set-up-jdk} - -You must [install JDK](https://jdk.java.net/15/) and set the JAVA_HOME environment variable (please ensure it points to the JDK, and not the JRE). - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - - -1. On Windows 10, click the Start Menu button, type `env`, and look at the search results. Click Edit the system environment variables. (If you are using an earlier version of Windows, use the “Search” or “Search programs and files” box in the Start Menu.) - -![A screenshot of the search results for 'env'.](/img/env.png "A screenshot of the search results for 'env'.") - -2. Click Environment Variables… at the bottom of the Advanced window. -3. In the Environment Variables window that appears, click New… and create a variable with the key `JAVA_HOME`. You can set the variable for only your user account, as in the screenshot below, or set it as a system variable - it will work either way. - -![A screenshot of 'Environment Variables'.](/img/javahome.png "A screenshot of 'Environment Variables'.") - -4. Set the `Value` to the folder where you installed JDK, in the format `D:\Programs\OpenJDK`. You can locate this folder with the Browse directory... button. - - - - - -First, find where Java is on your computer with this command: - -``` -which java -``` - -Check the environment variable `JAVA_HOME` with: - -``` -$JAVA_HOME/bin/java --version -``` - -To set the environment variable for the current Java version of your MacOS: - -``` -export JAVA_HOME="$(/usr/libexec/java_home)" -``` - -Or, for Java 13.x: - -``` -export JAVA_HOME="$(/usr/libexec/java_home -v 13)" -``` - - - - - -##### With the terminal {#with-the-terminal} - -Enter the following: - -``` -sudo apt install default-jre -``` - -This probably won’t install the latest JDK package available on the Java website, but it is faster and more straightforward. (At the time of writing, it installs OpenJDK 11.0.7.) - -##### Manually {#manually} - -First, [extract the JDK package](https://openjdk.java.net/install/) to the new directory `usr/lib/jvm`: - -``` -sudo mkdir -p /usr/lib/jvm -sudo tar -x -C /usr/lib/jvm -f /tmp/openjdk-14.0.1_linux-x64_bin.tar.gz -``` - -Then, navigate to this folder and confirm the final path (in this case, `usr/lib/jvm/jdk-14.0.1`). Open a terminal and type - -``` -sudo gedit /etc/profile -``` - -In the text window that opens, insert the following lines at the end of the `profile` file, using the path above: - -``` -JAVA_HOME=/usr/lib/jvm/jdk-14.0.1 -PATH=$PATH:$HOME/bin:$JAVA_HOME/bin -export JAVA_HOME -export PATH -``` - -Save and close the file. When you are back in the terminal, type - -``` -source /etc/environment -``` - -Exit the terminal and restart your system. You can then check that `JAVA_HOME` is set properly by opening another terminal and typing -``` -echo $JAVA_HOME -``` - -It should show the path you set above. - - - - - ---- - - - -### Maven (Optional) {#maven-optional} -OpenRefine's build script will download Maven for you and use it, if not found already locally installed. - -If you will be using your Maven installation instead of OpenRefine's build script download installation, then set the `MVN_HOME` environment variable. You may need to reboot your machine after setting these environment variables. If you receive a message `Could not find the main class: com.google.refine.Refine. Program will exit.` it is likely `JAVA_HOME` is not set correctly. - -Ensure that you set your `MAVEN_HOME` environment variable, for example: - -```shell -MAVEN_HOME=E:\Downloads\apache-maven-3.5.4-bin\apache-maven-3.5.4\ -``` - -NOTE: You can use Maven commands directly, but running some goals in isolation might fail (try adding the `compile test-compile` goals in your invocation if that is the case). - -### Building {#building} - -To see what functions are supported by OpenRefine's build system, type -```shell -./refine -h -``` - -To build the OpenRefine application from source type: -```shell -./refine clean -./refine build -``` - -### Testing {#testing} -Since OpenRefine is composed of two parts, a server and a in-browser UI, the testing system reflects that: - -* on the server side, it's powered by [TestNG](http://testng.org/) and the unit tests are written in Java; -* on the client side, we use [Cypress](https://www.cypress.io/) and the tests are written in Javascript - -To run all tests, use: -```shell -./refine test -``` -**this option is not available when using refine.bat** - - -If you want to run only the server side portion of the tests, use: -```shell -./refine server_test -``` - -If you are running the UI tests for the first time, [you must go through the installation process.](functional-tests) -If you want to run only the client side portion of the tests, use: -```shell -./refine ui_test chrome -``` - -## Running {#running} -To run OpenRefine from the command line (assuming you have been able to build from the source code successfully) -```shell -./refine -``` -By default, OpenRefine will use [refine.ini](https://github.com/OpenRefine/OpenRefine/blob/master/refine.ini) for configuration. You can copy it and rename it to `refine-dev.ini`, which will be used for configuration instead. `refine-dev.ini` won't be tracked by Git, so feel free to put your custom configurations into it. - -## Building Distributions (Kits) {#building-distributions-kits} - -The Refine build system uses Apache Ant to automate the creation of the installation packages for the different operating systems. The packages are currently optimized to run on Mac OS X which is the only platform capable of creating the packages for all three OS that we support. - -To build the distributions type - -```shell -./refine dist -``` -where 'version' is the release version. - -## Building, Testing and Running OpenRefine from Eclipse {#building-testing-and-running-openrefine-from-eclipse} -OpenRefine' source comes with Maven configuration files which are recognized by [Eclipse](http://www.eclipse.org/) if the Eclipse Maven plugin (m2e) is installed. - -At the command line, go to a directory **not** under your Eclipse workspace directory and check out the source: - -```shell -git clone https://github.com/OpenRefine/OpenRefine.git -``` - -In Eclipse, invoke the `Import...` command and select `Existing Maven Projects`. - -![Screenshot of Import a Maven project option](/img/eclipse-import-maven-project-1.png) - -Choose the root directory of your clone of the repository. You get to choose which modules of the project will be imported. You can safely leave out the `packaging` module which is only used to generate the Linux, Windows and MacOS distributions. - -Screenshot of Select maven projects to import - -To run and debug OpenRefine from Eclipse, you will need to add an execution configuration on the `server` sub-project. -Right click on the `server` subproject, click `Run as...` and `Run configurations...` and create a new `Maven Build` run configuration. Rename the run configuration `OpenRefine`. Enter the root directory of the project as `Base directory` and use `exec:java` as a Maven goal. - -![Screenshot of Add a run configuration with the exec:java goal](/img/eclipse-exec-config.png) - -This will add a run configuration that you can then use to run OpenRefine from Eclipse. - -## Testing in Eclipse {#testing-in-eclipse} - -You can run the server tests directly from Eclipse. To do that you need to have the TestNG launcher plugin installed, as well as the TestNG M2E plugin (for integration with Maven). If you don't have it, you can get it by [installing new software](https://help.eclipse.org/2020-03/index.jsp?topic=/org.eclipse.platform.doc.user/tasks/tasks-129.htm) from this update URL http://dl.bintray.com/testng-team/testng-eclipse-release/ - -Once the TestNG launching plugin is installed in your Eclipse, right click on the source folder "main/tests/server/src", select `Run As` -> `TestNG Test`. This should open a new tab with the TestNG launcher running the OpenRefine tests. - -### Test coverage in Eclipse {#test-coverage-in-eclipse} - -It is possible to analyze test coverage in Eclipse with the `EclEmma Java Code Coverage` plugin. It will add a `Coverage as…` menu similar to the `Run as…` and `Debug as…` menus which will then display the covered and missed lines in the source editor. - -### Debug with Eclipse {#debug-with-eclipse} -Here's an example of putting configuration in Eclipse for debugging, like putting values for the Google Data extension. Other type of configurations that can be set are memory, Wikidata login information and more. - -![Screenshot of Eclipse debug configuration](/img/eclipse-debug-config.png) - -## Building, Testing and Running OpenRefine from IntelliJ idea {#building-testing-and-running-openrefine-from-intellij-idea} - -At the command line, go to a directory you want to save the OpenRefine project and execute the following command to clone the repository: - -```shell -git clone https://github.com/OpenRefine/OpenRefine.git -``` - -Then, open the IntelliJ idea and go to `file -> open` and select the location of the cloned repository. - -![Screenshot of Open option on the IntelliJ File menu](/img/intellij-setup-1.png) - -It will prompt you to add as a maven project as the source code contains a pom.xml file in it. Allow `auto-import` so that it can add it as a maven project. -If it doesn't prompt something like this then you can go on the right side of the IDE and click on maven then, click on `reimport all the maven projects` that will add all the dependencies and jar files required for the project. - -![Screenshot of Maven project controls in IntelliJ](/img/intellij-maven.png) - -After this, you will be able to properly build, test, and run the OpenRefine project from the terminal. -But if you will go to any of the test folders and open some file it will show you some import errors because the project isn't yet set up at the module level. - -For removing those errors, and enjoying the features of the IDE like ctrl + click, etc you need to set up the project at the module level too. Open the different modules like `extensions/wikidata`, `main` as a project in the IDE. Then, right-click on the project folder and open the module settings. - -![Screenshot of open module settings menu in IntelliJ](/img/intellij-open-module-settings.png) - -In the module settings, add the source folder and test source folders of that module. - -![Screenshot of module settings in IntelliJ](/img/intellij-module-settings.png) - -Then, do the same thing for the main OpenRefine project and now you are good to go. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/contributing.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/contributing.md deleted file mode 100644 index 241f5698c..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/contributing.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -id: contributing -title: Contributing -sidebar_label: Contributing ---- - -Please read the general [guidelines on contributing to OpenRefine](https://github.com/OpenRefine/OpenRefine/blob/master/CONTRIBUTING.md) first, then review the information on [reporting and tracking issues](#reporting-and-tracking-issues), and on making your [first pull request](#your-first-pull-request) below) - -## Reporting and tracking issues {#reporting-and-tracking-issues} - -If you need to file a bug or request a feature, [create an Issue in the OpenRefine Github repository](https://github.com/OpenRefine/OpenRefine/issues). Github issues should be used for reporting specific bugs and requesting specific features. If you just don't know how to do something using OpenRefine, or want to discuss some ideas, please: - -- [Try the user manual](/) -- [post to our OpenRefine mailing list](http://groups.google.com/group/openrefine/) - -## Contributing to the documentation {#contributing-to-the-documentation} - -We use [Docusaurus](https://docusaurus.io/) for our docs. For small documentation changes, you should be able to edit the Markdown files directly and submit them as a pull request. A preview of the docs will be generated automatically. But it is also -possible to preview your changes locally. Assuming you have [Node.js](https://nodejs.org/en/download/) installed (which includes npm), you can install Docusaurus with: - -You will need to install [Yarn](https://yarnpkg.com/getting-started/install) before you can build the site. -```sh -npm install -g yarn -``` - -Once you have installed yarn, navigate to docs directory & set-up the dependencies. - -```sh -cd docs -yarn -``` - -Once this is done, generate the docs with: - -```sh -yarn build -``` - -You can also spin a local web server to serve the docs for you, with auto-refresh when you edit the source files, with: -```sh -yarn start -``` - -## Your first code pull request {#your-first-code-pull-request} - -This describes the overall steps to your first code contribution in OpenRefine. If you have trouble with any of these steps feel free to reach out on the [developer mailing list](https://groups.google.com/forum/#!forum/openrefine-dev) or the [Gitter channel](https://gitter.im/OpenRefine/OpenRefine). - -- Install OpenRefine, learn to use it by following some tutorials or watching [some videos](http://openrefine.org/). That will ensure you understand the user workflows and get familiar with the terminology used in the tool. - -- Fork the GitHub repository, clone it on your machine and set up your IDE to work on it. We have [instructions for this](https://github.com/OpenRefine/OpenRefine/wiki/Building-OpenRefine-From-Source). - -- Browse through the list of issues to find an issue that you find interesting. You should pick one where you understand what the problem is as a user, you can see why fixing it would be an improvement to the tool. It is also a good idea to pick an issue that matches your technical skills: some require work on the backend (in Java) or in the frontend (Javascript), often both. We try to maintain a list of [good first issues](https://github.com/OpenRefine/OpenRefine/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) which should be easier than others and should not require any difficult design decision. - -- Reproduce the issue locally, by following the steps described in the issue. You might need to locate a particular dialog, use a specific importer on a sample file, or follow any other user workflow. If you have followed all the steps described in the issue and cannot observe the issue mentioned, write a comment on the issue explaining that you are not able to reproduce it (perhaps it was fixed by another change). - -- Locate the code that is relevant for the issue you want to solve. Text search across files is often useful for that. For instance, if the issue you want to solve is about a dialog entitled "Columnize by key/values", you can search for "Columnize" in the entire source code. - -- Study how the current code works. You might want to use a debugger to put breakpoints at the relevant locations (for inspecting the backend, use your IDE's debugger, for the frontend, use your browser's developer tools). - -- Create a git branch for your fix. The name of your branch should contain the issue number, and a few words to describe the topic of the fix, for instance "issue-1234-columnize-layout". - -- Make changes to the code to fix the issue. If you are changing backend code, it would be great if you could also write a test in Java to demonstrate the fix. You can imitate existing tests for that. We currently do not have frontend tests. - -- commit your changes, using a message that contains "closes #1234" or "fixes #1234", this will link the commit to the issue you are working on. - -- push your branch to your fork and create a pull request for it, explaining the approach you have used, any design decisions you have made. - - -Thank you! diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/data-extension-api.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/data-extension-api.md deleted file mode 100644 index 683fbced9..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/data-extension-api.md +++ /dev/null @@ -1,261 +0,0 @@ ---- -id: data-extension-api -title: Data extension API -sidebar_label: Data extension API ---- - -This page describes a new optional API for reconciliation services, allowing clients to pull properties of reconciled records. It is supported from OpenRefine 2.8 onwards. A sample server implementation is available in the [Wikidata reconciliation interface](https://tools.wmflabs.org/openrefine-wikidata/). - -## Overview of the workflow {#overview-of-the-workflow} - -1. Reconcile a column with a standard reconciliation service - -2. Click "Add column from reconciled values" - -3. The user is proposed some properties to fetch, based on the type they reconciled their column against (if any). They can also pick their own property with the suggest widget (same as for the reconciliation dialog). - -4. A preview of the columns to be fetched is displayed on the right-hand side of the dialog, based on a sample of the rows. - -5. Once the user has clicked "OK", columns are fetched and added to the project. Columns corresponding to other items from the service are directly reconciled, and the column is marked as reconciled against the type suggested by the service for that -property. The user can run data extension again from that column. - -[GIF Screencast](http://pintoch.ulminfo.fr/92dcdd20f3/recorded.gif) - -## Specification {#specification} - -Services supporting data extension must add an `extend` field in their service metadata. This field is expected to have the following subfields, all optional: -* `propose_properties` stores the endpoint of an API which will be used to suggest properties to fetch (see specification below). The field contains an object with a `service_url` and `service_path` which will be concatenated to obtain the URL where the endpoint is available, just like the other services in the metadata. If this field is not provided, no property will be suggested in the dialog (the user will have to input them manually). -* `property_settings` stores the specification of a form where the user will be able to configure how a given property should be fetched (see specification below). If this field is not provided, the user will not be proposed with settings. - -The service endpoint must also accept a new parameter `extend` (in addition to `queries` which is used for reconciliation). Its behaviour is described in the following section. - -Example service metadata: -```json - "extend": { - "propose_properties": { - "service_url": "https://tools.wmflabs.org/openrefine-wikidata", - "service_path": "/en/propose_properties" - }, - "property_settings": [] - } -``` -### Property proposal protocol {#property-proposal-protocol} - -The role of the property proposal endpoint is to suggest a list of properties to fetch. As only input, it accepts GET parameters: -* the `type` of a column was reconciled against. If no type is provided, it should suggest properties for a column reconciled against no type. -* a `limit` on the number of results to return - -The type is specified by its id in the `type` GET parameter of the endpoint, as follows: - -https://tools.wmflabs.org/openrefine-wikidata/en/propose_properties?type=Q3354859&limit=3 - -The endpoint returns a JSON response as follows: - -```json - { - "properties": [ - { - "id": "P969", - "name": "located at street address" - }, - { - "id": "P1449", - "name": "nickname" - }, - { - "id": "P17", - "name": "country" - }, - ], - "type": "Q3354859", - "limit": 3 - } -``` -This endpoint must support JSONP via the `callback` parameter (just like all other endpoints of the reconciliation service). - -### Data extension protocol {#data-extension-protocol} - -After calling the property proposal endpoint, the consumer (OpenRefine) calls the service endpoint with a JSON object in the `extend` parameter, containing the following fields: -* `ids` is a list of strings, each of which being an identifier of a record as returned by the reconciliation method. These are the records whose properties should be retrieved. -* `properties` is a list of JSON objects. They specify the properties to be fetched for each item, and contain the following fields: - * `id` (a string): the identifier of the property as returned by the property suggest service (and optionally the property proposal service) - * `settings`: a JSON object storing parameters about how the property should be fetched (see below). - -Example: -```json - { - "ids": [ - "Q7205598", - "Q218765", - "Q845632", - "Q5661356" - ], - "properties": [ - { - "id": "P856" - }, - { - "id": "P159" - } - ] - } -``` -The service returns a JSON response formatted as follows: - -* `meta` contains a list of column metadata. The order of the properties must be the same - as the one provided in the query. Each element is an object containing the following keys: - * `id` (mandatory): the identifier of the property - * `name` (mandatory): the human-readable name of the property - * `type` (optional): an object with `id` and `name` keys representing the expected - type of values for that property. The notion of type is the same as the one - used for reconciliation. The `type` field should only be provided when the property returns - reconciled items. -* `rows` contains an object. Its keys must be exactly the record ids (`ids`) passed in the query. - The value for each record id is an object representing a row for that id. The keys of a row object must be exactly the property ids passed in the query (`"P856"` and `"P159"` in the example above). The value for a property id should be a list of cell objects. - -Cell objects are JSON objects which contain the representation of an OpenRefine cell. -* an object with a single `"str"` key and a string value for it represents - a cell with a (bare) string in it. - Example: `{"str": "193.54.0.0/15"}` - -* an object with `"id"` and `"name"` represents a reconciled value - (from the same reconciliation service). It will be stored as - a matched cell (with maximum reconciliation score). - Example: `{"name": "Warsaw","id": "Q270"}` - -* an empty object `{}` represents an empty cell - -* an object with `"date"` and an ISO-formatted date string represents a point in time. - Example: `{"date": "1987-02-01T00:00:00+00:00"}` - -* an object with `"float"` and a numerical value represents a quantity. - Example: `{"float": 48.2736}` - -* an object with `"int"` and an integer represents a number. - Example: `{"int": 54}` - -* an object with `"bool"` and `true` or `false` represents a boolean. - Example: `{"bool": false}` - -Example of a full response (for the example query above): -```json - { - "rows": { - "Q5661356": { - "P159": [], - "P856": [] - }, - "Q7205598": { - "P159": [ - { - "name": "Warsaw", - "id": "Q270" - } - ], - "P856": [ - { - "str": "http://www.polkomtel.com.pl/english" - }, - { - "str": "http://www.polkomtel.com.pl/" - } - ] - }, - "Q845632": { - "P159": [ - { - "name": "Bærum", - "id": "Q57076" - } - ], - "P856": [ - { - "str": "http://www.telenor.com/" - } - ] - }, - "Q218765": { - "P159": [ - { - "name": "Paris", - "id": "Q90" - } - ], - "P856": [ - { - "str": "http://www.sfr.fr/" - } - ] - } - }, - "meta": [ - { - "id": "P159", - "name": "headquarters location", - "type": { - "id": "Q7540126", - "name": "headquarters", - } - }, - { - "id": "P856", - "name": "official website", - } - ] - } -``` -### Settings specification {#settings-specification} - -The `property_settings` field in the service metadata allows the service to declare it accepts some settings for the properties it fetches. They are specified as a list of JSON objects which define the fields which should be exposed to the user. - -Each setting object looks like this: -```json - { - "default": 0, - "type": "number", - "label": "Limit", - "name": "limit", - "help_text": "Maximum number of values to return per row (0 for no limit)" - } -``` -It is essentially a definition of a form field in JSON, with self-explanatory fields. -The `type` field specifies the type of the form field (among `number`, `select`, `text`, `checkbox`). -The field `default` gives the default value of the form: the service must assume this value if the -client does not specify this setting. - -For the `select` field, an additional `choices` field defines the possible choices, with both labels and values: -```json - { - "default": "any", - "label": "References", - "name": "references", - "type": "select", - "choices": [ - { - "value": "any", - "name": "Any statement" - }, - { - "value": "referenced", - "name": "At least one reference" - }, - { - "value": "no_wiki", - "name": "At least one non-wiki reference" - } - ], - "help_text": "Filter statements by their references" - } -``` -When querying the service for rows, the client can pass an optional `settings` object in each of the requested columns: -```json - { - "id": "P342", - "settings": { - "limit": "20", - "references": "referenced", - } - } -``` -Each key of the settings object must correspond to one form field proposed by the service. The value of that key is the value of the form field represented as a string (for uniformity and consistency with JSON form serialization). -The settings are intended to modify the results returned by the service: of course, the semantics of the settings is up to the service (as the service defines itself what settings it accepts). diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/development-roadmap.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/development-roadmap.md deleted file mode 100644 index da35f2f77..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/development-roadmap.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -id: development-roadmap -title: Development roadmap -sidebar_label: Development roadmap ---- - -Please be aware that the OpenRefine roadmap is subject to change at any time, so please check back regularly, and monitor [milestones](https://github.com/OpenRefine/OpenRefine/milestones), [projects](https://github.com/OpenRefine/OpenRefine/projects) and [issues](https://github.com/OpenRefine/OpenRefine/issues) in Github to keep up to date with current plans. - -If there are features you would like to see that are not currently listed here or in current [milestones](https://github.com/OpenRefine/OpenRefine/milestones), [projects](https://github.com/OpenRefine/OpenRefine/projects) and [issues](https://github.com/OpenRefine/OpenRefine/issues), please add them to the [issue tracker](https://github.com/OpenRefine/OpenRefine/issues). - - -## Planned releases {#planned-releases} - -### 4.0 {#40} -[New backend storage option to allow using much bigger datasets at the expense of real-time feedback.](https://github.com/OpenRefine/OpenRefine/milestone/7) - -New UI (possibly Vue or React based) - -## Work in progress {#work-in-progress} -Alongside the planned releases there are often smaller pieces of work in progress. Check for [recently updated issues](https://github.com/OpenRefine/OpenRefine/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc) and [pull requests](https://github.com/OpenRefine/OpenRefine/pulls?q=is%3Apr+is%3Aopen+sort%3Aupdated-desc) to see what is currently in the works. - -## On the back burner {#on-the-back-burner} -Some aspects of OpenRefine have previously been targeted for release, but have not made it into a release and have not been worked on recently. If you would like to see features in these areas, please create an issue the describes what development you would like to see: - -- Streamlining traditional features -- Views: map, timeline, protovis (D3.js) charts -- Better machinery to guess and re-encode cell values (useful for fixing encoding issues) -- Collaborative editing support (see documentation on the '[broker protocol](https://github.com/OpenRefine/OpenRefine/wiki/Broker-Protocol)' to see where this work was going) -- Column groups diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/functional-tests.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/functional-tests.md deleted file mode 100644 index 8cece4b5e..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/functional-tests.md +++ /dev/null @@ -1,239 +0,0 @@ ---- -id: functional-tests -title: Functional tests -sidebar_label: Functional tests ---- - -import useBaseUrl from '@docusaurus/useBaseUrl'; - -## Introduction {#introduction} - -OpenRefine interface is tested with the [Cypress framework](https://www.cypress.io/). -With Cypress, tests are performing assertions using a real browser, the same way a real user would use the software. - -Cypress tests can be ran - -- using the Cypress test runner (development mode) -- using a command line (CI/CD mode) - -If you are writing tests, the Cypress test runner is good enough, and the command-line is mainly used by the CI/CD platform (Github actions) - -## Cypress brief overview {#cypress-brief-overview} - -Cypress operates insides a browser, it's internally using NodeJS. -That's a key difference with tools such as Selenium. - -**From the Cypress documentation:** - -> But what this also means is that your test code **is being evaluated inside the browser**. Test code is not evaluated in Node, or any other server side language. The **only** language we will ever support is the language of the web: JavaScript. - -Good starting points with Cypress are the [Getting started guide](https://docs.cypress.io/guides/getting-started/writing-your-first-test.html#Write-your-first-test), and the [Trade-offs](https://docs.cypress.io/guides/references/trade-offs.html#Permanent-trade-offs-1) - -The general workflow of a Cypress test is to - -- Start a browser (yarn run cypress open) -- Visit a URL -- Trigger user actions -- Assert that the DOM contains expected texts and elements using selectors - -## Getting started {#getting-started} - -If this is the first time you use Cypress, it is recommended for you to get familiar with the tool. - -- [Cypress overview](https://docs.cypress.io/guides/overview/why-cypress.html) -- [Cypress examples of tests and syntax](https://example.cypress.io/) - -### 1. Install Cypress {#1-install-cypress} - -You will need: - -- [Node.js 10 or 12 and above](https://nodejs.org) -- [Yarn or NPM](https://yarnpkg.com/) -- A Unix/Linux shell environment or the Windows command line - -To install Cypress and dependencies, run : - -```shell -cd ./main/tests/cypress -yarn install -``` - -### 2. Start the test runner {#2-start-the-test-runner} - -The test runner assumes that OpenRefine is up and running on the local machine, the tests themselves do not launch OpenRefine, nor restarts it. - -Start OpenRefine with - -```shell -./refine -``` - -Then start Cypress - -```shell -yarn --cwd ./main/tests/cypress run cypress open -``` - -### 3. Run the existing tests {#3-run-the-existing-tests} - -Once the test runner is up, you can choose to run one or several tests by selecting them from the interface. -Click on one of them and the test will start. - -### 4. Add your first test {#4-add-your-first-test} - -- Add a `test.spec.js` into the `main/tests/cypress/cypress/integration` folder. -- The test is instantly available in the list -- Click on the test -- Start to add some code - -## Tests technical documentation {#tests-technical-documentation} - -### A typical test {#a-typical-test} - -A typical OpenRefine test starts with the following code - -```javascript -it('Ensure cells are blanked down', function () { - cy.loadAndVisitProject('food.mini'); - cy.get('.viewpanel-sorting a').contains('Sort').click(); - cy.get('.viewpanel').should('to.contain', 'Something'); -}); -``` - -The first noticeable thing about a test is the description (`Ensure cells are blanked down`), which describes what the test is doing. -Lines usually starts with `cy.something...`, which is the main way to interact with the Cypress framework. - -A few examples: - -- `cy.get('a.my-class')` will retrieve the `` element -- `cy.click()` will click on the element -- eventually, `cy.should()` will perform an assertion, for example that the element contains an expected text with `cy.should('to.contains', 'my text')` - -On top of that, OpenRefine contributors have added some functions for common OpenRefine interactions. -For example - -- `cy.loadAndVisitProject` will create a fresh project in OpenRefine -- `cy.assertCellEquals` will ensure that a cell contains a given value - -See below on the dedicated section 'Testing utilities' - -### Testing guidelines {#testing-guidelines} - -- `cy.wait` should be used in the last resort scenario. It's considered a bad practice, though sometimes there is no other choice -- Tests should remain isolated from each other. It's best to try one feature at the time -- A test should always start with a fresh project -- The name of the files should mirror the OpenRefine UI organization - -### Testing utilities {#testing-utilities} - -OpenRefine contributors have added some utility methods on the top of the Cypress framework. -Those methods perform some common actions or assertions on OpenRefine, to avoid code duplication. - -Utilities can be found in `cypress/support/commands.js`. - -The most important utility method is `loadAndVisitProject`. -This method will create a fresh OpenRefine project based on a dataset given as a parameter. -The fixture parameter can be - -- An arbitrary array, the first row is for the column names, other rows are for the values - Use an arbitrary array **only** if the test requires some specific grid values - **Example:** - - ```javascript - const fixture = [ - ['Column A', 'Column B', 'Column C'], - ['0A', '0B', '0C'], - ['1A', '1B', '1C'], - ['2A', '2B', '2C'], - ]; - cy.loadAndVisitProject(fixture); - ``` - -- A referenced dataset: `food.small` or `food.mini` - Most of the time, tests does not require any specific grid values - Use food.mini as much as possible, it loads 2 rows and very few columns in the grid - Use food.small if the test requires a few hundred rows in the grid - - Those datasets live in `cypress/fixtures` - -### Browsers {#browsers} - -In terms of browsers, Cypress is using what is installed on your operating system. -See the [Cypress documentation](https://docs.cypress.io/guides/guides/launching-browsers.html#Browsers) for a list of supported browsers - -### Folder organization {#folder-organization} - -Tests are located in `main/tests/cypress/cypress` folder. -The test should not use any file outside the cypress folder. - -- `/fixtures` contains CSVs and OpenRefine project files used by the tests -- `/integration` contains the tests -- `/plugins` contains custom plugins for the OR project -- `/screenshots` and `/videos` contains the recording of the tests, Git ignored -- `/support` is a custom library of assertion and common user actions, to avoid code duplication in the tests themselves - -### Configuration {#configuration} - -Cypress execution can be configured with environment variables, they can be declared at the OS level, or when running the test - -Available variables are - -- OPENREFINE_URL, determine on which scheme://url:port to access OpenRefine, default to http://localhost:333 - -Cypress contains [exaustive documentation](https://docs.cypress.io/guides/guides/environment-variables.html#Setting) about configuration, but here are two simple ways to configure the execution of the tests: - -#### Overriding with a cypress.env.json file {#overriding-with-a-cypressenvjson-file} - -This file is ignored by Git, and you can use it to configure Cypress locally - -#### Command-line {#command-line} - -You can pass variables at the command-line level - -```shell -yarn --cwd ./main/tests/cypress run cypress open --env OPENREFINE_URL="http://localhost:1234" -``` - -### Visual testing {#visual-testing} - -Tests generally ensure application behavior by making assertions against the DOM, to ensure specific texts or css attributes are present in the document body. -Visual testing, on the contrary, is a way to test applications by comparing images. -A reference screenshot is taken the first time the test runs, and subsequent runs will compare a new screenshot against the reference, at the pixel level. - -Here is an [introduction to visual testing by Cypress](https://docs.cypress.io/plugins/directory#visual-testing). - -In some cases, we are using visual testing. -We are using [Cypress Image Snapshot](https://github.com/jaredpalmer/cypress-image-snapshot) - -Identified cases are so far: - -- testing images created by OpenRefine backend (scatterplots for example) - -Reference screenshots (Called snapshots), are stored in /cypress/snapshots. -And a snapshot can be taken for the whole page, or just a single part of the page. - -#### When a visual test fails {#when-a-visual-test-fails} - -First, Cypress will display the following error message: - -![Diff image when a visual test fails](/img/visual-test-cypress-failure.png) - -Then, a diff image will be created in /cypress/snapshots, this directory is ignored by Git. -The diff images shows the reference image on the left, the image that was taken during the test run on the right, and the diff in the middle. - -![Diff image when a visual test fails](/img/failed-visual-test.png) - -## CI/CD {#cicd} - -In CI/CD, tests are run headless, with the following command-line - -```shell -./refine ui_test chrome -``` - -Results are displayed in the standard output - -## Resources {#resources} - -[Cypress command line options](https://docs.cypress.io/guides/guides/command-line.html#Installation) -[Lots of good Cypress examples](https://example.cypress.io/) diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/homebrew-cask-process.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/homebrew-cask-process.md deleted file mode 100644 index 5615538f4..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/homebrew-cask-process.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -id: homebrew-cask-process -title: Maintaining OpenRefine's Homebrew cask -sidebar_label: Maintaining OpenRefine's Homebrew cask ---- - -[Homebrew](https://brew.sh) is a popular command-line package manager for macOS. Once Homebrew is installed, OpenRefine can be installed via the simple command, `brew install openrefine`. OpenRefine's presence on Homebrew is found in the Homebrew Cask repository project, as a "cask", at https://github.com/Homebrew/homebrew-cask/blob/HEAD/Casks/openrefine.rb. - -**Terminology:** "Homebrew Cask" is the segment of Homebrew where pre-built binaries and GUI applications go, whereas the original "Homebrew" project is reserved for command-line utilities that can be built from source. Because the macOS version of OpenRefine is released as an app bundle with GUI components, it is handled as a Homebrew Cask. - -When there is a new release of OpenRefine, registering the new release with Homebrew can be easily accomplished using Homebrew's `brew bump-cask-pr` command. Full directions for this utility as well as procedures for more complex PRs can be found on [the Homebrew Cask CONTRIBUTING page](https://github.com/Homebrew/homebrew-cask/blob/master/CONTRIBUTING.md), but, a simple version bump is a one-line command. For example, to update Homebrew's version of OpenRefine to 3.4.1, use this command: - -``` -brew bump-cask-pr --version 3.4.1 openrefine -``` - -This command will cause your local Homebrew installation to download the new version of OpenRefine, calculate the installer's new SHA-256 fingerprint value, and construct a pull request under your GitHub account. Once the pull request is submitted, continuous integration tests will run, and a member of the Homebrew community will review the PR. At times there is a backlog on the CI servers, but once tests pass, the community review is typically completed in a matter of hours. - -**Note:** It is important that the OpenRefine release tag and version number are identical, so that the Homebrew cask can find the installer's URL. This is because Homebrew's cask for OpenRefine uses the following formula for constructing the URL to OpenRefine's installer for a given release: - -``` -https://github.com/OpenRefine/OpenRefine/releases/download/#{version}/openrefine-mac-#{version}.dmg -``` - -That is, when you tell Homebrew that OpenRefine is now at version `3.4.1`, Homebrew will try to download the OpenRefine installed from the following URL: - -``` -https://github.com/OpenRefine/OpenRefine/releases/download/3.4.1/openrefine-mac-3.4.1.dmg -``` diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/migrating-older-extensions.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/migrating-older-extensions.md deleted file mode 100644 index 75f40015e..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/migrating-older-extensions.md +++ /dev/null @@ -1,169 +0,0 @@ ---- -id: migrating-older-extensions -title: Migrating older Extensions -sidebar_label: Migrating older Extensions ---- - -## Migrating from Ant to Maven {#migrating-from-ant-to-maven} - -### Why are we doing this change? {#why-are-we-doing-this-change} - -Ant is a fairly old (antique?) build system that does not incorporate any dependency management. -By migrating to Maven we are making it easier for developers to extend OpenRefine with new libraries, and stop having to ship dozens of .jar files in the repository. Using the Maven repository also encourages developers to add dependencies to released versions of libraries instead of custom snapshots that are hard to update. - -### When was this change made? {#when-was-this-change-made} - -The migration was done between 3.0 and 3.1-beta with this commit: -https://github.com/OpenRefine/OpenRefine/commit/47323a9e750a3bc9d43af606006b5eb20ca397b8 - -### How to migrate an extension {#how-to-migrate-an-extension} - -You will need to write a `pom.xml` in the root folder of your extension to configure the compilation process with Maven. Sample `pom.xml` files for extensions can be found in the extensions that are shipped with OpenRefine (`gdata`, `database`, `jython`, `pc-axis` and `wikidata`). A sample extension (`sample`) is also provided, with a minimal build file. - -For any library that your extension depends on, you should try to find a matching artifact in the Maven Central repository. If you can find such an artifact, delete the `.jar` file from your extension and add the dependency in your `pom.xml` file. If you cannot find such an artifact, it is still possible to incorporate your own `.jar` file using `maven-install-plugin` that you can configure in your `pom.xml` file as follows: - - - - org.apache.maven.plugins - maven-install-plugin - 2.5.2 - - - install-wdtk-datamodel - process-resources - - ${basedir}/lib/my-proprietary-library.jar - default - com.my.company - my-library - 0.5.3-SNAPSHOT - jar - true - - - install-file - - - - - - -And add the dependency to the `` section as usual: - - - com.my.company - my-library - 0.5.3-SNAPSHOT - - -## Migrating to Wikimedia's i18n jQuery plugin {#migrating-to-wikimedias-i18n-jquery-plugin} - -### Why are we doing this change? {#why-are-we-doing-this-change-1} - -This adds various important localization features, such as the ability to handle plurals or interpolation. This also restores the language fallback (displaying strings in English if they are not available in the target language) which did not work with the previous set up. - -### When was the migration made? {#when-was-the-migration-made} - -The migration was made between 3.1-beta and 3.1, with this commit: https://github.com/OpenRefine/OpenRefine/commit/22322bd0272e99869ab8381b1f28696cc7a26721 - -### How to migrate an extension {#how-to-migrate-an-extension-1} - -You will need to update your translation files, merging nested objets in one global object, concatenating keys. You can do this by running the following Python script on all your JSON translation files: - - import json - import sys - - with open(sys.argv[1], 'r') as f: - j = json.loads(f.read()) - - result = {} - def translate(obj, path): - res = {} - if type(obj) == str: - result['/'.join(path)] = obj - else: - for k, v in obj.items(): - new_path = path + [k] - translate(v, new_path) - - translate(j, []) - - with open(sys.argv[1], 'w') as f: - f.write(json.dumps(result, ensure_ascii=False, indent=4)) - -Then your javascript files which retrieve the translated strings should be updated: `$.i18n._('core-dialogs')['cancel']` becomes `$.i18n('core-dialogs/cancel')`. You can do this with the following `sed` script: - - sed -i "s/\$\.i18n._(['\"]\([A-Za-z0-9/_\\-]*\)['\"])\[['\"]\([A-Za-z0-9\-\_]*\)[\"']\]/$.i18n('\1\/\2')/g" my_javascript_file.js - -You can then chase down the places where you are concatenating translated strings, and replace that with more flexible patterns using [the plugin's features](https://github.com/wikimedia/jquery.i18n#jqueryi18n-plugin). - -## Migrating from org.json to Jackson {#migrating-from-orgjson-to-jackson} - -### Why are we doing this change? {#why-are-we-doing-this-change-2} - -The org.json (or json-java) library has multiple drawbacks. -* First, it has limited functionality - all the serialization and deserialization has to be done explicitly - an important proportion of OpenRefine's code was dedicated to implementing these; -* Second, its implementation is not optimized for speed - multiple projects have reported speedups when migrating to more modern JSON libraries; -* Third, and this was the decisive factor to initiate the migration: [its license](https://json.org/license) is the MIT license with an additional condition which makes it non-free. Getting rid of this dependency was required by the Software Freedom Conservancy as a prerequisite to become a fiscal sponsor for the project. - -### When was the migration made? {#when-was-the-migration-made-1} - -This change was made between 3.1 and 3.2-beta, with this commit: https://github.com/OpenRefine/OpenRefine/commit/5639f1b2f17303b03026629d763dcb6fef98550b - -### How to migrate an extension or fork {#how-to-migrate-an-extension-or-fork} - -You will need to use the Jackson library to serialize the classes that implement interfaces or extend classes exposed by OpenRefine. -The interface `Jsonizable` was deleted. Any class that used to implement this now needs to be serializable by Jackson, producing the same format as the previous serialization code. This applies to any operation, facet, overlay model or GREL function. If you are new to Jackson, have a look at [this tutorial](https://www.baeldung.com/jackson) to learn how to annotate your class for serialization. Once this is done, you can remove the `void write(JSONWriter writer, Properties options)` method from your class. Note that it is important that you do this migration for all classes implementing the `Jsonizable` interface that are exposed to OpenRefine's core. - -We encourage you to migrate out of org.json completely, but this is only required for the classes that interact with OpenRefine's core. - -#### General notes about migrating {#general-notes-about-migrating} - -OpenRefine's ObjectMapper is available at `ParsingUtilities.mapper`. It is configured to only serialize the fields and getters that have been explicitly marked with `@JsonProperty` (to avoid accidental JSON format changes due to refactoring). On deserialization it will ignore any field in the JSON payload that does not correspond to a field in the Java class. It has serializers and deserializers for `OffsetDateTime` and `LocalDateTime`. - -Useful snippets to use in tests: -* deserialize an instance: `MyClass instance = ParsingUtilities.mapper.readValue(jsonString, MyClass.class);` (replaces calls to `Jsonizable.write`); -* serialize an instance: `String json = ParsingUtilities.mapper.writeValueAsString(myInstance);` (replaces calls to static methods such as `load`, `loadStreaming` or `reconstruct`); -* the equivalent of `JSONObject` is `ObjectNode`, the equivalent of `JSONArray` is `ArrayNode`; -* create an empty JSON object: `ParsingUtilities.mapper.createObjectNode()` (replaces `new JSONObject()`); -* create an empty JSON array: `ParsingUtilities.mapper.createArrayNode()` (replaces `new JSONArray()`). - -Before undertaking the migration, we recommend that you write some tests which serialize and deserialize your objects. This will help you make sure that the JSON format is preserved during the migration. One way to do this is to collect some sample JSON representations of your objects, and check in your tests that deserializing these JSON payloads and serializing them back to JSON preserves the JSON payload. Some utilities are available to help you with that in [`TestUtils`](https://github.com/OpenRefine/OpenRefine/blob/master/main/tests/server/src/com/google/refine/tests/util/TestUtils.java) (we had [some to test org.json serialization](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/tests/server/src/com/google/refine/tests/util/TestUtils.java) before we got rid of the dependency, feel free to copy them). - -#### For functions {#for-functions} - -Before the migration, you had to explicitly define JSON serialization of functions with a `write` method. You should now override the getters returning the various documentation fields. - -Example: `Cos` function [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/expr/functions/math/Cos.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/expr/functions/math/Cos.java). - -#### For operations {#for-operations} - -Before the JSON migration we refactored engine-dependent operations so that the engine configuration is represented by an `EngineConfig` object instead of a `JSONObject`. Therefore the constructor for your operation should be updated to use this new class. Your constructor should also be annotated to be used during deserialization. - -Note that you do not need to explicitly serialize the operation type, this is already done for you by `AbstractOperation`. - -Example: `ColumnRemovalOperation` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java). - -#### For changes {#for-changes} - -Changes are serialized in plain text but often relies on JSON serialization for parts of the data. Just use the methods above with `ParsingUtilities.mapper` to maintain this behaviour. - -Example: `ReconChange` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/model/changes/ReconChange.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java). - -#### For importers {#for-importers} - -The importing options have been migrated from `JSONObject` to `ObjectNode`. Your compiler should help you propagate this change. Utility functions in `JSONUtilities` have been migrated to Jackson so you should have minimal changes if you used them. - -Example: `TabularImportingParserBase` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/importers/TabularImportingParserBase.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/importers/TabularImportingParserBase.java). - -#### For overlay models {#for-overlay-models} - -Migrate serialization and deserialization as for other objects. - -Example: `WikibaseSchema` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java#L203) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java#L60) - -#### For preference values {#for-preference-values} - -Any class that is stored in OpenRefine's preference now needs to implement the `com.google.refine.preferences.PreferenceValue` interface. The static `load` method and the `write` method used previously for deserialization should be deleted and regular Jackson serialization and deserialization should be implemented instead. Note that you do not need to explicitly serialize the class name, this is already done for you by the interface. - -Example: `TopList` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/preference/TopList.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/preference/TopList.java) diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/openrefine-api.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/openrefine-api.md deleted file mode 100644 index c1f0a9c0d..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/openrefine-api.md +++ /dev/null @@ -1,281 +0,0 @@ ---- -id: openrefine-api -title: OpenRefine API -sidebar_label: OpenRefine API ---- - -This is a generic API reference for interacting with OpenRefine's HTTP API. - -## Create project: {#create-project} - -> **Command:** _POST /command/core/create-project-from-upload_ - -When uploading files you will need to send the data as `multipart/form-data`. This is different to all other API calls which use a mixture of query string and POST parameters. - -multipart form-data: - - 'project-file' : file contents - 'project-name' : project name - 'format' : format of data in project-file (e.g. 'text/line-based/*sv') [optional] - 'options' : json object containing options relevant to the file format [optional - however, some importers may have required options, such as `recordPath` for the JSON & XML importers]. - -The formats supported will depend on the version of OpenRefine you are using and any Extensions you have installed. The common formats include: - -* 'text/line-based': Line-based text files -* 'text/line-based/*sv': CSV / TSV / separator-based files [separator to be used in specified in the json submitted to the options parameter] -* 'text/line-based/fixed-width': Fixed-width field text files -* 'binary/text/xml/xls/xlsx': Excel files -* 'text/json': JSON files -* 'text/xml': XML files - -If the format is omitted OpenRefine will try to guess the format based on the file extension and MIME type. -The values which can be specified in the JSON object submitted to the 'options' parameter will vary depending on the format being imported. If not specified the options will either be guessed at by OpenRefine (e.g. separator being used in a separated values file) or a default value used. The import options for each file format are not currently documented, but can be seen in the OpenRefine GUI interface when importing a file of the relevant format. - -If the project creation is successful, you will be redirected to a URL of the form: - - http://127.0.0.1:3333/project?project= - -From the project parameter you can extract the project id for use in future API calls. The content of the response is the HTML for the OpenRefine interface for viewing the project. - -### Get project models: {#get-project-models} - -> **Command:** _GET /command/core/get-models?_ - - 'project' : project id - -Recovers the models for the specific project. This includes columns, records, overlay models, scripting. In the columnModel a list of the columns is displayed, key index and name, and column groupings. - -### Response: {#response} -**On success:** -```JSON -{ - "columnModel":{ - "columns":[ - { - "cellIndex":0, - "originalName":"email", - "name":"email" - }, - { - "cellIndex":1, - "originalName":"name", - "name":"name" - }, - { - "cellIndex":2, - "originalName":"state", - "name":"state" - }, - { - "cellIndex":3, - "originalName":"gender", - "name":"gender" - }, - { - "cellIndex":4, - "originalName":"purchase", - "name":"purchase" - } - ], - "keyCellIndex":0, - "keyColumnName":"email", - "columnGroups":[ - - ] - }, - "recordModel":{ - "hasRecords":false - }, - "overlayModels":{ - - }, - "scripting":{ - "grel":{ - "name":"General Refine Expression Language (GREL)", - "defaultExpression":"value" - }, - "jython":{ - "name":"Python / Jython", - "defaultExpression":"return value" - }, - "clojure":{ - "name":"Clojure", - "defaultExpression":"value" - } - } -} -``` - - - -## Apply operations {#apply-operations} - -> **Command:** _POST /command/core/apply-operations?_ - -In the parameter - - 'project' : project id - -In the form data - - 'operations' : Valid JSON **Array** of OpenRefine operations - -Example of a Valid JSON **Array** -```JSON -'[ - { - "op":"core/column-addition", - "description":"Create column zip type at index 15 based on column Zip Code 2 using expression grel:value.type()", - "engineConfig":{ - "mode":"row-based", - "facets":[] - }, - "newColumnName":"zip type", - "columnInsertIndex":15, - "baseColumnName":"Zip Code 2", - "expression":"grel:value.type()", - "onError":"set-to-blank" - }, - { - "op":"core/column-addition", - "description":"Create column testing at index 15 based on column Zip Code 2 using expression grel:value.toString()0,5]", - "engineConfig":{ - "mode":"row-based", - "facets":[] - }, - "newColumnName":"testing", - "columnInsertIndex":15, - "baseColumnName":"Zip Code 2", - "expression":"grel:value.toString()[0,5]", - "onError":"set-to-blank" - } -] -``` - -On success returns JSON response -`{ "code" : "ok" }` - -## Export rows {#export-rows} - -> **Command:** _POST /command/core/export-rows_ - -In the parameter - - 'project' : project id - 'format' : format... (e.g 'tsv', 'csv') - -In the form data - - 'engine' : JSON string... (e.g. '{"facets":[],"mode":"row-based"}') - -Returns exported row data in the specified format. The formats supported will depend on the version of OpenRefine you are using and any Extensions you have installed. The common formats include: - -* csv -* tsv -* xls -* xlsx -* ods -* html - -## Delete project {#delete-project} - -> **Command:** _POST /command/core/delete-project_ - - 'project' : project id... - -Returns JSON response - -## Check status of async processes {#check-status-of-async-processes} - -> **Command:** _GET /command/core/get-processes_ - - 'project' : project id... - -Returns JSON response - -## Get all projects metadata: {#get-all-projects-metadata} - -> **Command:** _GET /command/core/get-all-project-metadata_ - -Recovers the meta data for all projects. This includes the project's id, name, time of creation and last time of modification. - -### Response: {#response-1} -```json -{ - "projects":{ - "[project_id]":{ - "name":"[project_name]", - "created":"[project_creation_time]", - "modified":"[project_modification_time]" - }, - ...[More projects]... - } -} -``` - -## Expression Preview {#expression-preview} -> **Command:** _POST /command/core/preview-expression_ - -Pass some expression (GREL or otherwise) to the server where it will be executed on selected columns and the result returned. - -### Parameters: {#parameters} -* **cellIndex**: _[column]_ -The cell/column you wish to execute the expression on. -* **rowIndices**: _[rows]_ -The rows to execute the expression on as JSON array. Example: `[0,1]` -* **expression**: _[language]_:_[expression]_ -The expression to execute. The language can either be grel, jython or clojure. Example: grel:value.toLowercase() -* **project**: _[project_id]_ -The project id to execute the expression on. -* **repeat**: _[repeat]_ -A boolean value (true/false) indicating whether or not this command should be repeated multiple times. A repeated command will be executed until the result of the current iteration equals the result of the previous iteration. -* **repeatCount**: _[repeatCount]_ -The maximum amount of times a command will be repeated. - -### Response: {#response-2} -**On success:** -```json -{ - "code": "ok", - "results" : [result_array] -} -``` - -The result array will hold up to ten results, depending on how many rows there are in the project that was specified by the [project_id] parameter. Each result is the string that would be put in the cell if the GREL command was executed on that cell. Note that any expression that would return an array or JSon object will be jsonized, although the output can differ slightly from the jsonize() function. - -**On error:** -```JSON -{ - "code": "error", - "type": "[error_type]", - "message": "[error message]" -} -``` - -## Third-party software libraries {#third-party-software-libraries} -Libraries using the [OpenRefine API](openrefine-api): - -### Python {#python} -* [refine-client-py](https://github.com/PaulMakepeace/refine-client-py/) - * Or this fork of the above with an extended CLI [openrefine-client](https://github.com/felixlohmeier/openrefine-client) -* [refine-python](https://github.com/maxogden/refine-python) - -### Ruby {#ruby} -* [refine-ruby](https://github.com/distillytics/refine-ruby) - * The above is a maintained fork of [google-refine](https://github.com/maxogden/refine-ruby) -* [google_refine](https://github.com/chengguangnan/google_refine) - -### NodeJS {#nodejs} -* [node-openrefine](https://github.com/pm5/node-openrefine) - -### R {#r} -* [rrefine](https://cran.r-project.org/web/packages/rrefine/index.html) - -### PHP {#php} -* [openrefine-php-client](https://github.com/keboola/openrefine-php-client) - -### Java {#java} -* [refine-java](https://github.com/dtap-gmbh/refine-java) - -### Bash {#bash} -* [bash-refine.sh](https://gist.github.com/felixlohmeier/d76bd27fbc4b8ab6d683822cdf61f81d) (templates for shell scripts) diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/reconciliation-api.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/reconciliation-api.md deleted file mode 100644 index 222cfef16..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/reconciliation-api.md +++ /dev/null @@ -1,259 +0,0 @@ ---- -id: reconciliation-api -title: Reconciliation API -sidebar_label: Reconciliation API ---- - -_This page is kept for the record. [A cleaner version of this specification](https://reconciliation-api.github.io/specs/0.1/) was written by the [W3C Entity Reconciliation Community Group](https://www.w3.org/community/reconciliation/), which has been formed to improve and promote this API. Join the community group to get involved!_ - -_This is a technical description of the mechanisms behind the reconciliation system in OpenRefine. For usage instructions, see [Reconciliation](/manual/reconciling)._ - -## Introduction {#introduction} - -A reconciliation service is a web service that, given some text which is a name or label for something, and optionally some additional details, returns a ranked list of potential entities matching the criteria. The candidate text does not have to match each entity's official name perfectly, and that's the whole point of reconciliation--to get from ambiguous text name to precisely identified entities. For instance, given the text "apple", a reconciliation service probably should return the fruit apple, the Apple Inc. company, and New York city (also known as the Big Apple). - -Entities are identified by strong identifiers in some particular identifier space. In the same identifier space, identifiers follow the same syntax. For example, given the string "apple", a reconciliation service might return entities identified by the strings " [Q89](https://www.wikidata.org/wiki/Q89)", "[Q312](https://www.wikidata.org/wiki/Q312)", and "[Q60](https://www.wikidata.org/wiki/Q60)", in the Wikidata ID space. Each reconciliation service can only reconcile to one single identifier space, but several reconciliation services can reconcile to the same identifier space. - -OpenRefine defines a reconciliation API so that users can use the reconciliation features of OpenRefine with various databases (this API was originally developed to work with the now deprecated "[Freebase](https://en.wikipedia.org/wiki/Freebase)" API). - -Informally, the main function of any reconciliation service is to find good candidates in the underlying database, given the following data: - -* A string, which is normally the name or title of the entity, in some language. -* Optionally, a type which can be used to narrow down the search to entities of this type. OpenRefine does not define a particular set of acceptable types: this choice is left to the reconciliation service (see the suggest API for that). -* Optionally, a list of properties and their values, which can be used to refine the search. For instance, when reconciling a database of books, the author name or the publication date are useful bits of information that can be transferred to the reconciliation service. This information will be sent to the reconciliation service if the user binds columns to properties. Again, the notion of property is not predefined in OpenRefine: its definition depends on the reconciliation service. - -A standard reconciliation service is a HTTP-based RESTful JSON-formatted API. It consists of various endpoints, each of which fulfills a specific function. Only the first one is mandatory. - -* The root URL. This is the URL that users will need to add the service to OpenRefine. For instance, the Wikidata reconciliation interface in English has the following URL: [https://tools.wmflabs.org/openrefine-wikidata/en/api](https://tools.wmflabs.org/openrefine-wikidata/en/api) -* _Optional._ The suggest API, which enables autocompletion at various places in OpenRefine; -* _Optional._ The preview API, which lets users preview the reconciled items directly from OpenRefine; -* _Optional._ The data extension API, which lets users add columns from reconciled values based on the properties of the items in the reconciliation service. - -The specification of each of these endpoints is given in the following sections. - -## Workflow overview {#workflow-overview} - -OpenRefine communicates with reconciliation services in the following way. - -* The user adds the service by inputting its endpoint in the dialog. OpenRefine queries this URL to retrieve its [service and metadata](reconciliation-api#service-metadata) which contains basic metadata about the service (such as its name and the available features). -* The user selects a service to configure reconciliation using this service. OpenRefine queries the reconciliation service in [batch mode](reconciliation-api#multiple-query-mode) on the first ten items of the column to be reconciled. The reconciliation results are not presented to the user, but the types of the candidate items are aggregated and proposed to the user to restrict the matching to one of them. For example, if your service reconciles both people and companies, and the query for the first 10 names returns 15 candidates which are people and 7 candidates which are companies, the types will be presented to the user in that order. They can override the order and pick whichever type they want, as well as chosen a different type by hand or choose to reconcile without any type information. -* The user configures the reconciliation. If a [suggest service](reconciliation-api#suggest-apis) is available, it will be used to provide auto-completion in the dialog, to choose types or properties. -* When reconciliation starts, OpenRefine queries the service in [batch mode](reconciliation-api#multiple-query-mode) for small batches of rows and stores the responses of the service. -* Once reconciliation is complete, the results are displayed. The user makes reconciliation decisions based on the choices provided. If a [suggest service](reconciliation-api#suggest-apis) is available, it will be used to input custom reconciliation decisions. If a [preview service](reconciliation-api#preview-api) is available, the user will be able to preview the reconciliation candidates without leaving OpenRefine. - -## Main reconciliation service {#main-reconciliation-service} - -The root URL has two functions: - -* it returns the [service and metadata](reconciliation-api#service-metadata) if no query is provided. -* when given the `queries` parameter, it performs a [set of queries in batch mode](reconciliation-api#multiple-query-mode) and returns the results for each query. This makes it efficient - -There is a deprecated "single query" mode which is used if the `query` parameter is given. This mode is no longer supported or used by OpenRefine and other API consumers should not rely on it. - -### Service metadata {#service-metadata} - -When a service is called with just a JSONP `callback` parameter and no other parameters, it must return its _service metadata_ as a JSON object literal with the following fields: - -* `"name"`: the name of the service, which will be used to display the service in the reconciliation menu; -* `"identifierSpace"`: an URI for the type of identifiers returned by the service; -* `"schemaSpace"`: an URI for the type of types understood by the service. -* `"view"` an object with a template URL to view a given item from its identifier: `"view": {"url":"http://example.com/object/{{id}}"} ` - -The last two parameters are mainly useful to assert that the identifiers returned by two different reconciliation services mean the same thing. Other fields are optional: they are used to specify the URLs for the other endpoints (suggest, preview and extend) described in the next sections. - -Here are two live examples: - -1. [https://tools.wmflabs.org/openrefine-wikidata/en/api](https://tools.wmflabs.org/openrefine-wikidata/en/api) -2. [http://refine.codefork.com/reconcile/viaf](http://refine.codefork.com/reconcile/viaf) - -```json -{ - "name" : "Wikidata Reconciliation for OpenRefine (en)", - "identifierSpace" : "http://www.wikidata.org/entity/", - "schemaSpace" : "http://www.wikidata.org/prop/direct/", - "view" : { - "url" : "https://www.wikidata.org/wiki/{{id}}" - }, - "defaultTypes" : [], - "preview" : { - ... - }, - "suggest" : { - ... - }, - "extend" : { - ... - }, -} -``` - -## Query Request {#query-request} - -### Multiple Query Mode {#multiple-query-mode} - -A call to a standard reconciliation service API for multiple queries looks like this: - - http://foo.com/bar/reconcile?queries={...json object literal...} - -The json object literal has zero or more key/value pairs with arbitrary keys where the value is in the same format as a single query, e.g. - - http://foo.com/bar/reconcile?queries={ "q0" : { "query" : "foo" }, "q1" : { "query" : "bar" } } - -"q0" and "q1" can be arbitrary strings. They will be used to key the results returned. - -For larger data, it can make sense to use POST requests instead of GET: - -```shell -curl -X POST -d 'queries={ "q0" : { "query" : "foo" }, "q1" : { "query" : "bar" } }' http://foo.com/bar/reconcile -``` - -OpenRefine uses POST for all requests, so make sure your service supports the format above. - -### **DEPRECATED** Single Query Mode {#deprecated-single-query-mode} - -A call to a reconciliation service API for a single query looks like either of these: - - http://foo.com/bar/reconcile?query=...string... - http://foo.com/bar/reconcile?query={...json object literal...} - -If the query parameter is a string, then it's an abbreviation of `query={"query":...string...}`. Here are two live examples: - -1. [https://tools.wmflabs.org/openrefine-wikidata/en/api?query=boston](https://tools.wmflabs.org/openrefine-wikidata/en/api?query=boston) -2. [https://tools.wmflabs.org/openrefine-wikidata/en/api?query={%22query%22:%22boston%22,%22type%22:%22Q515%22}](https://tools.wmflabs.org/openrefine-wikidata/en/api?query={%22query%22:%22boston%22,%22type%22:%22Q515%22}) - -### Query JSON Object {#query-json-object} - -The query json object literal has a few fields - -| Parameter | Description | -| --- | --- | -| "query" | A string to search for. Required. | -| "limit" | An integer to specify how many results to return. Optional. | -| "type" | A single string, or an array of strings, specifying the types of result e.g., person, product, ... The actual format of each type depends on the service (e.g., "Q515" as a Wikidata type). Optional. | -| "type\_strict" | A string, one of "any", "all", "should". Optional. | -| "properties" | Array of json object literals. Optional | - -Each json object literal of the `"properties"` array is of this form - -```json -{ - "p" : string, property name, e.g., "country", or - "pid" : string, property ID, e.g., "P17" as a Wikidata property ID - "v" : a single, or an array of, string or number or object literal, e.g., "Japan" - } -``` - -A `"v"` object literal would have a single key `"id"` whose value is an identifier resolved previously to the same identity space. - -Here is an example of a full query parameter: - -```json -{ - "query" : "Ford Taurus", - "limit" : 3, - "type" : "Q3231690", - "type_strict" : "any", - "properties" : [ - { "p" : "P571", "v" : 2009 }, - { "pid" : "P176" , "v" : { "id" : "Q20827633" } } - ] - } -``` - -## Query Response {#query-response} -For multiple queries, the response is a JSON literal object with the same keys as in the request - -```json -{ - "q0" : { - "result" : { ... } - }, - "q1" : { - "result" : { ... } - } - } -``` - -Each result consists of a JSON literal object with the structure - -```json -{ - "result" : [ - { - "id" : ... string, database ID ... - "name" : ... string ... - "type" : ... array of strings ... - "score" : ... double ... - "match" : ... boolean, true if the service is quite confident about the match ... - }, - ... more results ... - ], - ... potentially some useful envelope data, such as timing stats ... - } -``` - -The results should be sorted by decreasing score. -The service must also support JSONP through a callback parameter ie &callback=foo. - -## Preview API {#preview-api} - -The preview service API (complementary to the reconciliation service API) is quite simple. Pass it an identifier and it renders information about the corresponding entity in an HTML page, which will be shown in an iframe inside OpenRefine. The given width and height dimensions tell OpenRefine how to size that iframe. - -## Suggest APIs {#suggest-apis} - -In the "Start Reconciling" dialog box in OpenRefine, you can specify which type of entities the column in question contains. For instance, the column might contains titles of scientific journals. But you don't know the identifier corresponding to the "scientific journal" type. So we need a suggest API that translates "scientific journal" to something like, say, "[Q5633421](https://www.wikidata.org/wiki/Q5633421)" if we're reconciling against Wikidata. - -In the same dialog box, you can specify that other columns should be used to provide more details for the reconciliation. For instance, if there is a column specifying the journals ISSN (a standard identifier for serial publications), passing that data onto the reconciliation service might make reconciliation more accurate. You might want to specify how that second column is related to the column being reconciled, but you might not now how to specify "ISSN" as a precise relationship. So we need a suggest API that translates "ISSN" to something like "[P236](https://www.wikidata.org/wiki/Property:P236)" (once again using Wikidata as an example). - -There is also a need for a suggest service for entities rather than just for types and properties. When a cell has no good candidate, then you would want to perform a search yourself (by clicking on "search for match" in that cell). - -Each suggest API has 2 jobs to do: - -* translate what the user type into a ranked list of entities (and this is similar to the core reconciliation service and might share the same implementation) -* render a flyout when an entity is moused over or highlighted using arrow keys (and this is similar to the preview API and might share the same implementation) - -The metadata for each suggest API (type, property, or entity) is as follows: - -```json -{ - "service_url" : "... url including only the domain ...", - "service_path" : "... optional relative path ...", - "flyout_service_url" : "... optional url including only the domain ...", - "flyout_service_path" : "... optional relative path ..." -} -``` - -The `service_url` field is required and it should look like this: `http://foo.com`. There should be no trailing `/` at the end. The other fields are optional and have defaults if not provided: - -* `service_path` defaults to `/private/suggest` -* `flyout_service_url` defaults to the provided `service_url` field -* `flyout_service_path` defaults to `/private/flyout` - -Refer to [the Suggest API documentation](suggest-api) for further details. - -## Data Extension {#data-extension} - -From OpenRefine 2.8 it is possible to fetch values from reconcilied sources natively. This is only possible for the reconciliation endpoints that support this additional feature, described in the [Data Extension API documentation](Data-Extension-API). - -## Examples {#examples} - -We've cloned a number of the Refine reconciliation services as a way of providing them visibility. They can be found at [https://github.com/OpenRefine](https://github.com/OpenRefine) - -Some examples of reconciliation services which have made code available include: - -* [https://github.com/dergachev/redmine-reconcile](https://github.com/dergachev/redmine-reconcile) - Python & Flask implementation that just returns the given name/number with a base url prepended -* [https://github.com/okfn/helmut](https://github.com/okfn/helmut) - A generic Refine reconciliation API implementation using Python & Flask -* [https://github.com/mblwhoi/reconciliation_service_skeleton](https://github.com/mblwhoi/reconciliation_service_skeleton) - Skeleton for Standalone Python & Flask Reconciliation Service for Refine -* [https://github.com/mikejs/reconcile-demo](https://github.com/mikejs/reconcile-demo) -* [https://github.com/rdmpage/phyloinformatics/tree/master/services) - PHP examples (reconciliation\_\*.php](https://github.com/rdmpage/phyloinformatics/tree/master/services) -* [http://lucene.apache.org/solr/) and Python Django](https://github.com/opensemanticsearch/open-semantic-entity-search-api](https://github.com/opensemanticsearch/open-semantic-entity-search-api) - Open Source REST-API for Named Entity Extraction, Normalization, Reconciliation, Recommendation, Named Entity Disambiguation and Named Entity Linking of named entities in full-text documents by SKOS thesaurus, RDF ontologies, SQL databases and lists of names (powered by [Apache Solr) -* [https://github.com/granoproject/grano-reconcile](https://github.com/granoproject/grano-reconcile) - python example -* [https://github.com/codeforkjeff/conciliator](https://github.com/codeforkjeff/conciliator) - a Java framework for creating reconciliation services over the top of existing data sources. The code includes reconciliation services layered over [the Virtual International Authority File (VIAF)](http://viaf.org), [ORCID](http://orcid.org), [the Open Library](http://openlibrary.org) and [Apache Solr](http://lucene.apache.org/solr/). -* The open-reconcile project provides a complete Java based reconciliation service which queries a SQL database. [https://code.google.com/p/open-reconcile](https://code.google.com/p/open-reconcile) -* The [RDF Extension](http://refine.deri.ie) incorporates, among other things, reconciliation support with different approaches: - * a service to reconciliate against querying a SPARQL endpoint - * reconcile against a provided RDF file - * based on Apache Stanbol ([implementation details](https://github.com/fadmaa/grefine-rdf-extension/pull/59)) -* [Sunlight Labs](https://github.com/sunlightlabs) implemented a reconciliation service using Piston on Django for their [Influence Explorer](https://sunlightlabs.github.io/datacommons/). [The code is available](https://github.com/sunlightlabs/datacommons/blob/master/dcapi/reconcile/handlers.py) - -Also look at the [[Reconcilable Data Sources]] page for other examples of available reconciliation services that are compatible with Refine. Not all of them are open source, but they might spark some ideas. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/suggest-api.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/suggest-api.md deleted file mode 100644 index fdb49299c..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/suggest-api.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -id: suggest-api -title: Suggest API -sidebar_label: Suggest API ---- - -The Suggest API has 2 entry points: - -- `suggest`: translates some text that the user has typed, optionally constrained in some ways, to a ranked list of entities (this is very similar to a [Reconciliation API](reconciliation-api) and can share the same implementation) -- `flyout` : renders a small view of an entity - -For the `suggest` entry point, it is important to balance speed versus accuracy. The widget must respond in interactive time, meaning about 200 msec, for each change to the user's text input. At the same time, the ranked list of entities must seem quite relevant to what the user types. - -Similarly, for the `flyout` entry point, it is important to respond quickly while providing enough essential details so that the user can visually check if the highlighted entity is the desired one. You probably would want to embed a thumbnail image, as we have found that images are excellent for visual identification. - -## suggest Entry Point {#suggest-entry-point} - -The `suggest` entry point takes the following URL parameters - -Parameter | Description | Required/Optional -----------|-----------------------------|------------------ - "prefix" | a string the user has typed | required - "type" | optional, a single string, or an array of strings, specifying the types of result e.g., person, product, ... The actual format of each type depends on the service | optional | - "type\_strict" | optional, a string, one of "any", "all", "should" | optional | - "limit" | optional, an integer to specify how many results to return | optional | - "start" | optional, an integer to specify the first result to return (thus in conjunction with `limit`, support pagination) | optional | - -The Suggest API should return results as JSON (JSONP must also be supported). The JSON should consist of a 'result' array containing objects with at least a 'name' and 'id'. The JSON response can optionally include other information as illustrated in this structure for a full JSON response: -```json - { - "code" : "/api/status/ok", - "status" : "200 OK", - "prefix" : ... string, the prefix URL parameter echoed back ... - "result" : [ - { - "id" : ... string, identifier of entity ... - "name" : ... string, nameof entity ... - "notable" : [{ - "id" : ... string, identifier of type ... - "name" : ... string, name of type ... - }, ...] - }, - ... more results ... - ], - } -``` - -* `code` (optional) error/success state of the API above the level of HTTP. Use "/api/status/ok" for a successful request and "/api/status/error" if there has been an error. -* `status` (optional) should correspond to the HTTP response code. -* `prefix` (optional) the query string submitted to the Suggest API. -* `result` (required) array containing multiple results from the Suggest API consisting of at least an id and name. -* `id` (required) the id of an entity being suggested by the Suggest API -* `name` (required) a short string which labels or names the entity being suggested by the Suggest API -* `description` (optional) a short description of the item, which will be displayed below the name -* `notable` (optional) is a a list of JSON objects that describes the types of the entity. They are rendered in addition to the entity's name to provide more disambiguation details and stored in the reconciliation data of the cells. This list can also be supplied as a list of type identifiers (such as `["Q5"]` instead of `[{"id":"Q5","name":"human"}]`). - -Here is an example of a minimal request and response using the Suggest API layered over [Wikidata](https://www.wikidata.org): - -URL: https://tools.wmflabs.org/openrefine-wikidata/en/suggest/entity?prefix=A5 -JSON response: - -```json -{ - "result": [ - { - "name": "A5", - "description": "road", - "id": "Q429719" - }, - { - "name": "Apple A5", - "description": null, - "id": "Q420764" - }, - { - "name": "A5 autoroute", - "description": "controlled-access highway from Paris's Francilienne to the A31 near Beauchemin", - "id": "Q788832" - }, - ... - ] -} -``` - -## flyout Entry Point {#flyout-entry-point} - -The `flyout` entry point takes a single URL parameter: `id`, which is the identifier of the entity to render, as a string. It also takes a `callback` parameter to support JSONP. It returns a JSON object literal with a single field: `html`, which is the rendered view of the given entity. - -Here is an example of a minimal request and response using the Suggest API layered over [[Wikidata](https://www.wikidata.org) with only the required fields in each case: - -URL: https://tools.wmflabs.org/openrefine-wikidata/en/flyout/entity?id=Q786288 -JSON response: - -```json -{ - "html": "

national road in Latvia

", - "id": "Q786288" -} -``` - -OpenRefine incorporates a set of `fbs-` CSS class names which can be used in the flyout HTML if desired to render the flyout information in a standard style. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/technical-reference-index.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/technical-reference-index.md deleted file mode 100644 index 170b60958..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/technical-reference-index.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -id: technical-reference-index -title: OpenRefine technical reference -sidebar_label: Technical Reference Index ---- - -Technical reference index diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/translating.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/translating.md deleted file mode 100644 index 73de2fee3..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/translating.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -id: translating -title: Translate the OpenRefine interface -sidebar_label: Translate the OpenRefine interface ---- - -Currently supported languages include English, Spanish, Chinese, French, Hebrew, Italian and Japanese. - -![Translation status](https://hosted.weblate.org/widgets/openrefine/-/287x66-grey.png) - -You can help translate OpenRefine into your language by visiting [Weblate](https://hosted.weblate.org/engage/openrefine/?utm_source=widget) which provides a web based UI to edit and add translations and sends automatic pull requests back to our project. - -Click to help translate --> [Weblate](https://hosted.weblate.org/engage/openrefine/?utm_source=widget) - -## User entry of language data ## {#user-entry-of-language-data-} - -Localized strings are entered in a .json file, one per language. They are located in the folder `main/webapp/modules/core/langs/` in a file named `translation-xx`.json, where xx is the language code (i.e. fr for French). - -### Simple case of localized string ### {#simple-case-of-localized-string-} -This is an example of a simple string, with the start of the JSON file. This example is for French. -``` -{ - "name": "Français", - "core-index/help": "Aide", - (… more lines) -} -``` - -So the key `core-index/help` will render as `"Aide"` in French. - -### Localization with a parameterized value ### {#localization-with-a-parameterized-value-} -In this example, the name of the column (represented by `$1` in this example), will be substituted with the string of the name of the column. - -`"core-facets/edit-facet-title": "Cliquez ici pour éditer le nom de la facette\nColonne : $1",` - -### Localization with a singular/plural value ### {#localization-with-a-singularplural-value-} -In this example, one of the parameter will have a different string depending if the value is 1 or another value. -In this example, the string for page, the second parameter, `$2`, will have an « s » or not depending on the value of `$2`. - -`"core-views/goto-page": "$1 de $2 {{plural:$2|page|pages}}"` - -## Front End Coding {#front-end-coding} - -The OpenRefine front end has been localized using the [Wikidata jquery.i18n library](https://github.com/OpenRefine/OpenRefine/pull/1285. The localized text is stored in a JSON dictionary on the server and retrieved with a new OpenRefine command. - -### Adding a new string {#adding-a-new-string} - -There should be no hard-coded language strings in the HTML or JSON used for the front end. If you need a new string, first check the existing strings to make sure there isn't an equivalent string, **in an equivalent context**, that you can reuse. Context is important because it can affect how the same literal English text is translated. This cuts down on the amount of text which needs to be translated. - -Strings should be entire sentences or phrases and should include substitution variables for any parameters. Do not concatenate strings in either Java or Javascript (or implicitly by laying them out in a specific order). So, instead of `"You have " + count + " row(s)"` (or worse `count != 1 ? " rows" : " row"`), internationalize everything together so that it can be translated taking into account word ordering and plurals for different languages, ie `"You have $1 {{plural $1: row|rows}}"`, passing the parameter(s) into the `$.i18n` call. - -If there's no string you can reuse, allocate an available key in the appropriate translation dictionary and add the default string, e.g. - -```json -..., -"section/newkey": "new default string for this key", -... -``` - -and then set the text (or HTML) of your HTML element using i18n helper method. So given an HTML fragment like: -```html - -``` -we could set its text using: -``` -$('#new-html-element-id').text($.i18n('section/newkey'])); -``` -or, if you need to embed HTML tags: -``` -$('#new-html-element-id').html($.i18n('section/newkey']); -``` - -### Adding a new language {#adding-a-new-language} - -The language dictionaries are stored in the `langs` subdirectory for the module e.g. - -* https://github.com/OpenRefine/OpenRefine/tree/master/main/webapp/modules/core/langs for the main interface -* https://github.com/OpenRefine/OpenRefine/tree/master/extensions/gdata/module/langs for google spreadsheet connection -* https://github.com/OpenRefine/OpenRefine/tree/master/extensions/database/module/langs for database via JDBC -* https://github.com/OpenRefine/OpenRefine/tree/master/extensions/wikidata/module/langs for Wikidata - -To add support for a new language, copy `translation-en.json` to `translation-.json` and have your translator translate all the value strings (ie right hand side). - -#### Main interface {#main-interface} - The translation is best done [with Weblate](https://hosted.weblate.org/engage/openrefine/?utm_source=widget). Files are periodically merged by the developer team. - -Run the latest (hopefully cloned from github) version and check whether translated words fit to the layout. Not all items can be translated word by word, especially into non-Ìndo-European languages. - -If you see any text which remains in English even when you have checked all items, please create bug report in the issue tracker so that the developers can fix it. - -#### Extensions {#extensions} - -Extensions can be translated via Weblate just like the core software. - -The new extension for Wikidata contains lots of domain-specific concepts, with which you may not be familiar. The Wikidata may not have reconciliation service for your language. I recommend checking the glossary(https://www.wikidata.org/wiki/Wikidata:Glossary) to be consistent. - -By default, the system tries to load the language file corresponding to the currently in-use browser language. To override this setting a new menu item ("Language Settings") has been added at the index page. -To support a new language file, the developer should add a corresponding entry to the dropdown menu in this file: `/OpenRefine/main/webapp/modules/core/scripts/index/lang-settings-ui.html`. The entry should look like: -```javascript - -``` - -## Server / Backend Coding {#server--backend-coding} - -Currently no back end functions are translated, so things like error messages, undo history, etc may appear in English form. Rather than sending raw error text to the front end, it's better to send an error code which is translated into text on the front end. This allows for multiple languages to be supported. diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/version-release-process.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/version-release-process.md deleted file mode 100644 index 55ea4ec85..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/version-release-process.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -id: version-release-process -title: How to do an OpenRefine version release -sidebar_label: How to do an OpenRefine version release ---- diff --git a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/writing-extensions.md b/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/writing-extensions.md deleted file mode 100644 index 3511cc31f..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.4/technical-reference/writing-extensions.md +++ /dev/null @@ -1,326 +0,0 @@ ---- -id: writing-extensions -title: Writing Extensions -sidebar_label: Writing Extensions ---- - -## Introduction {#introduction} - -This is a very brief overview of the structure of OpenRefine extensions. For more detailed documentation and step-by-step guides please see the following external documentation/tutorials: - -* Giuliano Tortoreto has [written documentation detailling how to build extension for OpenRefine](https://github.com/giTorto/OpenRefineExtensionDoc/raw/master/main.pdf) -* Owen Stephens has written [a guide to developing an extension which adds new GREL functions to OpenRefine](http://www.meanboyfriend.com/overdue_ideas/2017/05/writing-an-extension-to-add-new-grel-functions-to-openrefine/). - -OpenRefine makes use of a modified version of the [Butterfly framework](https://github.com/OpenRefine/simile-butterfly/tree/openrefine) to provide an extension architecture. OpenRefine extensions are Butterfly modules. You don't really need to know about Butterfly itself, but you might encounter "butterfly" here and there in the code base. - -Extensions that come with the code base are located under [the extensions subdirectory](https://github.com/OpenRefine/OpenRefine/tree/master/extensions), but when you develop your own extension, you can put its code anywhere as long as you point Butterfly to it. That is done by any one of the following methods - -* refer to your extension's directory in [the butterfly.properties file](https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/WEB-INF/butterfly.properties) through a `butterfly.modules.path` setting. -* specify the butterfly.modules.path property on the command line when you run OpenRefine. This overrides the values in the property file, so you need to include the default values first e.g. `-Dbutterfly.modules.path=modules,../../extensions,/path/to/your/extension` - -Please note that you should bundle any dependencies yourself, so you are insulated from OpenRefine packaging changes over time. - -### Directory Layout {#directory-layout} - -A OpenRefine extension sits in a file directory that contains the following files and sub-directories: - -``` -pom.xml - src/ - com/foo/bar/... *.java source files - module/ - *.html, *.vt files - scripts/... *.js files - styles/... *.css and *.less files - images/... image files - MOD-INF/ - lib/*.jar files - classes/... java class files - module.properties - controller.js -``` - -The file named module.properties (see [example](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/sample/module/MOD-INF/module.properties)) contains the extension's metadata. Of importance is the name field, which gives the extension a name that's used in many other places to refer to it. This can be different from the extension's directory name. - -``` -name = my-extension-name -``` - -Your extension's client-side resources (.html, .js, .css files) stored in the module/ subdirectory will be accessible from http://127.0.0.1:3333/extension/my-extension-name/ when OpenRefine is running. - -Also of importance is the dependency - -``` -requires = core -``` - -which makes sure that the core module of OpenRefine is loaded before the extension attempts to hook into it. - -The file named controller.js is responsible for registering the extension's hooks into OpenRefine. Look at the sample-extension extension's [controller.js](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/sample/module/MOD-INF/controller.js) file for an example. It should have a function called init() that does the hook registrations. - -The `pom.xml` file is an [Apache Maven](http://maven.apache.org/) build file. You can make a copy of the sample extension's `pom.xml` file to get started. The important point here is that the Java classes should be built into the `module/MOD-INF/classes` sub-directory. - -Note that your extension's Java code would need to reference some libraries used in OpenRefine and OpenRefine's Java classes themselves. These dependencies are reflected in the Maven configuration for the extension. - -## Sample extension {#sample-extension} - -The sample extension is included in the code base so that you can copy it and get started on writing your own extension. After you copy it, make sure you change its name inside its `module/MOD-INF/controller.js` file. - -### Basic Structure {#basic-structure} - -The sample extension's code is in `refine/extensions/sample/`. In that directory, Java source code is contained under the `src` sub-directory, and webapp code is under the `module` sub-directory. Here is the full directory layout: - -``` -refine/extensions/sample/ - build.xml (ant build script) - src/ - com/google/refine/sampleExtension/ - ... Java source code ... - module/ - MOD-INF/ - module.properties (module settings) - controller.js (module init and routing logic in Javascript) - classes/ - ... compiled Java classes ... - lib/ - ... Java jars ... - ... velocity templates (.vt) ... - ... LESS css files ... - ... client-side files (.html, .css, .js, image files) ... -``` - -The sub-directory `MOD-INF` contains the Butterfly module's metadata and is what Butterfly looks for when it scans directories for modules. `MOD-INF` serves similar functions as `WEB-INF` in other web frameworks. - -Java code is built into the sub-directory `classes` inside `MOD-INF`, and supporting external Java jars are in the `lib` sub-directory. Those will be automatically loaded by Butterfly. (The build.xml script is wired to compile into the `classes` sub-directory.) - -Client-side code is in the inner `module` sub-directory. They can be plain old .html, .css, .js, and image files, or they can be [LESS](http://lesscss.org/) files that get processed into CSS. There are also Velocity .vt files, but they need to be routed inside `MOD-INF/controller.js`. - -`MOD-INF/controller.js` lets you configure the extension's initialization and URL routing in Javascript rather than in Java. For example, when the requested URL path is either `/` or an empty string, we process and return `MOD-INF/index.vt` ( [see http://127.0.0.1:3333/extension/sample/](http://127.0.0.1:3333/extension/sample/) if OpenRefine is running). - -The `init()` function in `controller.js` allows the extension to register various client-side handlers for augmenting pages served by Refine's core. These handlers are feature-specific. For example, [this is where the jython extension adds its parser](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/jython/module/MOD-INF/controller.js#L46). As for the sample extension, it adds its script `project-injection.js` and style `project-injection.less` into the `/project` page. If you [view the source of the /project page](http://127.0.0.1:3333/project), you will see references to those two files. - -### Wiring Up the Extension {#wiring-up-the-extension} - -The Extensions are loaded by the Butterfly framework. Butterfly refers to these as 'modules'. [The location of modules is set in the `main/webapp/butterfly.properties` file](https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/WEB-INF/butterfly.properties#L27). Butterfly simply descends into each of those paths and looks for any `MOD-INF` directories. - -For more information, see [Extension Points](https://github.com/OpenRefine/OpenRefine/wiki/Extension-Points). - -## Extension points {#extension-points} - -### Client-side: Javascript and CSS {#client-side-javascript-and-css} - -The UI in OpenRefine for working with a project is coded in [the /main/webapp/modules/core/project.vt file](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/project.vt). The file is quite small, and that's because almost all of its content is to be expanded dynamically through the Velocity variables $scriptInjection and $styleInjection. So that your own Javascript and CSS files get loaded, you need to register them with the ClientSideResourceManager, which is done in the /module/MOD-INF/controller.js file. See [the controller.js file in this sample extension code](http://github.com/OpenRefine/OpenRefine/blob/master/extensions/sample/module/MOD-INF/controller.js) for an example. - -In the registration call, the variable `module` is already available to your code by default, and it refers to your own extension. - -``` -ClientSideResourceManager.addPaths( - "project/scripts", - module, - [ - "scripts/foo.js", - "scripts/subdir/bar.js" - ] - ); -``` - -You can specify one or more files for registration, and their paths are relative to the `module` subdirectory of your extension. They are included in the order listed. - -Javascript Bundling: Note that `project.vt` belongs to the core module and is thus under the control of the core module's [controller.js file](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js). The Javascript files to be included in `project.vt` are by default bundled together for performance. When debugging, you can prevent this bundling behavior by setting `bundle` to `false` near the top of that `controller.js` file. (If you have commit access to this code base, be sure not to check that change in.) - -### Client-side: Images {#client-side-images} - -We recommend that you always refer to images through your CSS files rather than in your Javascript code. URLs to images will thus be relative to your CSS files, e.g., - -``` -.foo { - background: url(../images/x.png); - } -``` - -If you really really absolutely need to refer to your images in your Javascript code, then look up your extension's URL path in the global Javascript variable `ModuleWirings`: - -``` -ModuleWirings["my-extension"] + "images/x.png" -``` - -### Client-side: HTML Templates {#client-side-html-templates} - -Beside Javascript, CSS, and images, your extension might also include HTML templates that get loaded on the fly by your Javascript code and injected into the page's DOM. For example, here is [the Cluster edit dialog template](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/dialogs/clustering-dialog.html), which gets loaded by code in [the equivalent javascript file 'clustering-dialog.js'](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/dialogs/clustering-dialog.js): - -``` -var dialog = $(DOM.loadHTML("core", "scripts/dialogs/clustering-dialog.html")); -``` - -`DOM.loadHTML` returns the content of the file as a string, and `$(...)` turns it into a DOM fragment. Where `"core"` is, you would want your extension's name. The path of the HTML file is relative to your extension's `module` subdirectory. - -### Client-side: Project UI Extension Points {#client-side-project-ui-extension-points} - -Getting your extension's Javascript code included in `project.vt` doesn't accomplish much by itself unless your code also registers hooks into the UI. For example, you can surely implement an exporter in Javascript, but unless you add a corresponding menu command in the UI, your user can't use your exporter. - -#### Main Menu {#main-menu} - -The main menu can be extended by calling any one of the methods `MenuBar.appendTo`, `MenuBar.insertBefore`, and `MenuBar.insertAfter`. Each method takes 2 arguments: an array of strings that identify a particular existing menu item or submenu, and one new single menu item or submenu or an array of menu items and submenus. For example, to insert 2 menu items and a menu separator before the menu item Project > Export Filtered Rows > Templating..., write this Javascript code wherever that would execute when your Javascript files get loaded: - -``` -MenuBar.insertBefore( - ["core/project", "core/export", "core/export-templating"], - [ - { - "label":"Menu item 1", - "click": function() { ... } - }, - { - "label":"Menu item 2", - "click": function() { ... } - }, - {} // separator - ] - ); -``` - -The array `["core/project", "core/export", "core/export-templating"]` pinpoints the reference menu item. - -See the beginning of [/main/webapp/modules/core/scripts/project/menu-bar.js](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/project/menu-bar.js) for IDs of menu items and submenus. - -#### Column Header Menu {#column-header-menu} - -The drop-down menu of each column can also be extended, but the mechanism is slightly different compared to the main menu. Because the drop-down menu for a particular column is constructed on the fly when the user actually clicks the drop-down menu button, extending the column header menu can't really be done once at start-up time, but must be done every time a column header menu gets created. So, registration in this case involves providing a function that gets called each such time: - -``` -DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { ... do stuff to menu ... }); -``` - -That function takes in the column object (which contains the column's name), the column header UI object (generally not so useful), and the menu to extend. In the previous code line where it says "do stuff to menu", you can write something like this: - -``` -MenuSystem.appendTo(menu, ["core/facet"], [ - { - id: "core/text-facet", - label: "My Facet on " + column.name, - click: function() { - ... use column.name and do something ... - } - }, - ]); -``` - -In addition to `MenuSystem.appendTo`, you can also call `MenuSystem.insertBefore` and `MenuSystem.insertAfter` which the same 3 arguments. To see what IDs you can use, see the function `DataTableColumnHeaderUI.prototype._createMenuForColumnHeader` in [/main/webapp/modules/core/scripts/views/data-table/column-header-ui.js](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/views/data-table/column-header-ui.js). - -### Server-side: Ajax Commands {#server-side-ajax-commands} - -The client-side of OpenRefine gets things done by calling AJAX commands on the server-side. These commands must be registered with the OpenRefine servlet, so that the servlet knows how to route AJAX calls from the client-side. This can be done inside the `init` function in your extension's `controller.js` file, e.g., - -``` -function init() { - var RefineServlet = Packages.com.google.refine.RefineServlet; - RefineServlet.registerCommand(module, "my-command", new Packages.com.foo.bar.MyCommand()); - } -``` - -Your command will then be accessible at [http://127.0.0.1:3333/command/my-extension/my-command](http://127.0.0.1:3333/command/my-extension/my-command). - -### Server-side: Operations {#server-side-operations} - -Most commands change the project's data. Most of them do so by creating abstract operations. See the Changes, History, Processes, and Operations section of the [Server Side Architecture](https://github.com/OpenRefine/OpenRefine/wiki/Server-Side-Architecture) document. - -You can register an operation **class** in the `init` function as follows: - -``` -Packages.com.google.refine.operations.OperationRegistry.registerOperation( - module, - "operation-name", - Packages.com.foo.bar.MyOperation - ); -``` - -Do not call `new` to construct an operation instance. You must register the class itself. The class should have a static function for reconstructing an operation instance from a JSON blob: - -``` -static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - ... - } -``` - -### Server-side: GREL {#server-side-grel} - -GREL can be extended with new functions. This is also done in the `init` function in `controller.js`, e.g., - -``` -Packages.com.google.refine.grel.ControlFunctionRegistry.registerFunction( - "functionName", new Packages.com.foo.bar.TheFunctionClass()); -``` - -You might also want to provide new variables (beyond just `value`, `cells`, `row`, etc.) available to expressions. This is done by registering a binder that implements the interface `com.google.refine.expr.Binder`: - -``` -Packages.com.google.refine.expr.ExpressionUtils.registerBinder( - new Packages.com.foo.bar.MyBinder()); -``` - -### Server-side: Importers {#server-side-importers} - -You can register an importer as follows: - -``` -Packages.com.google.refine.importers.ImporterRegistry.registerImporter( - "importer-name", new Packages.com.foo.bar.MyImporter()); -``` - -The string `"importer-name"` isn't important at all. It's not really related to file extension or mime-type. Just use something unique. Your importer will be explicitly called to test if it can import something. - -### Server-side: Exporters {#server-side-exporters} - -You can register an exporter as follows: - -``` -Packages.com.google.refine.exporters.ExporterRegistry.registerExporter( - "exporter-name", new Packages.com.foo.bar.MyExporter()); -``` - -The string `"exporter-name"` isn't important at all. It's only used by the client-side to tell the server-side which exporter to use. Just use something unique and, of course, relevant. - -### Server-side: Overlay Models {#server-side-overlay-models} - -Overlay models are objects attached onto a core Project object to store and manage additional data for that project. For example, the schema alignment skeleton is managed by the Protograph overlay model. An overlay model implements the interface `com.google.refine.model.OverlayModel` and can be registered like so: - -``` -Packages.com.google.refine.model.Project.registerOverlayModel( - "model-name", - Packages.com.foo.bar.MyOverlayModel); -``` - -Note that you register the **class** , not an instance. The class should implement the following static method for reconstructing an overlay model instance from a JSON blob: - -``` -static public OverlayModel reconstruct(JSONObject o) throws JSONException { - ... - } -``` - -When the project gets saved, the overlay model instance's `write` method will be called: - -``` -public void write(JSONWriter writer, Properties options) throws JSONException { - ... - } -``` - -### Server-side: Scripting Languages {#server-side-scripting-languages} - -A scripting language (such as Jython) can be registered as follows: - -``` -Packages.com.google.refine.expr.MetaParser.registerLanguageParser( - "jython", - "Jython", - Packages.com.google.refine.jython.JythonEvaluable.createParser(), - "return value" - ); -``` - -The first string is the prefix that gets prepended to each expression so that we know which language the expression is in. This should be short, unique, and identifying. The second string is a user-friendly name of the language. The third is an object that implements the interface `com.google.refine.expr.LanguageSpecificParser`. The final string is the default expression in that language that would return the cell's value. - -In 2018 we are making important changes to OpenRefine to modernize it, for the benefit of users and contributors. This page describes the changes that impact developers of extensions or forks and is intended to minimize the effort required on their end to follow the transition. The instructions are written specifically with extension maintainers in mind, but fork maintainers should also find it useful. - -This document describes the migrations in the order they are committed to the master branch. This means that it should be possible to perform each migration in turn, with the ability to run the software between each stage by checking out the appropriate git commit. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/index.md b/OpenRefine/docs/versioned_docs/version-3.5/index.md deleted file mode 100644 index 367347fef..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/index.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -slug: / -id: index -title: OpenRefine user manual -sidebar_label: Introduction ---- - - -This manual is designed to comprehensively walk through every aspect of setting up and using OpenRefine 3.5.0, including every interface function and feature. - - - -This user manual starts with instructions for [installing or upgrading OpenRefine on Windows, Mac, and Linux computers](manual/installing). It then walks you through [the interface and how to run OpenRefine](manual/running#jvm-preferences) from a program or command line, with or without setting custom preferences and modifications. - -The manual then teaches you how to [start a project](manual/starting) by importing an existing dataset. We work through how to [view and learn about your data](manual/exploring) using facets, filters, and sorting. - -Then we launch into [transforming that data permanently](manual/transforming) through common and custom transformations, clustering, pulling data from the web, [reconciling](manual/reconciling), and [writing expressions](manual/expressions). - -Finally we discuss what to do with your improved dataset, whether [exporting](manual/exporting) it to a file or [uploading statements to Wikidata or another Wikibase instance](manual/wikibase/overview). - -If you're stuck on any aspect and can't find an answer in the manual, try the [Troubleshooting page](manual/troubleshooting) for links to various places to find help. - -If you are new and want to learn how to use OpenRefine using an example dataset, you may wish to start with a user-contributed tutorial from our [recommendations list](https://github.com/OpenRefine/OpenRefine/wiki/External-Resources). diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/cellediting.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/cellediting.md deleted file mode 100644 index 103c8ed45..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/cellediting.md +++ /dev/null @@ -1,165 +0,0 @@ ---- -id: cellediting -title: Cell editing -sidebar_label: Cell editing ---- -## Overview {#overview} - -OpenRefine offers a number of features to edit and improve the contents of cells automatically and efficiently. - -One way of doing this is editing through a [text facet](facets#text-facet). Once you have created a facet on a column, hover over the displayed results in the sidebar. Click on the small “edit” button that appears to the right of the facet, and type in a new value. This will apply to all the cells in the facet. - -You can apply a text facet on numbers, boolean values, and dates, but if you edit a value it will be converted into the text [data type](exploring#data-types) (regardless of whether you edit a date into another correctly-formatted date, or a “true” value into “false”, etc.). - -## Transform {#transform} - -Select Edit cellsTransform... to open up an expressions window. From here, you can apply [expressions](expressions) to your data. The simplest examples are GREL functions such as [`toUppercase()`](grelfunctions#touppercases) or [`toLowercase()`](grelfunctions#tolowercases), used in expressions as `toUppercase(value)` or `toLowercase(value)`. When used on a column operation, `value` is the information in each cell in the selected column. - -Use the preview to ensure your data is being transformed correctly. - -You can also switch to the Undo / Redo tab inside the expressions window to reuse expressions you’ve already attempted in this project, whether they have been undone or not. - -OpenRefine offers you some frequently-used transformations in the next menu option, Common transforms. For more custom transforms, read up on [expressions](expressions). - -## Common transforms {#common-transforms} - -### Trim leading and trailing whitespace {#trim-leading-and-trailing-whitespace} - -Often cell contents that should be identical, and look identical, are different because of space or line-break characters that are invisible to users. This function will get rid of any characters that sit before or after visible text characters. - -### Collapse consecutive whitespace {#collapse-consecutive-whitespace} - -You may find that some text cells contain what look like spaces but are actually tabs, or contain multiple spaces in a row. This function will remove all space characters that sit in sequence and replace them with a single space. - -### Unescape HTML {#unescape-html} - -Your data may come from an HTML-formatted source that expresses some characters through references (such as “&nbsp;” for a space, or “%u0107” for a ć) instead of the actual Unicode characters. You can use the “unescape HTML entities” transform to look for these codes and replace them with the characters they represent. For other formatting that needs to be escaped, try a custom transformation with [`escape()`](grelfunctions#escapes-s-mode). - -### Replace smart quotes with ASCII {#replace-smart-quotes-with-ascii} - -Smart quotes (or curly quotes) recognize whether they come at the beginning or end of a string, and will generate an “open” quote (“) and a “close” quote (”). These characters are not ASCII-compliant (though they are UTF8-compliant) so you can use this tranform to replace them with a straight double quote character (") instead. - -### Case transforms {#case-transforms} - -You can transform an entire column of text into UPPERCASE, lowercase, or Title Case using these three options. This can be useful if you are planning to do textual analysis and wish to avoid case-sensitivity (which some functions are) causing problems in your analysis. Consider also using a [custom facet](facets#custom-text-facet) to temporarily modify cases instead of this permanent operation if appropriate. - -### Data-type transforms {#data-type-transforms} - -As detailed in [Data types](exploring#data-types), OpenRefine recognizes different data types: string, number, boolean, and date. When you use these transforms, OpenRefine will check to see if the given values can be converted, then both transform the data in the cells (such as “3” as a text string to “3” as a number) and convert the data type on each successfully transformed cell. Cells that cannot be transformed will output the original value and maintain their original data type. - -:::caution -Be aware that dates may require manual intervention to transform successfully: see the section on [Dates](exploring#dates) for more information. -::: - -Because these common transforms do not offer the ability to output an error instead of the original cell contents, be careful to look for unconverted and untransformed values. You will see a yellow alert at the top of screen that will tell you how many cells were converted - if this number does not match your current row set, you will need to look for and manually correct the remaining cells. Also consider faceting by data type, with the GREL function [`type()`](grelfunctions#typeo). - -You can also convert cells into null values or empty strings. This can be useful if you wish to, for example, erase duplicates that you have identified and are analyzing as a subset. - -## Fill down and blank down {#fill-down-and-blank-down} - -Fill down and blank down are two functions most frequently used when encountering data organized into [records](exploring#row-types-rows-vs-records) - that is, multiple rows associated with one specific entity. - -If you receive information in rows mode and want to convert it to records mode, the easiest way is to sort your first column by the value that you want to use as a unique records key, [make that sorting permanent](transforming#edit-rows), then blank down all the duplicates in that column. OpenRefine will retain the first unique value and erase the rest. Then you can switch from “Show as rows” to “Show as records” and OpenRefine will associate rows to each other based on the remaining values in the first column. - -Be careful that your data is sorted properly before you begin blanking down - not just the first column but other columns you may want to have in a certain order. For example, you may have multiple identical entries in the first column, one with a value in the second column and one with an empty cell in the second column. In this case you want the row with the second-column value to come first, so that you can clean up empty rows later, once you blank down. - -If, conversely, you’ve received data with empty cells because it was already in something akin to records mode, you can fill down information to the rest of the rows. This will duplicate whatever value exists in the topmost cell with a value: if the first row in the record is blank, it will take information from the next cell, or the cell after that, until it finds a value. The blank cells above this will remain blank. - -## Split multi-valued cells {#split-multi-valued-cells} - -Splitting cells with more than one value in them is a common way to get your data from single rows into [multi-row records](exploring#rows-vs-records). Survey data, for example, frequently allows respondents to “Select all that apply,” or an inventory list might have items filed under more than one category. - -You can split a column based on any character or series of characters you input, such as a semi-colon (;) or a slash (/). The default is a comma. Splitting based on a separator will remove the separator characters, so you may wish to include a space with your separator (; ) if it exists in your data. - -You can use [expressions](expressions) to design the point at which a cell should split itself into two or more rows. This can be used to identify special characters or create more advanced evaluations. You can split on a line-break by entering `\n` and checking the “[regular expression](expressions#regular-expressions)” checkbox. - -Regular expressions can be useful if the split is not straightforward: say, if a capital letter (`[A-Z]`) indicates the beginning of a new string, or if you need to _not_ always split on a character that appears in both the strings and as a separator. Remember that this will remove all the matching characters. - -You can also split based on the lengths of the strings you expect to find. This can be useful if you have predictable data in the cells: for example, a 10-digit phone number, followed by a space, followed by another 10-digit phone number. Any characters past the explicit length you’ve specified will be discarded: if you split by “11, 10” any characters that may come after the 21st character will disappear. If some cells only have one phone number, you will end up with blank rows. - -If you have data that should be split into multiple columns instead of multiple rows, see [Split into several columns](columnediting#split-into-several-columns). - -## Join multi-valued cells {#join-multi-valued-cells} - -Joining will reverse the “split multi-valued cells” operation, or join up information from multiple rows into one row. All the strings will be compressed into the topmost cell in the record, in the order they appear. A window will appear where you can set the separator; the default is a comma and a space (, ). This separator is optional. We suggest the separator | as a sufficiently rare character. - -## Cluster and edit {#cluster-and-edit} - -Creating a facet on a column is a great way to look for inconsistencies in your data; clustering is a great way to fix those inconsistencies. Clustering uses a variety of comparison methods to find text entries that are similar but not exact, then shares those results with you so that you can merge the cells that should match. Where editing a single cell or text facet at a time can be time-consuming and difficult, clustering is quick and streamlined. - -Clustering always requires the user to approve each suggested edit - it will display values it thinks are variations on the same thing, and you can select which version to keep and apply across all the matching cells (or type in your own version). - -OpenRefine will do a number of cleanup operations behind the scenes in order to do its analysis, but only the merges you approve will modify your data. Understanding those different behind-the-scenes cleanups can help you choose which clustering method will be more accurate and effective. - -You can start the process in two ways: using the dropdown menu on your column, select Edit cellsCluster and edit…; or create a text facet and then press the “Cluster” button that appears in the facet box. - -![A screenshot of the Clustering window.](/img/cluster.png) - -The clustering pop-up window will take a small amount of time to analyze your column, and then make some suggestions based on the clustering method currently active. - -For each cluster identified, you can pick one of the existing values to apply to all cells, or manually type in a new value in the text box. And, of course, you can choose not to cluster them at all. OpenRefine will keep analyzing every time you make a change, with Merge selected & re-cluster, and you can work through all the methods this way. - -You can also export the currently identified clusters as a JSON file, or close the window with or without applying your changes. You can also use the histograms on the right to narrow down to, for example, clusters with lots of matching rows, or clusters of long or short values. - -### Clustering methods {#clustering-methods} - -You don’t need to understand the details behind each clustering method to apply them successfully to your data. The order in which these methods are presented in the interface and on this page is the order we recommend - starting with the most strict rules and moving to the most lax, which require more human supervision to apply correctly. - -The clustering pop-up window offers you a variety of clustering methods: - -* key collision - * fingerprint - * ngram-fingerprint - * metaphone3 - * cologne-phonetic - * Daitch-Mokotoff - * Beider-Morse -* nearest neighbor - * levenshtein - * ppm - -#### Key collision {#key-collision} - -**Key collisions** are very fast and can process millions of cells in seconds: - -**Fingerprinting** is the least likely to produce false positives, so it’s a good place to start. It does the same kind of data-cleaning behind the scenes that you might think to do manually: fix whitespace into single spaces, put all uppercase letters into lowercase, discard punctuation, remove diacritics (e.g. accents) from characters, split up all strings (words) and sort them alphabetically (so “Zhenyi, Wang” becomes “wang zhenyi”). - -**N-gram fingerprinting** allows you to set the _n_ value to whatever number you’d like, and will create n-grams of _n_ size (after doing some cleaning), alphabetize them, then join them back together into a fingerprint. For example, a 1-gram fingerprint will simply organize all the letters in the cell into alphabetical order - by creating segments one character in length. A 2-gram fingerprint will find all the two-character segments, remove duplicates, alphabetize them, and join them back together (for example, “banana” generates “ba an na an na,” which becomes “anbana”). - -This can help match cells that have typos, or incorrect spaces (such as matching “lookout” and “look out,” which fingerprinting itself won’t identify because it separates words). The higher the _n_ value, the fewer clusters will be identified. With 1-grams, keep an eye out for mismatched values that are near-anagrams of each other (such as “Wellington” and “Elgin Town”). - -##### Phonetic clustering {#phonetic-clustering} - -The next four methods are phonetic algorithms: they identify letters that sound the same when pronounced out loud, and assess text values based on that (such as knowing that a word with an “S” might be a mistype of a word with a “Z”). They are great for spotting mistakes made by not knowing the spelling of a word or name after hearing it spoken aloud. - -**Metaphone3 fingerprinting** is an English-language phonetic algorithm. For example, “Reuben Gevorkiantz” and “Ruben Gevorkyants” share the same phonetic fingerprint in English. - -**Cologne fingerprinting** is another phonetic algorithm, but for German pronunciation. - -**Daitch-Mokotoff** is a phonetic algorithm for Slavic and Yiddish words, especially names. **Baider-Morse** is a version of Daitch-Mokotoff that is slightly more strict. - -Regardless of the language of your data, applying each of them might find different potential matches: for example, Metaphone clusters “Cornwall” and “Corn Hill” and “Green Hill,” while Cologne clusters “Greenvale” and “Granville” and “Cornwall” and “Green Wall.” - -#### Nearest neighbor {#nearest-neighbor} - -**Nearest neighbor** clustering methods are slower than key collision methods. They allow the user to set a radius - a threshold for matching or not matching. OpenRefine uses a “blocking” method first, which sorts values based on whether they have a certain amount of similarity (the default is “6” for a six-character string of identical characters) and then runs the nearest-neighbor operations on those sorted groups. - -We recommend setting the block number to at least 3, and then increasing it if you need to be more strict (for example, if every value with “river” is being matched, you should increase it to 6 or more). Note that bigger block values will take much longer to process, while smaller blocks may miss matches. Increasing the radius will make the matches more lax, as bigger differences will be clustered. - -**Levenshtein distance** counts the number of edits required to make one value perfectly match another. As in the key collision methods above, it will do things like change uppercase to lowercase, fix whitespace, change special characters, etc. Each character that gets changed counts as 1 “distance.” “New York” and “newyork” have an edit distance value of 3 (“N” to “n”; “Y” to “y”; remove the space). It can do relatively advanced edits, such as understand the distance between “M. Makeba” and “Miriam Makeba” (5), but it may create false positives if these distances are greater than other, simpler transformations (such as the one-character distance to “B. Makeba,” another person entirely). - -**PPM (Prediction by Partial Matching)** uses compression to see whether two values are similar or different. In practice, this method is very lax even for small radius values and tends to generate many false positives, but because it operates at a sub-character level it is capable of finding substructures that are not easily identifiable by distances that work at the character level. So it should be used as a “last resort” clustering method. It is also more effective on longer strings than on shorter ones. - -For more of the theory behind clustering, see [Clustering In Depth](https://github.com/OpenRefine/OpenRefine/wiki/Clustering-In-Depth). - -## Replace {#replace} - -OpenRefine provides a find/replace function for you to edit your data. Selecting Edit cellsReplace will bring up a simple window where you can input a string to search and a string to replace it with. You can set case-sensitivity, and set it to only select whole words, defined by a string with spaces or punctuation around it (to prevent, for example, “house” selecting the “house” part of “doghouse”). You can use [regular expressions](expressions#regular-expressions) in this field. You may wish to preview the results of this operation by testing it with a [Text filter](facets#text-filter) first. - -You can also perform a sort of find/replace operation by editing one cell, and selecting “apply to all identical cells.” - -## Edit one cell at a time {#edit-one-cell-at-a-time} - -You can edit individual cells by hovering your mouse over that cell. You should see a tiny blue link labeled “edit.” Click it to edit the cell. That pops up a window with a bigger text field for you to edit. You can change the [data type](exploring#data-types) of that cell, and you can apply these changes to all identical cells (in the same column), using this pop-up window. - -You will likely want to avoid doing this except in rare cases - the more efficient means of improving your data will be through automated and bulk operations. \ No newline at end of file diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/columnediting.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/columnediting.md deleted file mode 100644 index 754a61c62..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/columnediting.md +++ /dev/null @@ -1,124 +0,0 @@ ---- -id: columnediting -title: Column editing -sidebar_label: Column editing ---- - -## Overview {#overview} - -Column editing contains some of the most powerful data-improvement methods in OpenRefine. The operations in the Edit column menu involve using one column of data to add entirely new columns and fields to your dataset. - -## Splitting or joining {#splitting-or-joining} - -Many users find that they frequently need to make their data more granular: for example, splitting a “Firstname Lastname” column into two columns, one for first names and one for last names. The reverse is also often true: you may have several columns of category values that you want to join into one “category” column. -. -### Split into several columns {#split-into-several-columns} - -![A screenshot of the settings window for splitting columns.](/img/columnsplit.png) - -You can find this operation at Edit columnSplit into several columns.... Splitting one column into several columns requires you to identify the character, string lengths, or evaluating expression you want to split on. Just like [splitting multi-valued cells into rows](cellediting#split-multi-valued-cells), splitting cells into multiple columns will remove the separator character or string you indicate. Splitting by lengths will discard any information that comes after the specified total length. - -You can also specify a maximum number of new columns to be made: separator characters after this limit will be ignored, and the remaining characters will end up in the last column. - -New columns will be named after the original column, with a number: “Location 1,” “Location 2,” etc. You can choose to remove the original column with this operation, and you can have [data types](exploring#data-types) identified where possible. This function will work best with converting strings to numbers, and may not work with [dates](exploring#dates). - -### Join columns {#join-columns} - -![A screenshot of the settings window for joining columns.](/img/columnjoin.png) - -You can join columns by selecting Edit columnJoin columns.... All the columns currently in your dataset will appear in the pop-up window. You can select or un-select all the columns you want to join, and drag columns to put them in the order you want to join them in. You will define a separator character (optional) and define a string to insert into empty cells (nulls). - -The joined data will appear in the column you originally selected, or you can create a new column for this content and specify a name. You can delete all the columns that were used in this join operation. - -## Add column based on this column {#add-column-based-on-this-column} - -Selecting Edit columnAdd column based on this column... will open up an [expressions](expressions) window where you can transform the data from this column (using `value`), or write a more complex expression that takes information from any number of columns or from external sources. - -Expressions used in this operation will rely on your knowledge of variables. You can learn more in the [Expressions section on variables](expressions#variables). - -The simplest way to use this operation is simply leave the default `value` in the expression field, to create an exact copy of your column. For a column of [reconciled data](reconciling), you can use the variable `cell` instead, to copy both the original string and the existing reconciliation data. This will include matched values, candidates, and new items. - -One useful expression is to create a column based on concatenating (merging) two other columns. Select either of the source columns, choose Edit columnAdd column based on this column..., name your new column, and use the following format in the expression window: - -``` -cells["Column 1"].value + cells["Column 2"].value -``` - -If your column names do not contain spaces, you can use the following format instead: - -``` -cells.Column1.value + cells.Column2.value -``` - -If you are in records mode instead of rows mode, you can concatenate using the following format: - -``` -row.record.cells.Column1.value + row.record.cells.Column2.value -``` - -You may wish to add separators or spaces, or modify your input during this operation with more advanced expressions. - -## Add column by fetching URLs {#add-column-by-fetching-urls} - -Through the Add column by fetching URLs function, OpenRefine supports the ability to fetch HTML or data from web pages or services. In this operation you will be building URL strings based on your column of data, by using `value` to insert a relevant substring. Your chosen column needs to contains parts of paths to valid HTML pages or files online. - -If you have a column of URLs and want to fetch the information that they point to, you can simply run the expression as `value`. If your column has, for example, unique identifiers for Wikidata entities (numerical values starting with Q), you can download the JSON-formatted metadata about each entity with - -``` -"https://www.wikidata.org/wiki/Special:EntityData/" + value + ".json" -``` - -or whatever metadata format you prefer. Information about the format options in Wikidata can be found [here](https://www.wikidata.org/wiki/Wikidata:Data_access). The service you are fetching data from may have similar documentation on its provided options. - -![A screenshot of the settings window for fetching URLs.](/img/fetchingURLs.png) - -This service is more useful when getting metadata files instead of HTML, but you may wish to work with a page’s entire HTML contents and then parse out information from that. - -:::caution -Be aware that the fetching process can take quite some time and that servers may not want to fulfill hundreds or thousands of page requests in seconds. Fetching allows you to set a “throttle delay” which determines the amount of time between requests. The default is 5 seconds per row in your dataset (5000 milliseconds). We recommend leaving this at 1000 or greater. -::: - -Note the following: -* Before pressing “OK,” copy and paste a URL or two from the preview and test them in another browser tab to make sure they work. -* In some situations you may need to set [HTTP request headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers). To set these, click the small “Show” button next to “HTTP headers to be used when fetching URLs” in the settings window. The authorization credentials get logged in your operation history in plain text, which may be a security concern for you. You can set the following request headers: - * [User-Agent](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent) - * [Accept](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept) - * [Authorization](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization) - -### Common errors {#common-errors} - -When OpenRefine attempts to fetch information from a web service, it can fail in a variety of ways. The following information is meant to help troubleshoot and fix problems encountered when using this function. - -First, make sure that your fetching operation is storing errors (check “store error”). Then run the fetch and look at the error messages. - -**“HTTP error 403 : Forbidden”** can be simply down to you not having access to the URL you are trying to use. If you can access the same URL with your browser, the remote site may be blocking OpenRefine because it doesn't recognize its request as valid. Changing the [User-Agent](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent) request header may help. If you believe you should have access to a site but are “forbidden,” you may wish to contract the administrators. - -**“HTTP error 404 : Not Found”** indicates that the information you are requesting does not exist, perhaps due to a problem with your cell values if it only happening in certain rows. - -**“HTTP error 500 : Internal Server Error”** indicates the remote server is having a problem filling your request. You may wish to simply wait and try again later, or double-check the URLs. - -**“error: javax.net.ssl.SSLHandshakeException: Received fatal alert: handshake_failure”** can occur when you are trying to retrieve information over HTTPS but the remote site is using an encryption not supported by the Java virtual machine being used by OpenRefine. - -You can check which encryption methods are supported by your OpenRefine/Java installation by using a service such as **How's my SSL**. Add the URL `https://www.howsmyssl.com/a/check` to an OpenRefine cell and run “Add column by fetching URLs” on it, which will provide a description of the SSL client being used. - -You can try installing additional encryption supports by installing the [Java Cryptography Extension](https://www.oracle.com/java/technologies/javase-jce8-downloads.html). -Note that for Mac users and for Windows users with the OpenRefine installation with bundled JRE, these updated cipher suites need to be dropped into the Java install within the OpenRefine application: - -* On Mac, it will look something like `/Applications/OpenRefine.app/Contents/PlugIns/jdk1.8.0_60.jdk/Contents/Home/jre/lib/security`. -* On Windows: `\server\target\jre\lib\security`. - -**“javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed”** can appear when the remote site is using an HTTPS certificate not trusted by your local Java installation. You will need to make sure that the certificate, or (more likely) the root certificate, is trusted. - -The list of trusted certificates is stored in an encrypted file called `cacerts` in your local Java installation. This can be read and updated by a tool called “keytool.” You can find directions on how to add a security certificate to the list of trusted certificates for a Java installation [here](http://magicmonster.com/kb/prg/java/ssl/pkix_path_building_failed.html) and [here](http://javarevisited.blogspot.co.uk/2012/03/add-list-certficates-java-keystore.html). - -Note that for Mac users and for Windows users with the OpenRefine installation with bundled JRE, the `cacerts` file within the OpenRefine application needs to be updated. - -* On Mac, it will look something like `/Applications/OpenRefine.app/Contents/PlugIns/jdk1.8.0_60.jdk/Contents/Home/jre/lib/security/cacerts`. -* On Windows: `\server\target\jre\lib\security\`. - -## Renaming, removing, and moving {#renaming-removing-and-moving} - -Every column's Edit column dropdown contains options to move it (to the beginning, end, left, or right), rename it, and delete it. -These operations can be undone, but a removed column cannot be restored later if you keep modifying your data. If you wish to temporarily hide a column, go to [View](sortview#view)Collapse this column instead. - -Be cautious about moving columns in [records mode](cellediting#rows-vs-records): if you change the first column in your dataset (the key column), your records may change in unintended ways. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/exploring.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/exploring.md deleted file mode 100644 index 23bafc1a7..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/exploring.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -id: exploring -title: Exploring data -sidebar_label: Overview ---- - -## Overview {#overview} - -OpenRefine offers lots of features to help you learn about your dataset, even if you don’t change a single character. In this section we cover different ways for sorting through, filtering, and viewing your data. - -Unlike spreadsheets, OpenRefine doesn’t store formulas and display the output of those calculations; it only shows the value inside each cell. It doesn’t support cell colors or text formatting. - -## Data types {#data-types} - -Each piece of information (each cell) in OpenRefine is assigned a data type. Some file formats, when imported, can set data types that are recognized by OpenRefine. Cells without an associated data type on import will be considered a “string” at first, but you can have OpenRefine convert cell contents into other data types later. This is set at the cell level, not at the column level. - -You can see data types in action when you preview a new project: check the box next to Attempt to parse cell text into numbers, and cells will be converted to the “number” data type based on their contents. You’ll see numbers change from black text to green if they are recognized. - -The data type will determine what you can do with the value. For example, if you want to add two values together, they must both be recognized as the number type. - -You can check data types at any time by: -* clicking “edit” on a single cell (where you can also edit the type) -* creating a Custom Text Facet on a column, and inserting `type(value)` into the Expression field. This will generate the data type in the preview, and you can facet by data type if you press OK. - -The data types supported are: -* string (one or more text characters) -* number (one or more characters of numbers only) -* boolean (values of “true” or “false”) -* [date](#dates) (ISO-8601-compliant extended format with time in UTC: YYYY-MM-DDTHH:MM:SSZ) - -OpenRefine recognizes two further data types as a result of its own processes: -* error -* null - -An “error” data type is created when the cell is storing an error generated during a transformation in OpenRefine. - -A “null” data type is a special type that means “this cell has no value.” It’s distinct from cells that have values such as “0” or “false”, or cells that look empty but have whitespace in them, or cells that contain empty strings. When you use `type(value)`, it will show you that the cell’s value is “null” and its type is “undefined.” You can opt to [show “null” values](sortview#showhide-null), by going to AllViewShow/Hide ‘null’ values in cells. - -Changing a cell's data type is not the same operation as transforming its contents. For example, using a column-wide transform such as TransformCommon transformsTo date may not convert all values successfully, but going to an individual cell, clicking “edit”, and changing the data type can successfully convert text to a date. These operations use different underlying code. Learn more about date formatting and transformations in the next section. - -To transform data from one type to another, see [Transforming data](cellediting#data-type-transforms) for information on using common tranforms, and see [Expressions](expressions) for information on using [toString()](grelfunctions#tostringo-string-format-optional), [toDate()](grelfunctions#todateo-b-monthfirst-s-format1-s-format2-), and other functions. - - -### Dates {#dates} - -A “date” type is created when a column is [transformed into dates](transforming#to-date), when an expression is used to [convert cells to dates](grelfunctions#todateo-b-monthfirst-s-format1-s-format2-) or when individual cells are set to have the data type “date”. - -Date-formatted data in OpenRefine relies on a number of conversion tools and standards. For something to be considered a date in OpenRefine, it will be converted into the ISO-8601-compliant extended format with time in UTC: YYYY-MM-DDTHH:MM:SSZ. - -When you run Edit cellsCommon transformsTo date, the following column of strings on the left will transform into the values on the right: - -|Input|→|Output| -|---|---|---| -|23/12/2019|→|2019-12-23T00:00:00Z| -|14-10-2015|→|2015-10-14T00:00:00Z| -|2012 02 16|→|2012-02-16T00:00:00Z| -|August 2nd 1964|→|1964-08-02T00:00:00Z| -|today|→|today| -|never|→|never| - -OpenRefine uses a variety of tools to recognize, convert, and format [dates](exploring#dates) and so some of the values above can be reformatted using other methods. In this case, clicking the “today” cell and editing its data type manually will convert “today” into a value such as “2020-08-14T00:00:00Z”. Attempting the same data-type change on “never” will give you an error message and refuse to proceed. - -You can do more precise conversion and formatting using expressions and arguments based on the state of your data: see the GREL functions reference section on [Date functions](grelfunctions#date-functions) for more help. - -You can convert dates into a more human-readable format when you [export your data using the custom tabular exporter](exporting#custom-tabular-exporter). You are given the option to keep your dates in the ISO 8601 format, to output short, medium, long, or full locale formats, or to specify a custom format. This means that you can format your dates into, for example, MM/DD/YY (the US short standard) with or without including the time, after working with ISO-8601-formatted dates in your project. - -The following table shows some example [date and time formatting styles for the U.S. and French locales](https://docs.oracle.com/javase/tutorial/i18n/format/dateFormat.html): - -|Style |U.S. Locale |French Locale| -|---|---|---| -|Default |Jun 30, 2009 7:03:47 AM |30 juin 2009 07:03:47| -|Short |6/30/09 7:03 AM |30/06/09 07:03| -|Medium |Jun 30, 2009 7:03:47 AM |30 juin 2009 07:03:47| -|Long |June 30, 2009 7:03:47 AM PDT |30 juin 2009 07:03:47 PDT| -|Full |Tuesday, June 30, 2009 7:03:47 AM PDT |mardi 30 juin 2009 07 h 03 PDT| - -## Rows vs. records {#rows-vs-records} - -A row is a simple way to organize data: a series of cells, one cell per column. Sometimes there are multiple pieces of information in one cell, such as when a survey respondent can select more than one response. - -In cases where there is more than one value for a single column in one or more rows, you may wish to use OpenRefine’s records mode: this defines a single record as potentially containing more than one row. From there you can transform cells into multiple rows, each cell containing one value you’d like to work with. - -Generally, when you import some data, OpenRefine reads that data in row mode. From the project screen, you can convert the project into records mode. OpenRefine remembers this action and will present you with records mode each time you open the project from then on. - -OpenRefine understands records based on the content of the first column, what we call the “key column.” Splitting a row into a multi-row record will base all association on the first column in your dataset. - -If you have more than one column to split out into multiple rows, OpenRefine will keep your data associated with its original record, and associate subgroups based on the top-most row in each group. - -You can imagine the structure as a tree with many branches, all leading back to the same trunk. - -For example, your key column may be a film or television show, with multiple cast members identified by name, associated to that work. You may have one or more roles listed for each person. The roles are linked to the actors, which are linked to the title. - -|Work|Actor|Role| -|---|---|---| -|The Wizard of Oz|Judy Garland|Dorothy Gale| -||Ray Bolger|"Hunk"| -|||The Scarecrow| -||Jack Haley|"Hickory"| -|||The Tin Man| -||Bert Lahr|"Zeke"| -|||The Cowardly Lion| -||Frank Morgan|Professor Marvel| -|||The Gatekeeper| -|||The Carriage Driver| -|||The Guard| -|||The Wizard of Oz| -||Margaret Hamilton|Miss Almira Gulch| -|||The Wicked Witch of the West| - -Once you are in records mode, you can still move some columns around, but if you move a column to the beginning, you may find your data becomes misaligned. The new key column will sort into records based on empty cells, and values in the old key column will be assigned to the last row in the old record (the key value sitting above those values). - -OpenRefine assigns a unique key behind the scenes, so your records don’t need a unique identifier in the key column. You can keep track of which rows are assigned to each record by the record number that appears under the All column. - -To [split multi-valued cells](transforming#split-multi-valued-cells) and apply other operations that take advantage of records mode, see [Transforming data](transforming). - -Be careful when in records mode that you do not accidentally delete rows based on being blank in one column where there is a value in another. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/exporting.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/exporting.md deleted file mode 100644 index 21b1516a9..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/exporting.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -id: exporting -title: Exporting your work -sidebar_label: Exporting ---- - -## Overview {#overview} - -Once your dataset is ready, you will need to get it out of OpenRefine and into the system of your choice. OpenRefine outputs a number of file formats, can upload your data directly into Google Sheets, and can create or update statements on Wikidata. - -You can also [export your full project data](#export-a-project) so that it can be opened by someone else using OpenRefine (or yourself, on another computer). - -## Export data {#export-data} - -![A screenshot of the Export dropdown.](/img/export-menu.png) - -Many of the options only export data in the current view - that is, with current filters and facets applied. Some will give you the choice to export your entire dataset or just the currently-viewed rows. - -To export data from a project, click the Export dropdown button in the top right corner and pick the format you want. Your options are: - -* Tab-separated value (TSV) or Comma-separated value (CSV) -* HTML-formatted table -* Excel spreadsheet (XLS or XLSX) -* Open Document Format (ODF) spreadsheet (ODS) -* Upload to Google Sheets (requires [Google account authorization](starting#google-sheet-from-drive)) -* [Custom tabular exporter](#custom-tabular-exporter) -* [SQL statement exporter](#sql-statement-exporter) -* [Templating exporter](#templating-exporter), which generates JSON by default - -You can also export reconciled data to Wikidata, or export your Wikidata schema for future use with other OpenRefine projects: - -* [Upload edits to Wikidata](wikibase/uploading#uploading-with-openrefine) -* [Export to QuickStatements](wikibase/uploading#uploading-with-quickstatements) (version 1) -* [Export Wikidata schema](wikibase/overview#import-and-export-schema) - -### Custom tabular exporter {#custom-tabular-exporter} - -![A screenshot of the custom tabular content tab.](/img/custom-tabular-exporter.png) - -With the custom tabular exporter, you can choose which of your data to export, the separator you wish to use, and whether you'd like to download the result to your computer or upload it into a Google Sheet. - -On the Content tab, you can drag and drop the columns appearing in the column list to reorder the output. The options for reconciled and date data are applied to each column individually. - -This exporter is especially useful with reconciled data, as you can choose whether you wish to output the cells' original values, the matched values, or the matched IDs. Ouputting “match entity's name”, “matched entity's ID”, or “cell's content” will output, respectively, the contents of `cell.recon.match.name`, `cell.recon.match.id`, and `cell.value`. - -“Output nothing for unmatched cells” will export empty cells for both newly-created matches and cells with no chosen matches. “Link to matched entity's page” will produce hyperlinked text in an HTML table output, but have no effect in other formats. - -At this time, the date-formatting options in this window do not work. You can [keep track of this issue on Github](https://github.com/OpenRefine/OpenRefine/issues/3368). -In the future, you will be able to choose how to [output date-formatted cells](exploring#dates). You can create a custom date output by using [formatting according to the SimpleDateFormat parsing key found here](grelfunctions#todateo-b-monthfirst-s-format1-s-format2-). - -![A screenshot of the custom tabular file download tab.](/img/custom-tabular-exporter2.png) - -On the Download tab, you can generate a preview of how the first ten rows of your dataset will output. If you do not choose one of the file formats on the right, the Download button will generate a text file. On the Upload tab, you can create a new Google Sheet. - -With the Option Code tab, you can copy JSON of your current custom settings to reuse on another export, or you can paste in existing JSON settings to apply to the current project. - -### SQL exporter {#sql-exporter} - -The SQL exporter creates a SQL statement containing the data you’ve exported, which you can use to overwrite or add to an existing database. Choosing ExportSQL exporter will bring up a window with two tabs: one to define what data to output, and another to modify other aspects of the SQL statement, with options to preview and download the statement. - -![A screenshot of the SQL statement content window.](/img/sql-exporter.png) - -The Content tab allows you to craft your dataset into an SQL table. From here, you can choose which columns to export, the data type to export for each (or choose "VARCHAR"), and the maximum character length for each field (if applicable based on the data type). You can set a default value for empty cells after unchecking “Allow null” in one or more columns. - -With this output tool, you can choose whether to output only currently visible rows, or all the rows in your dataset, as well as whether to include empty rows. The option to “Trim column names” will remove their whitespace characters. - -![A screenshot of the SQL statement download window.](/img/sql-exporter2.png) - -The Download tab allows you to finalize your complete SQL statement. - -Include schema means that you will start your statement with the creation of a table. Without that, you will only have an INSERT statement. - -Include content means including the INSERT statement with data from your project. Without that, you will only create empty columns. - -You can include DROP and IF EXISTS if you require them, and set a name for the table to which the statement will refer. - -You can then preview your statement, which will open up a new browser tab/window showing a statement with the first ten rows of your data (if included), or you can save a `.sql` file to your computer. - -### Templating exporter {#templating-exporter} - -If you pick Templating… from the Export dropdown menu, you can “roll your own” exporter. This is useful for formats that we don't support natively yet, or won't support. The Templating exporter generates JSON by default. - -![A screenshot of the Templating exporter generating JSON by default.](/img/templating-exporter.png) - -The Templating Export window allows you to set your own separators, prefix, and suffix to create a complete dataset in the language of your choice. In the Row template section, you can choose which columns to generate from each row by calling them with [variables](expressions#variables). - -This can be used to: -* output [reconciliation data](expressions#reconciliation), such as `cells["ColumnName"].recon.match.name` -* create multiple columns of output from different [member fields](expressions#variables) of a single project column -* employ [expressions](expressions) to modify data for output: for example, `cells["ColumnName"].value.toUppercase()`. - -Anything that appears inside doubled curly braces ({{ }}) is treated as a GREL expression; anything outside is generated as straight text. You can use Jython or Clojure by declaring it at the start: -``` -{{jython:return cells["ColumnName"].value}} -``` - -:::caution -Note that some syntax is different in this tool than elsewhere in OpenRefine: a forward slash must be escaped with a backslash, while other characters do not need escaping. You cannot, at this time, include a closing curly brace (}) anywhere in your expression, or it will cause it to malfunction. -::: - -You can include [regular expressions](expressions#regular-expressions) as usual (inside forward slashes, with any GREL function that accepts them). For example, you could output a version of your cells with punctuation removed, using an expression such as -``` -{{jsonize(cells["ColumnName"].value.replaceChars("/[.!?$&,/]/",""))}} -``` - -You could also simply output a plain-text document inserting data from your project into sentences: for example, "In `{{cells["Year"].value}}` we received `{{cells["RequestCount"].value}}` requests." - -You can use the shorthand `${ColumnName}` (no need for quotes) to insert column values directly. You cannot use this inside an expression, because of the closing curly brace. - -If your projects is in records mode, the Row separator field will insert a separator between records, rather than individual rows. Rows inside a single record will be directly appended to one another as per the content in the Row Template field. - -Once you have created your template, you may wish to save the text you produced in each field, in order to reuse it in the future. Once you click Export OpenRefine will output a simple `.txt` file, and your template will be discarded. - -We have recipes on using the Templating exporter to [produce several different formats](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#12-templating-exporter). - -## Export a project {#export-a-project} - -You can share a project in progress with another computer, a colleague, or with someone who wants to check your history. This can be useful for showing that your data cleanup didn’t distort or manipulate the information in any way. Once you have exported a project, another OpenRefine installation can [import it as a new project](starting#import-a-project). - -You can either save it locally or upload it to Google Drive (which requires you to authorize a Google account). - -:::caution -OpenRefine project archives contain confidential data from previous steps, which will still be accessible to anyone who has the archive. If you are hoping to keep your original dataset hidden for privacy reasons, such as using OpenRefine to anonymize information, do not share your project archive. -::: - -To save your project archive locally: from the Export dropdown, select OpenRefine project archive to file. OpenRefine exports your full project with all of its history. It does not export any current views or applied facets. Existing reconciliation information will be preserved, but the importing computer will need to add the same reconciliation services to keep working with that data. - -OpenRefine exports files in `.tar.gz` format. You can rename the file when you save it; otherwise it will bear the project name. - -To save your project archive to Google Drive: from the Export dropdown, select OpenRefine project archive to Google Drive.... OpenRefine will not share the link with you, only confirm that the file was uploaded. - -## Export operations {#export-operations} - -You can [save and re-apply the history of any project](running#reusing-operations) (all the operations shown in the Undo/Redo tab). This creates JSON that you can save for later reuse on another OpenRefine project. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/expressions.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/expressions.md deleted file mode 100644 index d10420378..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/expressions.md +++ /dev/null @@ -1,208 +0,0 @@ ---- -id: expressions -title: Expressions -sidebar_label: Overview ---- - -## Overview {#overview} - -You can use expressions in multiple places in OpenRefine to extend data cleanup and transformation. Expressions are available with the following functions: -* Facet: - - * Custom text facet... - * Custom numeric facet… - * Customized facets (click “change” after they have been created to bring up an expressions window) -* Edit cells: - - * Transform… - * Split multi-valued cells… - * Join multi-valued cells… -* Edit column: - - * Split - * Join - * Add column based on this column - * Add column by fetching URLs. - -In the expressions editor window you have the opportunity to select a supported language. The default is [GREL (General Refine Expression Language)](grel); OpenRefine also comes with support for [Clojure](jythonclojure#clojure) and [Jython](jythonclojure#jython). Extensions may offer support for more expressions languages. - -These languages have some syntax differences but support many of the same [variables](#variables). For example, the GREL expression `value.split(" ")[1]` would be written in Jython as `return value.split(" ")[1]`. - -This page is a general reference for available functions, variables, and syntax. For examples that use these expressions for common data tasks, look at the [Recipes section on the wiki](https://github.com/OpenRefine/OpenRefine/wiki/Documentation-For-Users#recipes-and-worked-examples). - -## Expressions {#expressions} - -There are significant differences between OpenRefine's expressions and the spreadsheet formulas you may be used to using for data manipulation. OpenRefine does not store formulas in cells and display output dynamically: OpenRefine’s transformations are one-time operations that can change column contents or generate new columns. These are applied using variables such as `value` or `cell` to perform the same modification to each cell in a column. - -Take the following example: - -|ID|Friend|Age| -|---|---|---| -|1.|John Smith|28| -|2.|Jane Doe|33| - -Were you to apply a transformation to the “friend” column with the expression - -``` - value.split(" ")[1] -``` - -OpenRefine would work through each row, splitting the “friend” values based on a space character. The `value` for row 1 is “John Smith” so the output would be “Smith” (as "[1]" selects the second part of the created output); the `value` for row 2 is “Jane Doe” so the output would be “Doe”. Using variables, a single expression yields different results for different rows. The old information would be discarded; you couldn't get "John" and "Jane" back unless you undid the operation in the [History](running#history-undoredo) tab. - -For another example, if you were to create a new column based on your data using the expression `row.starred`, it would generate a column of true and false values based on whether your rows were starred at that moment. If you were to then star more rows and unstar some rows, that data would not dynamically update - you would need to run the operation again to have current true/false values. - -Note that an expression is typically based on one particular column in the data - the column whose drop-down menu is first selected. Many variables are created to stand for things about the cell in that “base column” of the current row on which the expression is evaluated. There are also variables about rows, which you can use to access cells in other columns. - -## The expressions editor {#the-expressions-editor} - -When you select a function that accepts expressions, you will see a window overlay the screen with what we call the expressions editor. - -![The expressions editor window with a simple expression: value + 10.](/img/expression-editor.png) - -The expressions editor offers you a field for entering your formula and shows you a preview of its transformation on your first few rows of cells. - -There is a dropdown menu from which you can choose an expression language. The default at first is GREL; if you begin working with another language, that selection will persist across OpenRefine. Jython and Clojure are also offered with the installation package, and you may be able to add more language support with third-party extensions and customizations. - -There are also tabs for: -* History, which shows you formulas you’ve recently used from across all your projects -* Starred, which shows you formulas from your History that you’ve starred for reuse -* Help, a quick reference to GREL functions. - -Starring formulas you’ve used in the past can be helpful for repetitive tasks you’re performing in batches. - -You can also choose how formula errors are handled: replicate the original cell value, output an error message into the cell, or ouput a blank cell. - -## Regular expressions {#regular-expressions} - -OpenRefine offers several fields that support the use of regular expressions (regex), such as in a Text filter or a Replace… operation. GREL and other expressions can also use regular expression markup to extend their functionality. - -If this is your first time working with regex, you may wish to read [this tutorial specific to the Java syntax that OpenRefine supports](https://docs.oracle.com/javase/tutorial/essential/regex/). We also recommend this [testing and learning tool](https://regexr.com/). - -### GREL-supported regex {#grel-supported-regex} - -To write a regular expression inside a GREL expression, wrap it between a pair of forward slashes (/) much like the way you would in Javascript. For example, in - -``` -value.replace(/\s+/, " ") -``` - -the regular expression is `\s+`, and the syntax used in the expression wraps it with forward slashes (`/\s+/`). Though the regular expression syntax in OpenRefine follows that of Java (normally in Java, you would write regex as a string and escape it like "\\s+"), a regular expression within a GREL expression is similar to Javascript. - -Do not use slashes to wrap regular expressions outside of a GREL expression. - -On the [GREL functions](grelfunctions) page, functions that support regex will indicate that with a “p” for “pattern.” The GREL functions that support regex are: -* [contains](grelfunctions#containss-sub-or-p) -* [replace](grelfunctions#find-and-replace) -* [find](grelfunctions#find-and-replace) -* [match](grelfunctions#matchs-p) -* [partition](grelfunctions#partitions-s-or-p-fragment-b-omitfragment-optional) -* [rpartition](grelfunctions#rpartitions-s-or-p-fragment-b-omitfragment-optional) -* [split](grelfunctions#splits-s-or-p-sep-b-preservetokens-optional) -* [smartSplit](grelfunctions#smartsplits-s-or-p-sep-optional) - -### Jython-supported regex {#jython-supported-regex} - -You can also use [regex with Jython expressions](http://www.jython.org/docs/library/re.html), instead of GREL, for example with a Custom Text Facet: - -``` -python import re g = re.search(ur"\u2014 (.*),\s*BWV", value) return g.group(1) -``` - -### Clojure-supported regex {#clojure-supported-regex} - -[Clojure](https://clojure.org/reference/reader) uses the same regex engine as Java, and can be invoked with [re-find](http://clojure.github.io/clojure/clojure.core-api.html#clojure.core/re-find), [re-matches](http://clojure.github.io/clojure/clojure.core-api.html#clojure.core/re-matches), etc. You can use the #"pattern" reader macro as described [in the Clojure documentation](https://clojure.org/reference/other_functions#regex). For example, to get the nth element of a returned sequence, you can use the nth function: - -``` -clojure (nth (re-find #"\u2014 (.*),\s*BWV" value) 1) -``` - -## Variables {#variables} - -Most OpenRefine variables have attributes: aspects of the variables that can be called separately. We call these attributes “member fields” because they belong to certain variables. For example, you can query a record to find out how many rows it contains with `row.record.rowCount`: `rowCount` is a member field specific to the `record` variable, which is a member field of `row`. Member fields can be called using a dot separator, or with square brackets (`row["record"]`). The square bracket syntax is also used for variables that can call columns by name, for example, `cells["Postal Code"]`. - -|Variable |Meaning | -|-|-| -| `value` | The value of the cell in the current column of the current row (can be null) | -| `row` | The current row | -| `row.record` | One or more rows grouped together to form a record | -| `cells` | The cells of the current row, with fields that correspond to the column names (or row.cells) | -| `cell` | The cell in the current column of the current row, containing value and other attributes | -| `cell.recon` | The cell's reconciliation information returned from a reconciliation service or provider | -| `rowIndex` | The index value of the current row (the first row is 0) | -| `columnName` | The name of the current cell's column, as a string | - -### Row {#row} - -The `row` variable itself is best used to access its member fields, which you can do using either a dot operator or square brackets: `row.index` or `row["index"]`. - -|Field |Meaning | -|-|-| -| `row.index` | The index value of the current row (the first row is 0) | -| `row.cells` | The cells of the row, returned as an array | -| `row.columnNames` | An array of the column names of the project. This will report all columns, even those with null cell values in that particular row. Call a column by number with `row.columnNames[3]` | -| `row.starred` | A boolean indicating if the row is starred | -| `row.flagged` | A boolean indicating if the row is flagged | -| `row.record` | The [record](#record) object containing the current row | - -For array objects such as `row.columnNames` you can preview the array using the expressions window, and output it as a string using `toString(row.columnNames)` or with something like: - -``` -forEach(row.columnNames,v,v).join("; ") -``` - -### Cells {#cells} - -The `cells` object is used to call information from the columns in your project. For example, `cells.Foo` returns a [cell](#cell) object representing the cell in the column named “Foo” of the current row. If the column name has spaces, use square brackets, e.g., `cells["Postal Code"]`. To get the corresponding column's value inside the `cells` variable, use `.value` at the end, for example, `cells["Postal Code"].value`. There is no `cells.value` - it can only be used with member fields. - -### Cell {#cell} - -A `cell` object contains all the data of a cell and is stored as a single object. - -You can use `cell` on its own in the expressions editor to copy all the contents of a column to another column, including reconciliation information. Although the preview in the expressions editor will only show a small representation (“[object Cell]”), it will actually copy all the cell's data. Try this with Edit ColumnAdd Column based on this column .... - -|Field |Meaning |Member fields | -|-|-|-| -| `cell` | An object containing the entire contents of the cell | .value, .recon, .errorMessage | -| `cell.value` | The value in the cell, which can be a string, a number, a boolean, null, or an error | | -| `cell.recon` | An object encapsulating reconciliation results for that cell | See the [reconciliation](expressions#reconciliation) section | -| `cell.errorMessage` | Returns the message of an *EvalError* instead of the error object itself (use value to return the error object) | .value | - -### Reconciliation {#reconciliation} - -Several of the fields here provide the data used in [reconciliation facets](reconciling#reconciliation-facets). You must type `cell.recon`; `recon` on its own will not work. - -|Field|Meaning |Member fields | -|-|-|-| -| `cell.recon.judgment` | A string: either “matched”, "new”, "none” | | -| `cell.recon.judgmentAction` | A string: either "single” or “similar” (or “unknown”) | | -| `cell.recon.judgmentHistory` | A number, the epoch timestamp (in milliseconds) of your judgment | | -| `cell.recon.matched` | A boolean, true if judgment is “matched” | | -| `cell.recon.match` | The recon candidate that has been matched against this cell (or null) | .id, .name, .type | -| `cell.recon.best` | The highest scoring recon candidate from the reconciliation service (or null) | .id, .name, .type, .score | -| `cell.recon.features` | An array of reconciliation features to help you assess the accuracy of your matches | .typeMatch, .nameMatch, .nameLevenshtein, .nameWordDistance | -| `cell.recon.features.typeMatch` | A boolean, true if your chosen type is “matched” and false if not (or “(no type)” if unreconciled) | | -| `cell.recon.features.nameMatch` | A boolean, true if the cell and candidate strings are identical and false if not (or “(unreconciled)”) | | -| `cell.recon.features.nameLevenshtein` | A number representing the [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance): larger if the difference is greater between value and candidate | | -| `cell.recon.features.nameWordDistance` | A number based on the [word similarity](reconciling#reconciliation-facets) | | -| `cell.recon.candidates` | An array of the top 3 candidates (default) | .id, .name, .type, .score | - -The `cell.recon.candidates` and `cell.recon.best` objects have a few deeper fields: `id`, `name`, `type`, and `score`. `type` is an array of type identifiers for a list of candidates, or a single string for the best candidate. - -Arrays such as `cell.recon.candidates` and `cell.recon.candidates.type` can be joined into lists and stored as strings with something like: -``` -forEach(cell.recon.candidates,v,v.name).join("; ") -``` - -### Record {#record} - -A `row.record` object encapsulates one or more rows that are grouped together, when your project is in records mode. You must call it as `row.record`; `record` will not return values. - -|Field|Meaning | -|-|-| -| `row.record.index` | The index of the current record (starting at 0) | -| `row.record.cells` | An array of the [cells](#cells) in the given column of the record | -| `row.record.fromRowIndex` | The row index of the first row in the record | -| `row.record.toRowIndex` | The row index of the last row in the record + 1 (i.e. the next record) | -| `row.record.rowCount` | A count of the number of rows in the record | - -For example, you can facet by number of rows in each record by creating a Custom Numeric Facet (or a Custom Text Facet) and entering `row.record.rowCount`. \ No newline at end of file diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/facets.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/facets.md deleted file mode 100644 index 06e54b751..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/facets.md +++ /dev/null @@ -1,329 +0,0 @@ ---- -id: facets -title: Exploring facets -sidebar_label: Facets ---- - -## Overview {#overview} - -Facets are one of OpenRefine’s strongest features - that’s where the diamond logo comes from! - -Faceting allows you to look for patterns and trends. Facets are essentially aspects or angles of data variance in a given column. For example, if you had survey data where respondents indicated one of five responses from “Strongly agree” to “Strongly disagree,” those five responses make up a text facet, showing how many people selected each option. - -Faceted browsing gives you a big-picture look at your data (do they agree or disagree?) and also allows you to filter down to a specific subset to explore it more (what do people who disagree say in other responses?). - -Typically, you create a facet on a particular column. That facet selection appears on the left, in the Facet/Filter tab, and you can click on a displayed facet to view all the records that match. You can also “exclude” the facet, to view every record that does _not_ match, and you can select more than one facet by clicking “include.” - - -### An example {#an-example} - -You can learn about facets and filtering with the following example. You can copy the following table and paste it using the Clipboard method of starting a project if you would like to try it yourself. - -We collected a list of the [10 most populous cities from Wikidata](https://w.wiki/3Em), using an example query of theirs. We removed the GPS coordinates and added the country. - -| cityLabel | population | countryLabel | -|-|-|-| -| Shanghai | 23390000 | People's Republic of China | -| Beijing | 21710000 | People's Republic of China | -| Lagos | 21324000 | Nigeria | -| Dhaka | 16800000 | Bangladesh | -| Mumbai | 15414288 | India | -| Istanbul | 14657434 | Turkey | -| Tokyo | 13942856 | Japan | -| Tianjin | 13245000 | People's Republic of China | -| Guangzhou | 13080500 | People's Republic of China | -| São Paulo | 12106920 | Brazil | - -If we want to see which countries have the most populous cities, we can create a text facet on the “countryLabel” column and OpenRefine will generate a list of all the different strings used in these cells. - -We will see in the sidebar that the countries identified are displayed, along with the number of matches (the “count”). We can sort this list alphabetically or by the count. If you sort by count at the top of the facet window, you’ll learn which countries hold the most populous cities. - -|Facet|Count| -|---|---| -|People's Republic of China|4| -|Bangladesh|1| -|Brazil|1| -|India|1| -|Japan|1| -|Nigeria|1| -|Turkey|1| - -If we want to learn more about a particular country, we can click on its appearance in the facet sidebar. This narrows our dataset down temporarily to only rows matching that facet. - -You’ll see the “10 rows” indicator change to “4 matching rows (10 total)” if you click on “People’s Republic of China”. In the data grid, you’ll see fewer rows: only the ones matching your current filter. Each row will maintain its original numbering, though - in this case, rows #1, 2, and 8. - -If you want to go back to the original dataset, click Reset All or the small “exclude” text next to the facet. If you want to view the most populous cities in both China and India, click “include” next to each facet. Now you’ll see 5 rows - #1, 2, 5, 8, 9. - -We can also explore our data using the population information. In this case, because population is a number, we can create a numeric facet. This will give us the ability to explore by range rather than by exact matching values. - -With the numeric facet, we are given a scale from the smallest to the largest value in the column. We can drag the range minimum and maximum to narrow the results. In this case, if we narrow down to only cities with more than 20 million in population, we get 3 matching rows out of the original 10. - -When you look back at the text facet display of country names, you should see a smaller list with a reduced count: OpenRefine is now displaying the facets of the 3 matching rows, not the total dataset of 10 rows. - -We can combine these facets - say, by narrowing to only the Chinese cities with populations greater than 20 million - simply by clicking in both. You should see 2 matching rows for both these criteria. - -### Things to know about facets {#things-to-know-about-facets} - -When you have facets applied, you will see “matching rows” in the [project grid header](running#project-grid-header). If you click Export and copy your data out of OpenRefine while facets are active, many of the exporting options will only export the matching rows, not all the rows in your project. - -OpenRefine has several default facets, which you’ll learn about below. The most powerful facets are the ones designed by you - custom facets, written using [expressions](expressions) to transform the data behind the scenes and help you narrow down to precisely what you’re looking for. - -Facets are not saved in the project along with the data. But you can save a link to the current state of the application. Find the [Permalink](running#the-project-bar) next to the project’s name. - -You can modify any facet expression by clicking the “change” button to the right of the column name in the facet sidebar. - -Facet boxes that appear in the sidebar can be resized and rearranged. You can drag and drop the title bar of each box to reorder them, and drag on the bottom bar of text facet boxes. - -## Text facet {#text-facet} - -A text facet can be generated on any column with the “text” data type. Select the column dropdown and go to FacetText facet. The created facet will be sorted alphabetically, and can be sorted by count. - -A text facet is very simple: it takes the total contents of the cells of the column in question and matches them up. It does no guessing about typos or near-matches. - -You can edit any entry that appears in the facet display, by hovering over the facet and clicking the “edit” button that appears. You can then type in a new value manually. This will mass-edit every identical cell in the column. This is a great way to fix typos, whitespace, and other issues that may be affecting the way facets appear. You can also automate the cleanup of facets by using [clustering](transforming#cluster-and-edit): a “Cluster” button is displayed within the facet window. It may be most efficient to cluster cells to one value, and then mass-edit that value to your desired string within the clustering operation window. - -Each text facet shows up to 2,000 choices by default. You can [increase this limit on the Preferences screen](running#preferences) if you need to, which may slow down your browser. If your applied facet has more choices than the current limit, you'll be offered the option to increase the limit, which will permanently edit that preference for you. - -The choices and counts displayed in each facet can be copied as tab-separated values. To do so, click on the "X choices" link near the top left corner of the facet. This can be useful to generate small summary tables of your data. - -![A column of years faceted as text and numbers, and with the count ready to be copied.](/img/yeardata.png) - -## Numeric facet {#numeric-facet} - -![A screenshot of an example numeric facet.](/img/numericfacet.png) - -Whereas a text facet groups unique text values into groups, a numeric facet sorts numbers by their range - smallest to biggest. This displays visually as a histogram, and allows you to set a custom facet within that range. You can drag the minimum and maximum range markers to set a range. OpenRefine snaps to some basic equal-sized divisions - 19 in the example set above. - -You will be offered the option to include blank, non-numeric, and error values in your numeric visualization; these will appear in the visual range as “0” values. - -:::info Numbers as text -You can create a text facet on numeric data, which will treat each entry as a string. This can be useful if you wish, for example, to manually include facets instead of selecting a range, or sort by count, or copy that count. -::: - -:::info Faceting customization -As mentioned in the overview, facets can be modified or customized by GREL [expressions](expressions) in many ways. For example, to facet by clusters of [row](expressions#variables) numbers with `row.index/100` or better visualizing numbers greater than 1000 with `max(row.index, 1000)`. -::: - -## Timeline facet {#timeline-facet} - -![A screenshot of an example timeline facet.](/img/timelinefacet.png) - -Much like a numeric facet, a timeline facet will display as a small histogram with the values sorted: in this case, chronologically. A timeline facet only works on cells formatted as the [“date” data type](exploring#dates). - -The facet appears with a count of blank cells and those with errors, which can help you analyze whether your date cells are correctly converted. - -## Scatterplot facet {#scatterplot-facet} - -A scatterplot is a visual representation of two related sets of numeric data. - -You have the option to generate linear scatterplots (where the X and Y axes show continuous increases) or logarithmic scatterplots (where the X and Y axes show exponential or scaled increases). You can also rotate the plot by 45 degrees in either direction, and you can choose the size of the dot indicating a datapoint. You can make these choices in both the preview and in the facet display. - -A scatterplot facet can be generated on any column. You require two or more number columns to generate scatterplots. Selecting FacetScatterplot facet will create a preview of data plotted from every number-formatted column in your dataset, comparing every column against every other column. Each scatterplot will show in its own square, allowing you to choose which data comparison you would like to analyze further. You can control which columns are on the X and Y axes by rearranging the columns in your dataset. - -![A simple scatterplot of two numeric values.](/img/scatterplot.png) - -When you click on your desired square, that two-column comparison will appear in the facets sidebar. From here, you can drag your mouse to draw a rectangle inside the scatterplot, which will narrow down to just the rows matching the points plotted inside that rectangle (as shown by the rectangle inside the square in the image above). This rectangle can be resized by dragging any of the four edges. To draw a new rectangle, simply click and drag your mouse again. To add more scatterplots to the facet sidebar, re-run this process and select a different square. - -If you have multiple facets applied, plotted points in your scatterplot displays will be greyed out if they are not part of the current matching data subset. If the rectangle you have drawn within a scatterplot display only includes grey dots, you will see no matching rows. - -If you would like to export a scatterplot, OpenRefine will open a new tab with a generated PNG file that you can save. - -## Custom text facet {#custom-text-facet} - -You may want to explore your textual data with modifications that aren't permanent. Creating custom text facets will load your column into memory, transform the data temporarily, and store those transformations inside the facet. - -You can also use custom text facets to analyze numerical data, such as by analyzing a number as a string, or by creating a test that will return “true” and “false” as values. - -Clicking on FacetCustom text facet… will bring up an [expressions](expressions) window where you can enter in a GREL, Jython, or Clojure expression to modify how the facet works. - -A custom text facet operates just like a [text facet](#text-facet) by default. Unlike a text facet, however, you cannot click “edit” on the facets that appear in the sidebar and change the matching cells in your dataset - because what they display is modified, not the original entries. - -For example, you may wish to analyze only the first word in a text field - perhaps the first name in a column of “[First Name] [Last Name]” entries. In this case, you can tell OpenRefine to facet only on the information that comes before the first space: - -``` -value.split(" ")[0] -``` - -In this case, `split()` is creating an array of text strings based on every space in the cells ["Firstname", "Lastname"]. Because arrays number their entries starting with 0, we want the first value, so we ask for `[0]`. (Assuming the first name is one word, not something like “Mary Anne.”) We can do the same splitting and ask for the last name with - -``` -value.split(" ")[1] -``` - -You may want to create a facet that references several columns. For example, let’s say you have two columns, “First Name” and “Last Name”, and you want out how many people have the same initial letter for both names (e.g., Marilyn Monroe, Steven Segal). To do so, create a custom text facet on either column and enter the expression - -``` -cells["First Name"].value[0] == cells["Last Name"].value[0] -``` - -That expression will look for the first letter (the character at index 0) of each entry and compare them. Then it will facet your rows into “true” and “false.” - -You can learn more about text-modification functions on the [Expressions page](expressions). - -## Custom numeric facet {#custom-numeric-facet} - -You may want to explore your numerical data with modifications that aren't permanent. You can also use custom numeric facets to analyze textual data, such as by getting the length of text strings (with `value.length()`), or by analyzing it as though it were formatted as numbers (with `toNumber(value)`). - -If you would like to build your own version of a numeric facet, you can use the Custom Numeric Facet option. Clicking on FacetCustom Numeric Facet… will bring up an [expressions](expressions) window where you can enter in a GREL, Jython, or Clojure expression to modify how the facet works. A custom numeric facet operates just like a [numeric facet](#numeric-facet) by default. - -For example, you may wish to create a numeric facet that rounds your value to the nearest integer, enter - -``` -round(value) -``` - -If you have two columns of numbers and for each row you wish to create a numeric facet only on the larger of the two, enter - -``` -max(cells["Column1"].value, cells["Column2"].value) -``` - -If the numeric values in a column are drawn from a power law distribution, then it's better to group them by their logs: - -``` -value.log() -``` - -If the values are periodic you could take the modulus by the period to understand if there's a pattern: - -``` -mod(value, 7) -``` - -You can learn more about numeric-modification functions on the [Expressions page](expressions). - -## Customized facets {#customized-facets} - -Customized facets have been added to expand the number of default facets users can apply with a single click. They represent some common and useful functions you shouldn’t have to work out using an [expression](expressions). - -All facets that display in the Facet/Filter tab can be edited by clicking on the “change” button to the right of the column title. This brings up the expressions window that will allow you to modify and preview the expression being used. - -### Word facet {#word-facet} - -A Word facet is a simple version of a text facet: it splits up the content of the cells based on spaces, and outputs each character string as a facet: - -``` -value.split(" ") -``` - -This can be useful for exploring the language used in a corpus, looking for common first and last names or titles, or seeing what’s in multi-valued cells you don’t wish to split up. - -Word facet is case-sensitive and only splits by spaces, not by line breaks or other natural divisions. - -### Duplicates facet {#duplicates-facet} - -A Duplicates facet will return only rows that have non-unique values in the column you’ve selected. It will create a facet of “true” and “false” values - true being cells that are not unique, and “false” being cells that are. The actual expression being used is - -``` -facetCount(value, 'value', '[Column]') > 1 -``` - -Duplicates facets are case-sensitive and you may wish to filter out things like leading and trailing whitespace or other hard-to-see issues. You can modify the facet expression, for example, with: - -``` -facetCount(trim(toLowercase(value)), 'trim(toLowercase(value))', 'cityLabel') > 1 -``` - -### Numeric log facet {#numeric-log-facet} - -Logarithmic scales reduce wide-ranging quantities to more compact and manageable ranges. A log transformation can be used to make highly skewed distributions less skewed. If your numerical data is unevenly distributed (say, lots of values in one range, and then a long tail extending off into different magnitudes), a Numeric log facet can represent that range better than a simple numeric facet. It will break these values down into more navigable segments than the buckets of a numeric facet. This facet can make patterns in your data more visible. OpenRefine uses a base-10 log, the “common logarithm.” - -For example, we can look at [this data about the body weight of various mammals](http://wiki.stat.ucla.edu/socr/index.php/SOCR_Data_Brain2BodyWeight): - -|Species|BodyWeight (kg)| -|---|---| -| Newborn_Human | 3.2 | -| Adult_Human | 73 | -| Pithecanthropus_Man | 70 | -| Squirrel | 0.8 | -| Hamster | 0.15 | -| Chimpanzee | 50 | -| Rabbit | 1.4 | -| Dog_(Beagle) | 10 | -| Cat | 4.5 | -| Rat | 0.4 | -| Sperm_Whale | 35000 | -| Turtle | 3 | -| Alligator | 270 | - -Most values will be clustered in the 0-100 range, but 35,000 is many magnitudes above that. A numeric facet will create 36 equal buckets of 1,000 each - containing almost all the cells in the first bucket. A numeric log facet will instead display the data more evenly across the visual range. - -![A screenshot of a numeric facet first and a numeric log facet second.](/img/numericlogfacet.png) - -A 1-bounded numeric log facet can be used if you'd like to exclude all the values below 1 (including zero and negative numbers). - -### Text-length facet {#text-length-facet} - -The Text-length facet returns a numerical value for each cell and plots it on a numeric facet chart. The expression used is - -``` -value.length() -``` - -This can be useful to, for example, look for values that did not successfully split on an earlier split operation, or to validate that data is a certain expected length (such as whether a date in YYYY/MM/DD is eight to ten characters). - -You can also employ a Log of text-length facet that allows you to navigate more easily a wide range of string lengths. This can be useful in the case of web-scraping, where lots of textual data is loaded into single cells and needs to be parsed out. - - -### Unicode character-code facet {#unicode-character-code-facet} - -![A screenshot of the Unicode facet.](/img/unicodefacet.png) - -The Unicode facet identifies and returns [Unicode decimal values](https://en.wikipedia.org/wiki/List_of_Unicode_characters). It generates a list of the Unicode numerical values of each character used in each text cell, which allows you to narrow down and search for special characters, punctuation, and other data formatting issues. - -This facet creates a numerical chart, which offers you the ability to narrow down to a range of numbers. For example, lowercase characters are numbers 97-122, uppercase characters are numbers 65-90, and numerical digits are numbers 48-57. - -### Facet by error {#facet-by-error} - -An error is a data type created by OpenRefine in the process of transforming data. For example, say you had converted a column to the number data type. If one cell had text characters in it, OpenRefine could either output the original text string unchanged or output an error. If you allow errors to be created, you can facet by them later to search for them and fix them. - -![A view of the expressions window with an error converting a string to a number.](/img/error.png) - -To store errors in cells, ensure that you have store error selected for the “On error” option in the expressions window. - -### Facet by null, empty, or blank {#facet-by-null-empty-or-blank} - -Any column can be faceted for [null and/or empty cells](#cell-data-types). These can help you find cells where you want to manually enter content. - -“Blank” means both null values and empty values. All three facets will generate “true” and “false” facets, “true” being blank. - -An empty cell is a cell that is set to contain a string, but doesn’t have any characters in it (a zero-length string). This can be left over from an operation that removed characters, or from manually editing a cell and deleting its contents. - -### Facet by star or flag {#facet-by-star-or-flag} - -Stars and flags offer you the opportunity to mark specific rows for yourself for later focus. Stars and flags persist through closing and opening your project, and thus can provide a different function than using a permalink to persist your facets. Stars and flags can be used in any way you want, although they are designed to help you flag errors and star rows of particular importance. - -You can manually star or flag rows simply by clicking on the icons to the left of each row. - -You can also apply stars or flags to all matching rows by using the All dropdown menu (on the first column) and selecting Edit rowsStar rows or Flag rows. This will create “true” and “false” facets in the Facet/Filter. These operations will modify all matching rows in your current subset. You can unstar or unflag them as well. - -You may wish to create a custom subset of your data through a series of separate faceting activities (rather than successively narrowing down with multiple facets applied). For example, you may wish to: -* apply a facet -* star all the matching rows -* remove that facet -* apply another, unrelated facet -* star all the new matching rows (which will not modify already-starred rows) -* remove that facet -* and then work with all of the cumulative starred rows. - -You can also create a text facet on any column with the expression `row.starred` or `row.flagged`. - -## Text filter {#text-filter} - -Filters allow you to narrow down your data based on whether a given column includes a text string. - -When you choose Text filter a box appears in the Facet/Filter tab that allows you to enter in text. Matching rows will narrow dynamically with every character you enter. You can set the search to be case-sensitive or not, and you can use this box to enter in a regular expression. - -For example, you can enter in “side” as a text filter, and it will return all cells in that column containing “side,” “sideways,” “offside,” etc. - -The text filter field supports [regular expressions](expressions#regular-expressions). For example, you can employ a regular expression to view all properly-formatted emails: - -``` -([a-zA-Z0-9_\-\.\+]+)@([a-zA-Z0-9\-\.]+)\.([a-zA-Z0-9\-]{2,15}) -``` - -You can press “invert” on this facet to then see blank cells or invalid email addresses. - -This filter works differently than facets because it is always active as long as it appears in the sidebar. If you “reset” it, you will delete all the text or expression you have entered. - -You can apply multiple text filters in succession, which will successively narrow your data subset. This can be useful if you apply multiple inverted filters, such as to filter out all rows that respond “yes” or “maybe” and only look at the remaining responses. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/grel.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/grel.md deleted file mode 100644 index 7ec90b464..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/grel.md +++ /dev/null @@ -1,154 +0,0 @@ ---- -id: grel -title: General Refine Expression Language -sidebar_label: General Refine Expression Language ---- - -## Basics {#basics} - -GREL (General Refine Expression Language) is designed to resemble Javascript. Formulas use variables and depend on data types to do things like string manipulation or mathematical calculations: - -|Example|Output| -|---|---| -| `value + " (approved)"` | Concatenate two strings; whatever is in the cell gets converted to a string first | -| `value + 2.239` | Add 2.239 to the existing value (if a number); append text "2.239" to the end of the string otherwise | -| `value.trim().length()`     | Trim leading and trailing whitespace of the cell value and then output the length of the result | -| `value.substring(7, 10)` | Output the substring of the value from character index 7, 8, and 9 (excluding character index 10) | -| `value.substring(13)` | Output the substring from index 13 to the end of the string | - -Note that the operator for string concatenation is `+` (not “&” as is used in Excel). - -Evaluating conditions uses symbols such as <, >, *, /, etc. To check whether two objects are equal, use two equal signs (`value=="true"`). - -See the [GREL functions page for a thorough reference](grelfunctions) on each function and its inputs and outputs. Read on below for more about the general nature of GREL expressions. - -## Syntax {#syntax} - -In GREL, functions can use either of these two forms: -* functionName(arg0, arg1, ...) -* arg0.functionName(arg1, ...) - -The second form is a shorthand to make expressions easier to read. It simply pulls the first argument out and appends it to the front of the function, with a dot: - -|Dot notation |Full notation | -|-|-| -| `value.trim().length()` | `length(trim(value))` | -| `value.substring(7, 10)` | `substring(value, 7, 10)` | - -So, in the dot shorthand, the functions occur from left to right in the order of calling, rather than in the reverse order with parentheses. This allows you to string together multiple functions in a readable order. - -The dot notation can also be used to access the member fields of [variables](expressions#variables). For referring to column names that contain spaces (anything not a continuous string), use square brackets instead of dot notation: - -|Example |Description | -|-|-| -| `FirstName.cells` | Access the cell in the column named “FirstName” of the current row | -| `cells["First Name"]` | Access the cell in the column called “First Name” of the current row | - -Square brackets can also be used to get substrings and sub-arrays, and single items from arrays: - -|Example |Description | -|-|-| -| `value[1,3]` | A substring of value, starting from character 1 up to but excluding character 3 | -| `"internationalization"[1,-2]` | Will return “nternationalizati” (negative indexes are counted from the end) | -| `row.columnNames[5]` | Will return the name of the fifth column | - -Any function that outputs an array can use square brackets to select only one part of the array to output as a string (remember that the index of the items in an array starts with 0). - -For example, [partition()](grelfunctions#partitions-s-or-p-fragment-b-omitfragment-optional) would normally output an array of three items: the part before your chosen fragment, the fragment you've identified, and the part after. Selecting only the third part with `"internationalization".partition("nation")[2]` will output “alization” (and so will [-1], indicating the final item in the array). - -## Controls {#controls} - -GREL offers controls to support branching and looping (that is, “if” and “for” functions), but unlike functions, their arguments don't all get evaluated before they get run. A control can decide which part of the code to execute and can affect the environment bindings. Functions, on the other hand, can't do either. Each control decides which of their arguments to evaluate to `value`, and how. - -Please note that the GREL control names are case-sensitive: for example, the isError() control can't be called with iserror(). - -#### if(e, eTrue, eFalse) {#ife-etrue-efalse} - -Expression e is evaluated to a value. If that value is true, then expression eTrue is evaluated and the result is the value of the whole if() expression. Otherwise, expression eFalse is evaluated and that result is the value. - -Examples: - -| Example expression | Result | -| ------------------------------------------------------------------------ | ------------ | -| `if("internationalization".length() > 10, "big string", "small string")` | “big string” | -| `if(mod(37, 2) == 0, "even", "odd")` | “odd” | - -Nested if (switch case) example: - - if(value == 'Place', 'http://www.example.com/Location', - - if(value == 'Person', 'http://www.example.com/Agent', - - if(value == 'Book', 'http://www.example.com/Publication', - - null))) - -#### with(e1, variable v, e2) {#withe1-variable-v-e2} - -Evaluates expression e1 and binds its value to variable v. Then evaluates expression e2 and returns that result. - -| Example expression | Result | -| ------------------------------------------------------------------------------------ | ---------- | -| `with("european union".split(" "), a, a.length())` | 2 | -| `with("european union".split(" "), a, forEach(a, v, v.length()))` | [ 8, 5 ] | -| `with("european union".split(" "), a, forEach(a, v, v.length()).sum() / a.length())` | 6.5 | - -#### filter(e1, v, e test) {#filtere1-v-e-test} - -Evaluates expression e1 to an array. Then for each array element, binds its value to variable v, evaluates expression test - which should return a boolean. If the boolean is true, pushes v onto the result array. - -| Expression | Result | -| ---------------------------------------------- | ------------- | -| `filter([ 3, 4, 8, 7, 9 ], v, mod(v, 2) == 1)` | [ 3, 7, 9 ] | - -#### forEach(e1, v, e2) {#foreache1-v-e2} - -Evaluates expression e1 to an array. Then for each array element, binds its value to variable v, evaluates expression e2, and pushes the result onto the result array. When e1 is a JSON object, `forEach` iterates over its keys. - -| Expression | Result | -| ------------------------------------------ | ------------------- | -| `forEach([ 3, 4, 8, 7, 9 ], v, mod(v, 2))` | [ 1, 0, 0, 1, 1 ] | - -#### forEachIndex(e1, i, v, e2) {#foreachindexe1-i-v-e2} - -Evaluates expression e1 to an array. Then for each array element, binds its index to variable i and its value to variable v, evaluates expression e2, and pushes the result onto the result array. - -| Expression | Result | -| ------------------------------------------------------------------------------- | --------------------------- | -| `forEachIndex([ "anne", "ben", "cindy" ], i, v, (i + 1) + ". " + v).join(", ")` | 1. anne, 2. ben, 3. cindy | - -#### forRange(n from, n to, n step, v, e) {#forrangen-from-n-to-n-step-v-e} - -Iterates over the variable v starting at from, incrementing by the value of step each time while less than to. At each iteration, evaluates expression e, and pushes the result onto the result array. - -#### forNonBlank(e, v, eNonBlank, eBlank) {#fornonblanke-v-enonblank-eblank} - -Evaluates expression e. If it is non-blank, forNonBlank() binds its value to variable v, evaluates expression eNonBlank and returns the result. Otherwise (if e evaluates to blank), forNonBlank() evaluates expression eBlank and returns that result instead. - -Unlike other GREL functions beginning with “for,” forNonBlank() is not iterative. forNonBlank() essentially offers a shorter syntax to achieving the same outcome by using the isNonBlank() function within an “if” statement. - -#### isBlank(e), isNonBlank(e), isNull(e), isNotNull(e), isNumeric(e), isError(e) {#isblanke-isnonblanke-isnulle-isnotnulle-isnumerice-iserrore} - -Evaluates the expression e, and returns a boolean based on the named evaluation. - -Examples: - -| Expression | Result | -| ------------------- | ------- | -| `isBlank("abc")` | false | -| `isNonBlank("abc")` | true | -| `isNull("abc")` | false | -| `isNotNull("abc")` | true | -| `isNumeric(2)` | true | -| `isError(1)` | false | -| `isError("abc")` | false | -| `isError(1 / 0)` | true | - -Remember that these are controls and not functions: you can’t use dot notation (for example, the format `e.isX()` will not work). - -## Constants {#constants} -|Name |Meaning | -|-|-| -| true | The boolean constant true | -| false | The boolean constant false | -| PI | From [Java's Math.PI](https://docs.oracle.com/javase/8/docs/api/java/lang/Math.html#PI), the value of pi (that is, 3.1415...) | diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/grelfunctions.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/grelfunctions.md deleted file mode 100644 index 5e4893851..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/grelfunctions.md +++ /dev/null @@ -1,535 +0,0 @@ ---- -id: grelfunctions -title: GREL functions -sidebar_label: GREL functions ---- - -## Reading this reference {#reading-this-reference} - -For the reference below, the function is given in full-length notation and the in-text examples are written in dot notation. Shorthands are used to indicate the kind of [data type](exploring#data-types) used in each function: s for string, b for boolean, n for number, d for date, a for array, p for a regex pattern, and o for object (meaning any data type), as well as “null” and “error” data types. - -If a function can take more than one kind of data as input or can output more than one kind of data, that is indicated with more than one letter (as with “s or a”) or with o for object, meaning it can take any type of data (string, boolean, date, number, etc.). - -We also use shorthands for substring (“sub”) and separator string (“sep”). -Optional arguments will say “(optional)”. - -In places where OpenRefine will accept a string (s) or a regex pattern (p), you can supply a string by putting it in quotes. If you wish to use any [regex](expressions#regular-expressions) notation, wrap the pattern in forward slashes. - -## Boolean functions {#boolean-functions} - -###### and(b1, b2, ...) {#andb1-b2-} - -Uses the logical operator AND on two or more booleans to output a boolean. Evaluates multiple statements into booleans, then returns true if all of the statements are true. For example, `(1 < 3).and(1 < 0)` returns false because one condition is true and one is false. - -###### or(b1, b2, ...) {#orb1-b2-} - -Uses the logical operator OR on two or more booleans to output a boolean. For example, `(1 < 3).or(1 > 7)` returns true because at least one of the conditions (the first one) is true. - -###### not(b) {#notb} - -Uses the logical operator NOT on a boolean to output a boolean. For example, `not(1 > 7)` returns true because 1 > 7 itself is false. - -###### xor(b1, b2, ...) {#xorb1-b2-} - -Uses the logical operator XOR (exclusive-or) on two or more booleans to output a boolean. Evaluates multiple statements, then returns true if only one of them is true. For example, `(1 < 3).xor(1 < 7)` returns false because more than one of the conditions is true. - -## String functions {#string-functions} - -###### length(s) {#lengths} - -Returns the length of string s as a number. - -###### toString(o, string format (optional)) {#tostringo-string-format-optional} - -Takes any value type (string, number, date, boolean, error, null) and gives a string version of that value. - -You can use toString() to convert numbers to strings with rounding, using an [optional string format](https://docs.oracle.com/javase/8/docs/api/java/util/Formatter.html). For example, if you applied the expression `value.toString("%.0f")` to a column: - -|Input|Output| -|-|-| -|3.2|3| -|0.8|1| -|0.15|0| -|100.0|100| - -You can also convert dates to strings, using date parsing syntax built into OpenRefine (see [the toDate() function for details](#todateo-b-monthfirst-s-format1-s-format2-)). For example, `value.toString("MMM-dd-yyyy")` would convert the date value [2024-10-15T00:00:00Z] to “Oct-15-2024”. - -Note: In OpenRefine, using toString() on a null cell outputs the string “null”. - -### Testing string characteristics {#testing-string-characteristics} - -###### startsWith(s, sub) {#startswiths-sub} - -Returns a boolean indicating whether s starts with sub. For example, `"food".startsWith("foo")` returns true, whereas `"food".startsWith("bar")` returns false. - -###### endsWith(s, sub) {#endswiths-sub} - -Returns a boolean indicating whether s ends with sub. For example, `"food".endsWith("ood")` returns true, whereas `"food".endsWith("odd")` returns false. - -###### contains(s, sub or p) {#containss-sub-or-p} - -Returns a boolean indicating whether s contains sub, which is either a substring or a regex pattern. For example, `"food".contains("oo")` returns true whereas `"food".contains("ee")` returns false. - -You can search for a regular expression by wrapping it in forward slashes rather than quotes: `"rose is a rose".contains(/\s+/)` returns true. startsWith() and endsWith() can only take strings, while contains() can take a regex pattern, so you can use contains() to look for beginning and ending string patterns. - -### Basic string modification {#basic-string-modification} - -#### Case conversion {#case-conversion} - -###### toLowercase(s) {#tolowercases} - -Returns string s converted to all lowercase characters. - -###### toUppercase(s) {#touppercases} - -Returns string s converted to all uppercase characters. - -###### toTitlecase(s) {#totitlecases} - -Returns string s converted into titlecase: a capital letter starting each word, and the rest of the letters lowercase. For example, `"Once upon a midnight DREARY".toTitlecase()` returns the string “Once Upon A Midnight Dreary”. - -#### Trimming {#trimming} - -###### trim(s) {#trims} - -Returns a copy of the string s with leading and trailing whitespace removed. For example, `" island ".trim()` returns the string “island”. Identical to strip(). - -###### strip(s) {#strips} - -Returns a copy of the string s with leading and trailing whitespace removed. For example, `" island ".strip()` returns the string “island”. Identical to trim(). - -###### chomp(s, sep) {#chomps-sep} - -Returns a copy of string s with the string sep removed from the end if s ends with sep; otherwise, just returns s. For example, `"barely".chomp("ly")` and `"bare".chomp("ly")` both return the string “bare”. - -#### Substring {#substring} - -###### substring(s, n from, n to (optional)) {#substrings-n-from-n-to-optional} - -Returns the substring of s starting from character index from, and up to (excluding) character index to. If the to argument is omitted, substring will output to the end of s. For example, `"profound".substring(3)` returns the string “found”, and `"profound".substring(2, 4)` returns the string “of”. - -Remember that character indices start from zero. A negative character index counts from the end of the string. For example, `"profound".substring(0, -1)` returns the string “profoun”. - -###### slice(s, n from, n to (optional)) {#slices-n-from-n-to-optional} - -Identical to substring() in relation to strings. Also works with arrays; see [Array functions section](#slicea-n-from-n-to-optional). - -###### get(s, n from, n to (optional)) {#gets-n-from-n-to-optional} - -Identical to substring() in relation to strings. Also works with named fields. Also works with arrays; see [Array functions section](#geta-n-from-n-to-optional). - -#### Find and replace {#find-and-replace} - -###### indexOf(s, sub) {#indexofs-sub} - -Returns the first character index of sub as it first occurs in s; or, returns -1 if s does not contain sub. For example, `"internationalization".indexOf("nation")` returns 5, whereas `"internationalization".indexOf("world")` returns -1. - -###### lastIndexOf(s, sub) {#lastindexofs-sub} - -Returns the first character index of sub as it last occurs in s; or, returns -1 if s does not contain sub. For example, `"parallel".lastIndexOf("a")` returns 3 (pointing at the second “a”). - -###### replace(s, s or p find, s replace) {#replaces-s-or-p-find-s-replace} - -Returns the string obtained by replacing the find string with the replace string in the inputted string. For example, `"The cow jumps over the moon and moos".replace("oo", "ee")` returns the string “The cow jumps over the meen and mees”. Find can be a regex pattern. For example, `"The cow jumps over the moon and moos".replace(/\s+/, "_")` will return “The_cow_jumps_over_the_moon_and_moos”. - -You cannot find or replace nulls with this, as null is not a string. You can instead: - -1. Facet by null and then bulk-edit them to a string, or -2. Transform the column with an expression such as `if(value==null,"new",value)`. - -###### replaceChars(s, s find, s replace) {#replacecharss-s-find-s-replace} - -Returns the string obtained by replacing a character in s, identified by find, with the corresponding character identified in replace. For example, `"Téxt thát was optícálly recógnízéd".replaceChars("áéíóú", "aeiou")` returns the string “Text that was optically recognized”. You cannot use this to replace a single character with more than one character. - -###### find(s, sub or p) {#finds-sub-or-p} - -Outputs an array of all consecutive substrings inside string s that match the substring or [regex](expressions#grel-supported-regex) pattern p. For example, `"abeadsabmoloei".find(/[aeio]+/)` would result in the array [ "a", "ea", "a", "o", "oei" ]. - -You can supply a substring instead of p, by putting it in quotes, and OpenRefine will compile it into a regex pattern. Anytime you supply quotes, OpenRefine interprets the contents as a string, not regex. If you wish to use any regex notation, wrap the pattern in forward slashes. - -###### match(s, p) {#matchs-p} - -Attempts to match the string s in its entirety against the [regex](expressions#grel-supported-regex) pattern p and, if the pattern is found, outputs an array of all [capturing groups](https://www.regular-expressions.info/brackets.html) (found in order). For example, `"230.22398, 12.3480".match(/.*(\d\d\d\d)/)` returns an array of 1 substring: [ "3480" ]. It does not find 2239 as the first sequence with four digits, because the regex indicates the four digits must come at the end of the string. - -You will need to convert the array to a string to store it in a cell, with a function such as toString(). An empty array [] is returned when there is no match to the desired substrings. A null is output when the entire regex does not match. - -Remember to enclose your regex in forward slashes, and to escape characters and use parentheses as needed. Parentheses denote a desired substring (capturing group); for example, “.*(\d\d\d\d)” would return an array containing a single value, while “(.*)(\d\d\d\d)” would return two. So, if you are looking for a desired substring anywhere within a string, use the syntax `value.match(/.*(desired-substring-regex).*/)`. - -For example, if `value` is “hello 123456 goodbye”, the following would occur: - -|Expression|Result| -|-|-| -|`value.match(/\d{6}/)` |null (does not match the full string)| -|`value.match(/.*\d{6}.*/)` |[ ] (no indicated substring)| -|`value.match(/.*(\d{6}).*/)` |[ "123456" ] (array with one value)| -|`value.match(/(.*)(\d{6})(.*)/)` |[ "hello ", "123456", " goodbye" ] (array with three values)| - -### String parsing and splitting {#string-parsing-and-splitting} - -###### toNumber(s) {#tonumbers} - -Returns a string converted to a number. Will attempt to convert other formats into a string, then into a number. If the value is already a number, it will return the number. - -###### split(s, s or p sep, b preserveTokens (optional)) {#splits-s-or-p-sep-b-preservetokens-optional} - -Returns the array of strings obtained by splitting s by sep. The separator can be either a string or a regex pattern. For example, `"fire, water, earth, air".split(",")` returns an array of 4 strings: [ "fire", " water", " earth", " air" ]. Note that the space characters are retained but the separator is removed. If you include “true” for the preserveTokens boolean, empty segments are preserved. - -###### splitByLengths(s, n1, n2, ...) {#splitbylengthss-n1-n2-} - -Returns the array of strings obtained by splitting s into substrings with the given lengths. For example, `"internationalization".splitByLengths(5, 6, 3)` returns an array of 3 strings: [ "inter", "nation", "ali" ]. Excess characters are discarded. - -###### smartSplit(s, s or p sep (optional)) {#smartsplits-s-or-p-sep-optional} - -Returns the array of strings obtained by splitting s by sep, or by guessing either tab or comma separation if there is no sep given. Handles quotes properly and understands cancelled characters. The separator can be either a string or a regex pattern. For example, `value.smartSplit("\n")` will split at a carriage return or a new-line character. - -Note: [`value.escape('javascript')`](#escapes-s-mode) is useful for previewing unprintable characters prior to using smartSplit(). - -###### splitByCharType(s) {#splitbychartypes} - -Returns an array of strings obtained by splitting s into groups of consecutive characters each time the characters change [Unicode categories](https://en.wikipedia.org/wiki/Unicode_character_property#General_Category). For example, `"HenryCTaylor".splitByCharType()` will result in an array of [ "H", "enry", "CT", "aylor" ]. It is useful for separating letters and numbers: `"BE1A3E".splitByCharType()` will result in [ "BE", "1", "A", "3", "E" ]. - -###### partition(s, s or p fragment, b omitFragment (optional)) {#partitions-s-or-p-fragment-b-omitfragment-optional} - -Returns an array of strings [ a, fragment, z ] where a is the substring within s before the first occurrence of fragment, and z is the substring after fragment. Fragment can be a string or a regex. For example, `"internationalization".partition("nation")` returns 3 strings: [ "inter", "nation", "alization" ]. If s does not contain fragment, it returns an array of [ s, "", "" ] (the original unpartitioned string, and two empty strings). - -If the omitFragment boolean is true, for example with `"internationalization".partition("nation", true)`, the fragment is not returned. The output is [ "inter", "alization" ]. - -You can use regex for your fragment. The expresion `"abcdefgh".partition(/c.e/)` will output [“abc”, "cde", defgh” ]. - -###### rpartition(s, s or p fragment, b omitFragment (optional)) {#rpartitions-s-or-p-fragment-b-omitfragment-optional} - -Returns an array of strings [ a, fragment, z ] where a is the substring within s before the last occurrence of fragment, and z is the substring after the last instance of fragment. (Rpartition means “reverse partition.”) For example, `"parallel".rpartition("a")` returns 3 strings: [ "par", "a", "llel" ]. Otherwise works identically to partition() above. - -### Encoding and hashing {#encoding-and-hashing} - -###### diff(s1, s2, s timeUnit (optional)) {#diffs1-s2-s-timeunit-optional} - -Takes two strings and compares them, returning a string. Returns the remainder of s2 starting with the first character where they differ. For example, `"cacti".diff("cactus")` returns "us". Also works with dates; see [Date functions](#diffd1-d2-s-timeunit). - -###### escape(s, s mode) {#escapes-s-mode} - -Escapes s in the given escaping mode. The mode can be one of: "html", "xml", "csv", "url", "javascript". Note that quotes are required around your mode. See the [recipes](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#question-marks--showing-in-your-data) for examples of escaping and unescaping. - -###### unescape(s, s mode) {#unescapes-s-mode} - -Unescapes s in the given escaping mode. The mode can be one of: "html", "xml", "csv", "url", "javascript". Note that quotes are required around your mode. See the [recipes](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#atampampt----att) for examples of escaping and unescaping. - -###### md5(o) {#md5o} - -Returns the [MD5 hash](https://en.wikipedia.org/wiki/MD5) of an object. If fed something other than a string (array, number, date, etc.), md5() will convert it to a string and deliver the hash of the string. For example, `"internationalization".md5()` will return 2c55a1626e31b4e373ceedaa9adc12a3. - -###### sha1(o) {#sha1o} - -Returns the [SHA-1 hash](https://en.wikipedia.org/wiki/SHA-1) of an object. If fed something other than a string (array, number, date, etc.), sha1() will convert it to a string and deliver the hash of the string. For example, `"internationalization".sha1()` will return cd05286ee0ff8a830dbdc0c24f1cb68b83b0ef36. - -###### phonetic(s, s encoding) {#phonetics-s-encoding} - -Returns a phonetic encoding of a string, based on an available phonetic algorithm. See the [section on phonetic clustering](cellediting#clustering-methods) for more information. Can be one of the following supported phonetic methods: [metaphone, doublemetaphone, metaphone3](https://www.wikipedia.org/wiki/Metaphone), [soundex](https://en.wikipedia.org/wiki/Soundex), [cologne](https://en.wikipedia.org/wiki/Cologne_phonetics). Quotes are required around your encoding method. For example, `"Ruth Prawer Jhabvala".phonetic("metaphone")` outputs the string “R0PRWRJHBFL”. - -###### reinterpret(s, s encoderTarget, s encoderSource) {#reinterprets-s-encodertarget-s-encodersource} - -Returns s reinterpreted through the given character encoders. You must supply one of the [supported encodings](http://java.sun.com/j2se/1.5.0/docs/guide/intl/encoding.doc.html) for each of the original source and the target output. Note that quotes are required around your character encoder. - -When an OpenRefine project is started, data is imported and interpreted. A specific character encoding is identified or manually selected at that time (such as UTF-8). You can reinterpret a column into another specificed encoding using this function. This function may not fix your data; it may be better to use this in conjunction with new projects to test the interpretation, and pre-format your data as needed. - -###### fingerprint(s) {#fingerprints} - -Returns the fingerprint of s, a string that is the first step in [fingerprint clustering methods](cellediting#clustering-methods): it will trim whitespaces, convert all characters to lowercase, remove punctuation, sort words alphabetically, etc. For example, `"Ruth Prawer Jhabvala".fingerprint()` outputs the string “jhabvala prawer ruth”. - -###### ngram(s, n) {#ngrams-n} - -Returns an array of the word n-grams of s. That is, it lists all the possible consecutive combinations of n words in the string. For example, `"Ruth Prawer Jhabvala".ngram(2)` would output the array [ "Ruth Prawer", "Prawer Jhabvala" ]. A word n-gram of 1 simply lists all the words in original order; an n-gram larger than the number of words in the string will only return the original string inside an array (e.g. `"Ruth Prawer Jhabvala".ngram(4)` would simply return ["Ruth Prawer Jhabvala"]). - -###### ngramFingerprint(s, n) {#ngramfingerprints-n} - -Returns the [n-gram fingerprint](cellediting#clustering-methods) of s. For example, `"banana".ngram(2)` would output “anbana”, after first generating the 2-grams “ba an na an na”, removing duplicates, and sorting them alphabetically. - -###### unicode(s) {#unicodes} - -Returns an array of strings describing each character of s in their full unicode notation. For example, `"Bernice Rubens".unicode()` outputs [ 66, 101, 114, 110, 105, 99, 101, 32, 82, 117, 98, 101, 110, 115 ]. - -###### unicodeType(s) {#unicodetypes} - -Returns an array of strings describing each character of s by their unicode type. For example, `"Bernice Rubens".unicodeType()` outputs [ "uppercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "space separator", "uppercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter", "lowercase letter" ]. - -## Format-based functions (JSON, HTML, XML) {#format-based-functions-json-html-xml} - -###### jsonize(o) {#jsonizeo} - -Quotes a value as a JSON literal value. - -###### parseJson(s) {#parsejsons} - -Parses a string as JSON. get() can then be used with parseJson(): for example, `parseJson(" { 'a' : 1 } ").get("a")` returns 1. - -For example, from the following JSON array in `value`, we want to get all instances of “keywords” having the same object string name of “text”, and combine them, using the forEach() function to iterate over the array. - - { - "status":"OK", - "url":"", - "language":"english", - "keywords":[ - { - "text":"York en route", - "relevance":"0.974363" - }, - { - "text":"Anthony Eden", - "relevance":"0.814394" - }, - { - "text":"President Eisenhower", - "relevance":"0.700189" - } - ] - } - -The GREL expression `forEach(value.parseJson().keywords,v,v.text).join(":::")` will output “York en route:::Anthony Eden:::President Eisenhower”. - -### Jsoup XML and HTML parsing {#jsoup-xml-and-html-parsing} - -###### parseHtml(s) {#parsehtmls} -Given a cell full of HTML-formatted text, parseHtml() simplifies HTML tags (such as by removing “ /” at the end of self-closing tags), closes any unclosed tags, and inserts linebreaks and indents for cleaner code. You cannot pass parseHtml() a URL, but you can pre-fetch HTML with the [Add column by fetching URLs](columnediting#add-column-by-fetching-urls) menu option. - -A cell cannot store the output of parseHtml() unless you convert it with toString(): for example, `value.parseHtml().toString()`. - -When parseHtml() simplifies HTML, it can sometimes introduce errors. When closing tags, it makes its best guesses based on line breaks, indentation, and the presence of other tags. You may need to manually check the results. - -You can then extract or [select()](#selects-element) which portions of the HTML document you need for further splitting, partitioning, etc. An example of extracting all table rows from a div using parseHtml().select() together is described more in depth at [StrippingHTML](https://github.com/OpenRefine/OpenRefine/wiki/StrippingHTML). - -###### parseXml(s) {#parsexmls} -Given a cell full of XML-formatted text, parseXml() returns a full XML document and adds any missing closing tags. You can then extract or [select()](#selects-element) which portions of the XML document you need for further splitting, partitioning, etc. Functions the same way as parseHtml() is described above. - -###### select(s, element) {#selects-element} -Returns an array of all the desired elements from an HTML or XML document, if the element exists. Elements are identified using the [Jsoup selector syntax](https://jsoup.org/apidocs/org/jsoup/select/Selector.html). For example, `value.parseHtml().select("img.portrait")[0]` would return the entirety of the first “img” tag with the “portrait” class found in the parsed HTML inside `value`. Returns an empty array if no matching element is found. Use with toString() to capture the results in a cell. A tutorial of select() is shown in [StrippingHTML](https://github.com/OpenRefine/OpenRefine/wiki/StrippingHTML). - -You can use select() more than once: - -``` -value.parseHtml().select("div#content")[0].select("tr").toString() -``` - -###### htmlAttr(s, element) {#htmlattrs-element} -Returns a string from an attribute on an HTML element. Use it in conjunction with parseHtml() as in the following example: `value.parseHtml().select("a.email")[0].htmlAttr("href")` would retrieve the email address attached to a link with the “email” class. - -###### xmlAttr(s, element) {#xmlattrs-element} -Returns a string from an attribute on an XML element. Functions the same way htmlAttr() is described above. Use it in conjunction with parseXml(). - -###### htmlText(element) {#htmltextelement} -Returns a string of the text from within an HTML element (including all child elements), removing HTML tags and line breaks inside the string. Use it in conjunction with parseHtml() and select() to provide an element, as in the following example: `value.parseHtml().select("div.footer")[0].htmlText()`. - -###### xmlText(element) {#xmltextelement} -Returns a string of the text from within an XML element (including all child elements). Functions the same way htmlText() is described above. Use it in conjunction with parseXml() and select() to provide an element. - -###### wholeText(element) {#wholetextelement} - -Selects the (unencoded) text of an element and its children, including any new lines and spaces, and returns a string of unencoded, un-normalized text. Use it in conjunction with parseHtml() and select() to provide an element as in the following example: `value.parseHtml().select("div.footer")[0].wholeText()`. - -###### innerHtml(element) {#innerhtmlelement} -Returns the [inner HTML](https://developer.mozilla.org/en-US/docs/Web/API/Element/innerHTML) of an HTML element. This will include text and children elements within the element selected. Use it in conjunction with parseHtml() and select() to provide an element. - -###### innerXml(element) {#innerxmlelement} -Returns the inner XML elements of an XML element. Does not return the text directly inside your chosen XML element - only the contents of its children. To select the direct text, use ownText(). To select both, use xmlText(). Use it in conjunction with parseXml() and select() to provide an element. - -###### ownText(element) {#owntextelement} -Returns the text directly inside the selected XML or HTML element only, ignoring text inside children elements (for this, use innerXml()). Use it in conjunction with a parser and select() to provide an element. - -## Array functions {#array-functions} - -###### length(a) {#lengtha} -Returns the size of an array, meaning the number of objects inside it. Arrays can be empty, in which case length() will return 0. - -###### slice(a, n from, n to (optional)) {#slicea-n-from-n-to-optional} -Returns a sub-array of a given array, from the first index provided and up to and excluding the optional last index provided. Remember that array objects are indexed starting at 0. If the to value is omitted, it is understood to be the end of the array. For example, `[0, 1, 2, 3, 4].slice(1, 3)` returns [ 1, 2 ], and `[ 0, 1, 2, 3, 4].slice(2)` returns [ 2, 3, 4 ]. Also works with strings; see [String functions](#slices-n-from-n-to-optional). - -###### get(a, n from, n to (optional)) {#geta-n-from-n-to-optional} -Returns a sub-array of a given array, from the first index provided and up to and excluding the optional last index provided. Remember that array objects are indexed starting at 0. - -If the to value is omitted, only one array item is returned, as a string, instead of a sub-array. To return a sub-array from one index to the end, you can set the to argument to a very high number such as `value.get(2,999)` or you can use something like `with(value,a,a.get(1,a.length()))` to count the length of each array. - -Also works with strings; see [String functions](#gets-n-from-n-to-optional). - -###### inArray(a, s) {#inarraya-s} -Returns true if the array contains the desired string, and false otherwise. Will not convert data types; for example, `[ 1, 2, 3, 4 ].inArray("3")` will return false. - -###### reverse(a) {#reversea} -Reverses the array. For example, `[ 0, 1, 2, 3].reverse()` returns the array [ 3, 2, 1, 0 ]. - -###### sort(a) {#sorta} -Sorts the array in ascending order. Sorting is case-sensitive, uppercase first and lowercase second. For example, `[ "al", "Joe", "Bob", "jim" ].sort()` returns the array [ "Bob", "Joe", "al", "jim" ]. - -###### sum(a) {#suma} -Return the sum of the numbers in the array. For example, `[ 2, 1, 0, 3 ].sum()` returns 6. - -###### join(a, sep) {#joina-sep} -Joins the items in the array with sep, and returns it all as a string. For example, `[ "and", "or", "not" ].join("/")` returns the string “and/or/not”. - -###### uniques(a) {#uniquesa} -Returns the array with duplicates removed. Case-sensitive. For example, `[ "al", "Joe", "Bob", "Joe", "Al", "Bob" ].uniques()` returns the array [ "Joe", "al", "Al", "Bob" ]. - -As of OpenRefine 3.4.1, uniques() reorders the array items it returns; in 3.4 beta 644 and onwards, it preserves the original order (in this case, [ "al", "Joe", "Bob", "Al" ]). - -## Date functions {#date-functions} - -###### now() {#now} - -Returns the current time according to your system clock, in the [ISO 8601 extended format](exploring#data-types) (converted to UTC). For example, 10:53am (and 00 seconds) on November 26th 2020 in EST returns [date 2020-11-26T15:53:00Z]. - -###### toDate(o, b monthFirst, s format1, s format2, ...) {#todateo-b-monthfirst-s-format1-s-format2-} - -Returns the inputted object converted to a date object. Without arguments, it returns the ISO 8601 extended format. With arguments, you can control the output format: -* monthFirst: set false if the date is formatted with the day before the month. -* formatN: attempt to parse the date using an ordered list of possible formats. Supply formats based on the [SimpleDateFormat](https://docs.oracle.com/javase/8/docs/api/java/text/SimpleDateFormat.html) syntax (and see the table below for a handy reference). - -For example, you can parse a column containing dates in different formats, such as cells with “Nov-09” and “11/09”, using `value.toDate('MM/yy','MMM-yy').toString('yyyy-MM')` and both will output “2009-11”. For another example, “1/4/2012 13:30:00” can be parsed into a date using `value.toDate('d/M/y H:m:s')`. - -| Letter | Date or Time Component | Presentation | Examples | -|-|-|-|-| -| G | Era designator | Text | AD | -| y | Year | Year | 1996; 96 | -| Y | [Week year](https://en.wikipedia.org/wiki/ISO_week_date#First_week) | Year | 2009; 09 | -| M | Month in year | Month | July; Jul; 07 | -| w | Week in year | Number | 27 | -| W | Week in month | Number | 2 | -| D | Day in year | Number | 189 | -| d | Day in month | Number | 10 | -| F | Day of week in month | Number | 2 | -| E | Day name in week | Text | Tuesday; Tue | -| u | Day number of week (1 = Monday, ..., 7 = Sunday) | Number | 1 | -| a | AM/PM marker | Text | PM | -| H | Hour in day (0-23) | Number | 0 | -| k | Hour in day (1-24) | Number | 24 | -| K | Hour in AM/PM (0-11) | Number | 0 | -| h | Hour in AM/PM (1-12) | Number | 12 | -| m | Minute in hour | Number | 30 | -| s | Second in minute | Number | 55 | -| S | Millisecond | Number | 978 | -| n | Nanosecond | Number | 789000 | -| z | Time zone | General time zone | Pacific Standard Time; PST; GMT-08:00 | -| Z | Time zone | RFC 822 time zone | \-0800 | -| X | Time zone | ISO 8601 time zone | \-08; -0800; -08:00 | - -###### diff(d1, d2, s timeUnit) {#diffd1-d2-s-timeunit} - -Given two dates, returns a number indicating the difference in a given time unit (see the table below). For example, `diff(("Nov-11".toDate('MMM-yy')), ("Nov-09".toDate('MMM-yy')), "weeks")` will return 104, for 104 weeks, or two years. The later date should go first. If the output is negative, invert d1 and d2. - -Also works with strings; see [diff() in string functions](#diffsd1-sd2-s-timeunit-optional). - -###### inc(d, n, s timeUnit) {#incd-n-s-timeunit} - -Returns a date changed by the given amount in the given unit of time (see the table below). The default unit is “hour”. A positive value increases the date, and a negative value moves it back in time. For example, if you want to move a date backwards by two months, use `value.inc(-2,"month")`. - -###### datePart(d, s timeUnit) {#datepartd-s-timeunit} - -Returns part of a date. The data type returned depends on the unit (see the table below). - -OpenRefine supports the following values for timeUnit: - -| Unit | Date part returned | Returned data type | Example using [date 2014-03-14T05:30:04.000789000Z] as value | -|-|-|-|-| -| years | Year | Number | value.datePart("years") → 2014 | -| year | Year | Number | value.datePart("year") → 2014 | -| months | Month | Number | value.datePart("months") → 2 | -| month | Month | Number | value.datePart("month") → 2 | -| weeks | Week of the month | Number | value.datePart("weeks") → 3 | -| week | Week of the month | Number | value.datePart("week") → 3 | -| w | Week of the month | Number | value.datePart("w") → 3 | -| weekday | Day of the week | String | value.datePart("weekday") → Friday | -| hours | Hour | Number | value.datePart("hours") → 5 | -| hour | Hour | Number | value.datePart("hour") → 5 | -| h | Hour | Number | value.datePart("h") → 5 | -| minutes | Minute | Number | value.datePart("minutes") → 30 | -| minute | Minute | Number | value.datePart("minute") → 30 | -| min | Minute | Number | value.datePart("min") → 30 | -| seconds | Seconds | Number | value.datePart("seconds") → 04 | -| sec | Seconds | Number | value.datePart("sec") → 04 | -| s | Seconds | Number | value.datePart("s") → 04 | -| milliseconds | Millseconds | Number | value.datePart("milliseconds") → 789 | -| ms | Millseconds | Number | value.datePart("ms") → 789 | -| S | Millseconds | Number | value.datePart("S") → 789 | -| n | Nanoseconds | Number | value.datePart("n") → 789000 | -| nano | Nanoseconds | Number | value.datePart("n") → 789000 | -| nanos | Nanoseconds | Number | value.datePart("n") → 789000 | -| time | Milliseconds between input and the [Unix Epoch](https://en.wikipedia.org/wiki/Unix_time) | Number | value.datePart("time") → 1394775004000 | - -## Math functions {#math-functions} - -For integer division and precision, you can use simple evaluations such as `1 / 2`, which is equivalent to `floor(1/2)` - that is, it returns only whole number results. If either operand is a floating point number, they both get promoted to floating point and a floating point result is returned. You can use `1 / 2.0` or `1.0 / 2` or `1.0 * x / y` (if you're working with variables of unknown contents). - -:::caution -Some of these math functions don't recognize integers when supplied as the first argument in dot notation (e.g., `5.cos()` simply returns 5 instead of the expected result). To ensure operations are successful, always wrap the first argument in brackets, such as `(value).cos()`. -::: - -|Function|Use|Example| -|-|-|-| -|`abs(n)`|Returns the absolute value of a number.|`abs(-6)` returns 6.| -|`acos(n)`|Returns the arc cosine of an angle, in the range 0 through [PI](https://docs.oracle.com/javase/8/docs/api/java/lang/Math.html#PI).|`acos(0.345)` returns 1.218557541697832.| -|`asin(n)`|Returns the arc sine of an angle in the range of -PI/2 through PI/2.|`asin(0.345)` returns 0.35223878509706474.| -|`atan(n)`|Returns the arc tangent of an angle in the range of -PI/2 through PI/2.|`atan(0.345)` returns 0.3322135507465967.| -|`atan2(n1, n2)`|Converts rectangular coordinates (n1, n2) to polar (r, theta). Returns number theta.|`atan2(0.345,0.6)` returns 0.5218342798144103.| -|`ceil(n)`|Returns the ceiling of a number.|`3.7.ceil()` returns 4 and `-3.7.ceil()` returns -3.| -|`combin(n1, n2)`|Returns the number of combinations for n2 elements as divided into n1.|`combin(20,2)` returns 190.| -|`cos(n)`|Returns the trigonometric cosine of a value.|`cos(5)` returns 0.28366218546322625.| -|`cosh(n)`|Returns the hyperbolic cosine of a value.|`cosh(5)` returns 74.20994852478785.| -|`degrees(n)`|Converts an angle from radians to degrees.|`degrees(5)` returns 286.4788975654116.| -|`even(n)`|Rounds the number up to the nearest even integer.|`even(5)` returns 6.| -|`exp(n)`|Returns [e](https://en.wikipedia.org/wiki/E_(mathematical_constant)) raised to the power of n.|`exp(5)` returns 148.4131591025766.| -|`fact(n)`|Returns the factorial of a number, starting from 1.|`fact(5)` returns 120.| -|`factn(n1, n2)`|Returns the factorial of n1, starting from n2.|`factn(10,3)` returns 280.| -|`floor(n)`|Returns the floor of a number.|`3.7.floor()` returns 3 and `-3.7.floor()` returns -4.| -|`gcd(n1, n2)`|Returns the greatest common denominator of two numbers.|`gcd(95,135)` returns 5.| -|`lcm(n1, n2)`|Returns the least common multiple of two numbers.|`lcm(95,135)` returns 2565.| -|`ln(n)`|Returns the natural logarithm of n.|`ln(5)` returns 1.6094379124341003.| -|`log(n)`|Returns the base 10 logarithm of n.|`log(5)` returns 0.6989700043360189.| -|`max(n1, n2)`|Returns the larger of two numbers.|`max(3,10)` returns 10.| -|`min(n1, n2)`|Returns the smaller of two numbers.|`min(3,10)` returns 3.| -|`mod(n1, n2)`|Returns n1 modulus n2. Note: `value.mod(9)` will work, whereas `74.mod(9)` will not work.|`mod(74, 9)` returns 2. | -|`multinomial(n1, n2 …(optional))`|Calculates the multinomial of one number or a series of numbers.|`multinomial(2,3)` returns 10.| -|`odd(n)`|Rounds the number up to the nearest odd integer.|`odd(10)` returns 11.| -|`pow(n1, n2)`|Returns n1 raised to the power of n2. Note: value.pow(3)` will work, whereas `2.pow(3)` will not work.|`pow(2, 3)` returns 8 (2 cubed) and `pow(3, 2)` returns 9 (3 squared). The square root of any numeric value can be called with `value.pow(0.5)`.| -|`quotient(n1, n2)`|Returns the integer portion of a division (truncated, not rounded), when supplied with a numerator and denominator.|`quotient(9,2)` returns 4.| -|`radians(n)`|Converts an angle in degrees to radians.|`radians(10)` returns 0.17453292519943295.| -|`randomNumber(n lowerBound, n upperBound)`|Returns a random integer in the interval between the lower and upper bounds (inclusively). Will output a different random number in each cell in a column.| -|`round(n)`|Rounds a number to the nearest integer.|`3.7.round()` returns 4 and `-3.7.round()` returns -4.| -|`sin(n)`|Returns the trigonometric sine of an angle.|`sin(10)` returns -0.5440211108893698.| -|`sinh(n)`|Returns the hyperbolic sine of an angle.|`sinh(10)` returns 11013.232874703393.| -|`sum(a)`|Sums the numbers in an array. Ignores non-number items. Returns 0 if the array does not contain numbers.|`sum([ 10, 2, three ])` returns 12.| -|`tan(n)`|Returns the trigonometric tangent of an angle.|`tan(10)` returns 0.6483608274590866.| -|`tanh(n)`|Returns the hyperbolic tangent of a value.|`tanh(10)` returns 0.9999999958776927.| - -## Other functions {#other-functions} - -###### type(o) {#typeo} -Returns a string with the data type of o, such as undefined, string, number, boolean, etc. For example, a [Transform](cellediting#transform) operation using `value.type()` will convert all cells in a column to strings of their data types. - -###### facetCount(choiceValue, s facetExpression, s columnName) {#facetcountchoicevalue-s-facetexpression-s-columnname} -Returns the facet count corresponding to the given choice value, by looking for the facetExpression in the choiceValue in columnName. For example, to create facet counts for the following table, we could generate a new column based on “Gift” and enter in `value.facetCount("value", "Gift")`. This would add the column we've named “Count”: - -| Gift | Recipient | Price | Count | -|-|-|-|-| -| lamp | Mary | 20 | 1 | -| clock | John | 57 | 2 | -| watch | Amit | 80 | 1 | -| clock | Claire | 62 | 2 | - -The facet expression, wrapped in quotes, can be useful to manipulate the inputted values before counting. For example, you could do a textual cleanup using fingerprint(): `(value.fingerprint()).facetCount(value.fingerprint(),"Gift")`. - -###### hasField(o, s name) {#hasfieldo-s-name} -Returns a boolean indicating whether o has a member field called [name](expressions#variables). For example, `cell.recon.hasField("match")` will return false if a reconciliation match hasn’t been selected yet, or true if it has. You cannot chain your desired fields: for example, `cell.hasField("recon.match")` will return false even if the above expression returns true). - -###### coalesce(o1, o2, o3, ...) {#coalesceo1-o2-o3-} -Returns the first non-null from a series of objects. For example, `coalesce(value, "")` would return an empty string “” if `value` was null, but otherwise return `value`. - -###### cross(cell, s projectName (optional), s columnName (optional)) {#crosscell-s-projectname-optional-s-columnname-optional} -Returns an array of zero or more rows in the project projectName for which the cells in their column columnName have the same content as the cell in your chosen column. For example, if two projects contained matching names, and you wanted to pull addresses for people by their names from a project called “People” you would apply the following expression to your column of names: -``` -cell.cross("People","Name").cells["Address"].value[0] -``` - -This would match your current column to the “Name” column in “People” and, using those matches, pull the respective “Address” value into your current project. - -You may need to do some data preparation with cross(), such as using trim() on your key columns or deduplicating values. - -The first argument will be interpreted as `cell.value` if set to `cell`. If you omit projectName and columnName, they will default to the current project and index column (number 0). - -Recipes and more examples for using cross() can be found [on our wiki](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#combining-datasets). diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/installing.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/installing.md deleted file mode 100644 index bf280a3e3..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/installing.md +++ /dev/null @@ -1,425 +0,0 @@ ---- -id: installing -title: Installing OpenRefine -sidebar_label: Installing ---- - -## System requirements {#system-requirements} - -OpenRefine does not require internet access to run its basic functions. Once you download and install it, it runs as a small web server on your own computer, and you access that local web server by using your browser. It only requires an internet connection to import data from the web, reconcile data using a web service, or export data to the web. - -OpenRefine requires three things on your computer in order to function: - -#### Compatible operating system {#compatible-operating-system} - -OpenRefine is designed to work with **Windows**, **Mac**, and **Linux** operating systems. [Our team releases packages for each](https://openrefine.org/download.html). - -#### Java {#java} - -[Java](https://java.com/en/download/) must be installed and configured on your computer to run OpenRefine. The Mac version of OpenRefine includes Java; new in OpenRefine 3.4, there is also a Windows package with Java included. - -If you install and start OpenRefine on a Windows computer without Java, it will automatically open up a browser window to the [Java downloads page](https://java.com/en/download/), and you can simply follow the instructions there. - -We recommend you [download](https://java.com/en/download/) and install Java before proceeding with the OpenRefine installation. Please note that OpenRefine works with Java 8 to Java 17 for OpenRefine 3.5. - -#### Compatible browser {#compatible-browser} - -OpenRefine works best on browsers based on Webkit, such as: - -* Google Chrome -* Chromium -* Opera -* Microsoft Edge - -We are aware of some minor rendering and performance issues on other browsers such as Firefox. We don't support Internet Explorer. If you are having issues running OpenRefine, see the [section on Running](running.md#troubleshooting). - -### Release versions {#release-versions} - -OpenRefine always has a [latest stable release](https://github.com/OpenRefine/OpenRefine/releases/latest), as well as some more recent developments available in beta, release candidate, or [snapshot releases](https://github.com/OpenRefine/OpenRefine-snapshot-releases/releases). If you are installing for the first time, we recommend [the latest stable release](https://github.com/OpenRefine/OpenRefine/releases/latest). - -If you wish to use an extension that is only compatible with an earlier version of OpenRefine, and do not require the latest features, you may find that [an older stable version is best for you](https://github.com/OpenRefine/OpenRefine/releases) in our list of releases. Look at later releases to see which security vulnerabilities are being fixed, in order to assess your own risk tolerance for using earlier versions. Look for “final release” versions instead of “beta” or “release candidate” versions. - -#### Unstable versions {#unstable-versions} - -If you need a recently developed function, and are willing to risk some untested code, you can look at [the most recent items in the list](https://github.com/OpenRefine/OpenRefine/releases) and see what changes appeal to you. - -“Beta” and “release candidate” versions may both have unreported bugs and are most suitable for people who are willing to help us troubleshoot these versions by [creating bug reports](https://github.com/OpenRefine/OpenRefine/issues). - -For the absolute latest development updates, see the [snapshot releases](https://github.com/OpenRefine/OpenRefine-snapshot-releases/releases). These are created with every commit. - -#### What’s changed {#whats-changed} - -Our [latest version is OpenRefine 3.5.0](https://github.com/OpenRefine/OpenRefine/releases/tag/3.5.0), released November 7th 2021. The major changes in this version are listed on the [3.5.0 release page](https://github.com/OpenRefine/OpenRefine/releases/tag/3.5.0) with the downloadable packages. - -You can find information about all OpenRefine versions on the [Releases page on Github](https://github.com/OpenRefine/OpenRefine/releases). - -:::info Other distributions -OpenRefine may also work in other environments, such as [Chromebooks](https://gist.github.com/organisciak/3e12e5138e44a2fed75240f4a4985b4f) where Linux terminals are available. Look at our list of [Other Distributions on the Downloads page](https://openrefine.org/download.html) for other ways of running OpenRefine, and refer to our contributor community to see new environments in development. -::: - -## Installing or upgrading {#installing-or-upgrading} -### Back up your data {#back-up-your-data} - -If you are upgrading from an older version of OpenRefine and have projects already on your computer, you should create backups of those projects before you install a new version. - -First, [locate your workspace directory](#where-is-data-stored). Then copy everything you find there and paste it into a folder elsewhere on your computer. - -For extra security you can [export your existing OpenRefine projects](exporting#export-a-project). - -:::caution -Take note of the [extensions](#installing-extensions) you have currently installed. They may not be compatible with the upgraded version of OpenRefine. Installations can be installed in two places, so be sure to check both your workspace directory and the existing installation directory. -::: - -### Install or upgrade OpenRefine {#install-or-upgrade-openrefine} - -If you are upgrading an existing OpenRefine installation, you can delete the old program files and install the new files into the same space. Do not overwrite the files as some obsolete files may be left over unnecessarily. - -:::caution -If you have extensions installed, do not delete the `webapp\extensions` folder where you installed them. You may wish to install extensions into the workspace directory instead of the program directory. There is no guarantee that extensions will be forward-compatible with new versions of OpenRefine, and we do not maintain extensions. -::: - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - - -Once you have downloaded the `.zip` file, extract it into a folder where you wish to store program files (such as `D:\Program Files\OpenRefine`). - -You can right-click on `openrefine.exe` or `refine.bat` and pin one of those programs to your Start Menu or create shortcuts for easier access. - - - - - -Once you have downloaded the `.dmg` file, open it and drag the OpenRefine icon onto the Applications folder icon (just like you would normally install Mac applications). - - - - - -The quick version: - -1. Install [Homebrew](http://brew.sh) -2. In Terminal enter ` brew cask install openrefine` -1. Then find OpenRefine in your Applications folder. - -The long version: - -[Homebrew](http://brew.sh) is a popular command-line package manager for Mac. Installing Homebrew is accomplished by pasting the installation command on the Homebrew website into a Terminal window. Once Homebrew is installed, applications like OpenRefine can be installed via a simple command. You can [install Homebrew from their website](http://brew.sh). - -###### Install {#install} - -Install OpenRefine with this command: - -``` -brew cask install openrefine -``` - -You should see output like this: - -``` -==> Downloading https://github.com/OpenRefine/OpenRefine/releases/download/2.7/openrefine-mac-2.7.dmg -########################### 100.0% - ==> Verifying checksum for Cask openrefine - ==> Installing Cask openrefine - ==> Moving App 'OpenRefine.app' to '/Applications/OpenRefine.app'. - 🍺 openrefine was successfully installed! -``` - -Behind the scenes, this command causes Homebrew to download the OpenRefine installer, verify the file’s authenticity (using a SHA-256 checksum), mount the disk image, copy the `OpenRefine.app` application bundle into the Applications folder, unmount the disk image, and save a copy of the installer and metadata about the installation for future use. - -If an existing `OpenRefine.app` is found in the Applications folder, Homebrew will not overwrite it, so installing via Homebrew requires either deleting or renaming previously installed copies. - -###### Uninstall {#uninstall} - -To uninstall OpenRefine, paste this command into the Terminal: - -``` - brew cask uninstall openrefine -``` - -You should see output like this: - -``` - ==> Removing App '/Applications/OpenRefine.app'. -``` - -###### Update {#update} - -To update to the latest version of OpenRefine, paste this command into the Terminal: - -``` - brew cask reinstall openrefine -``` - -You should see output like this: - -``` - ==> Downloading https://github.com/OpenRefine/OpenRefine/releases/download/2.7/openrefine-mac-2.7.dmg -########################### 100.0% - ==> Verifying checksum for Cask openrefine - ==> Removing App '/Applications/OpenRefine.app'. - ==> Moving App 'OpenRefine.app' to '/Applications/OpenRefine.app'. - 🍺 openrefine was successfully installed! -``` - -If you had previously installed the `openrefine-dev` cask (containing a release candidate) and you want to move to the stable release, you need to first uninstall the old cask and then install the new one: - -``` - brew cask uninstall openrefine-dev - brew cask install openrefine -``` - - - - -Once you have downloaded the `.tar.gz` file, open a shell, navigate to the folder containing the download, and type: - -``` -tar xzf openrefine-linux-3.5.0.tar.gz -``` - - - - - ---- - - -### Set where data is stored {#set-where-data-is-stored} - -OpenRefine stores data in two places: program files in the program directory, wherever it is you’ve installed it; and project files in what we call the “workspace directory.” You can access this folder easily from OpenRefine by going to the [home screen](running#the-home-screen) (at [http://127.0.0.1:3333/](http://127.0.0.1:3333/)) and clicking Browse workspace directory. - -By default this is: - - - - - -Depending on your version of Windows, the data is in one of these directories: -* `%appdata%\OpenRefine` -* `%localappdata%\OpenRefine` -* `C:\Documents and Settings\(user id)\Local Settings\Application Data\OpenRefine` -* `C:\Users\(user id)\AppData\Roaming\OpenRefine` -* `C:\Users\(user id)\AppData\Local\OpenRefine` -* `C:\Users\(user id)\OpenRefine` - -For older Google Refine releases, replace `OpenRefine` with `Google\Refine`. - -You can change this by adding this line to the file `openrefine.l4j.ini` and specifying your desired drive and folder path: - -``` --Drefine.data_dir=D:\MyDesiredFolder -``` - -If your folder path has spaces, use neutral quotation marks around it: - -``` --Drefine.data_dir="D:\My Desired Folder" -``` - -If the folder does not exist, OpenRefine will create it. - - - - - -``` -~/Library/Application Support/OpenRefine/ -``` - -For older versions, as Google Refine: - -``` -~/Library/Application Support/Google/Refine/ -``` - -Logging is to `/var/log/daemon.log` - grep for `com.google.refine.Refine`. - - - - - -``` -~/.local/share/openrefine/ -``` - -You can change this when you run OpenRefine from the terminal, by pointing to the workspace directory through the `-d` parameter: - -``` - ./refine -p 3333 -i 0.0.0.0 -m 6000M -d /My/Desired/Folder -``` - - - - - ---- - - -### Logs {#logs} - -OpenRefine does not currently output an error log, but because the OpenRefine console window is always open (on Linux and Windows) while OpenRefine runs in your browser, you can copy information from the console if an error occurs. - -Using a Mac, you can [run OpenRefine using the terminal](running#starting-and-exiting) in order to capture errors. - ---- - -## Increasing memory allocation {#increasing-memory-allocation} - -OpenRefine relies on having computer memory available to it to work effectively. If you are planning to work with large datasets, you may wish to set up OpenRefine to handle it at the outset. By “large” we generally mean one of the following indicators: -* more than one million total cells -* an input file size of more than 50 megabytes (MB) -* more than 50 [rows per record in records mode](running#records-mode) - -By default OpenRefine is set to operate with 1 gigabyte (GB) of memory (1024MB). If you feel that OpenRefine is running slowly, or you are getting “out of memory” errors (for example, `java.lang.OutOfMemoryError`), you can try allocating more memory. - -A good practice is to start with no more than 50% of whatever memory is left over after the estimated usage of your operating system, to leave memory for your browser to run. - -All of the settings below use a four-digit number to specify the megabytes (MB) used (actually [mebibytes](https://en.wikipedia.org/wiki/Mebibyte)). The default is usually 1024MB, but the new value doesn't need to be a multiple of 1024. - -:::info Dealing with large datasets -If your project is big enough to need more than the default amount of memory, consider turning off Parse cell text into numbers, dates, ... on import. It's convenient, but less efficient than explicitly converting any columns that you need as a data type other than the default “string” type. -::: - - - - - -#### Using openrefine.exe {#using-openrefineexe} - -If you run `openrefine.exe`, you will need to edit the `openrefine.l4j.ini` file found in the program directory and edit the line - -``` -# max memory memory heap size --Xmx1024M -``` - -The line “-Xmx1024M” defines the amount of memory available in megabytes. Change the number “1024” - for example, edit the line to “-Xmx2048M” to make 2048MB [2GB] of memory available. - -:::caution openrefine.exe not running? -Once you increase the memory allocation, you may find that you cannot run `openrefine.exe`. In this case, your computer needs a 64-bit version of [Java](https://www.java.com/en/download/help/index_installing.xml) (this is different from [Java JDK](#install-or-upgrade-java). Look for the “Windows Offline (64-bit)” download on the Downloads page and install that. Your system must also be set to use the 64-bit version of Java by [changing the Java configuration](https://www.java.com/en/download/help/update_runtime_settings.xml). -::: - -#### Using refine.bat {#using-refinebat} - -On Windows, OpenRefine can also be run by using the file `refine.bat` in the program directory. If you start OpenRefine using `refine.bat`, the memory available to OpenRefine can be specified either through command line options, or through the `refine.ini` file. - -To set the maximum amount of memory on the command line when using `refine.bat`, `cd` to the program directory, then type - -```refine.bat /m 2048m``` - -where “2048” is the maximum amount of MB that you want OpenRefine to use. - -To change the default that `refine.bat` uses, edit the `refine.ini` line that reads - -```REFINE_MEMORY=1024M``` - -Note that this file is only read if you use `refine.bat`, not `openrefine.exe`. - - - - -If you have downloaded the `.dmg` package and you start OpenRefine by double-clicking on it: - -* close OpenRefine -* control-click on the OpenRefine icon (opens the contextual menu) -* click on "show package content” (a finder window opens) -* open the “Contents” folder -* open the `Info.plist` file with any text editor (like Mac's default TextEdit) -* Change “-Xmx1024M” into, for example, “-Xmx2048M” or “-Xmx8G” -* save the file -* restart OpenRefine. - - - - -If you have downloaded the `.tar.gz` package and you start OpenRefine from the command line, add the “-m xxxxM” parameter like this: -`./refine -m 2048m` - -#### Setting a default {#setting-a-default} - -If you don't want to set this option on the command line each time, you can also set it in the `refine.ini` file. Edit the line - -``` -REFINE_MEMORY=1024M -``` - -Make sure it is not commented out (that is, that the line doesn't start with a “#” character), and change “1024” to a higher value. Save the file, and when you next start OpenRefine it will use this value. - - - - - ---- - - -## Installing extensions {#installing-extensions} - -Extensions have been created by our contributor community to add functionality or provide convenient shortcuts for common uses of OpenRefine. [We list extensions we know about on our downloads page](https://openrefine.org/download.html). - -:::info Contributing extensions -If you’d like to create or modify an extension, [see our developer documentation here](https://github.com/OpenRefine/OpenRefine/wiki/Documentation-For-Developers). If you’re having a problem, [use our downloads page](https://openrefine.org/download.html) to go to the extension’s page and report the issue there. -::: - -### Two ways to install extensions {#two-ways-to-install-extensions} - -You can [install extensions in one of two places](#set-where-data-is-stored): - -* Into your OpenRefine program folder, so they will only be available to that version/installation of OpenRefine (meaning the extension will not run if you upgrade OpenRefine), or -* Into your workspace, where your projects are stored, so they will be available no matter which version of OpenRefine you’re using. - -We provide these options because you may wish to reinstall a given extension manually each time you upgrade OpenRefine, in order to be sure it works properly. - -### Find the right place to install {#find-the-right-place-to-install} - -If you want to install the extension into the program folder, go to your program directory and then go to `webapp\extensions` (or create it if not does not exist). - -If you want to install the extension into your workspace, you can: -* launch OpenRefine and click Open Project in the sidebar -* At the bottom of the screen, click Browse workspace directory -* A file-explorer or finder window will open in your workspace -* Create a new folder called “extensions” inside the workspace if it does not exist. - -You can also [find your workspace on each operating system using these instructions](#set-where-data-is-stored). - -### Install the extension {#install-the-extension} - -Some extensions have their own instructions: make sure you read the documentation before you begin installing. - -Some extensions may have multiple versions, to match OpenRefine versions, so be sure to choose the right release for your installation. If you have questions about compatibility or want to request or voice your support for an update, [use our downloads page](https://openrefine.org/download.html) to go to the extension’s page and report the issue there. - -Generally, the installation process will be: - -* Download the extension (usually as a zip file from GitHub) -* Extract the zip contents into the “extensions” directory, making sure all the contents go into one folder with the name of the extension -* Start (or restart) OpenRefine. - -To confirm that installation was a success, follow the instructions provided by the extension. Each extension will appear in its own way inside the OpenRefine interface. Make sure you read its documentation to know where the functionality will appear, such as under specific dropdown menus. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/jythonclojure.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/jythonclojure.md deleted file mode 100644 index a93d18155..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/jythonclojure.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -id: jythonclojure -title: Jython & Clojure -sidebar_label: Jython & Clojure ---- - -## Jython {#jython} - -Jython 2.7.2 comes bundled with the default installation of OpenRefine 3.4.1. You can add libraries and code by following [this tutorial](https://github.com/OpenRefine/OpenRefine/wiki/Extending-Jython-with-pypi-modules). A large number of Python files (`.py` or `.pyc`) are compatible. - -Python code that depends on C bindings will not work in OpenRefine, which uses Java / Jython only. Since Jython is essentially Java, you can also import Java libraries and utilize those. - -You will need to restart OpenRefine, so that new Jython or Python libraries are initialized during startup. - -OpenRefine now has [most of the Jsoup.org library built into GREL functions](grelfunctions#jsoup-xml-and-html-parsing-functions) for parsing and working with HTML and XML elements. - -### Syntax {#syntax} - -Expressions in Jython must have a `return` statement: - -``` - return value[1:-1] -``` - -``` - return rowIndex%2 -``` - -Fields have to be accessed using the bracket operator rather than dot notation: - -``` - return cells["col1"]["value"] -``` - -For example, to access the [edit distance](reconciling#reconciliation-facets) between a reconciled value and an original cell value using [recon variables](#reconciliation): - -``` - return cell["recon"]["features"]["nameLevenshtein"] -``` - -To return the lower case of `value` (if the value is not null): - -``` - if value is not None: - return value.lower() - else: - return None -``` - -### Tutorials {#tutorials} -- [Extending Jython with pypi modules](https://github.com/OpenRefine/OpenRefine/wiki/Extending-Jython-with-pypi-modules) -- [Working with phone numbers using Java libraries inside Python](https://github.com/OpenRefine/OpenRefine/wiki/Jython#tutorial---working-with-phone-numbers-using-java-libraries-inside-python) - -Full documentation on the Jython language can be found on its official site: [http://www.jython.org](http://www.jython.org). - -## Clojure {#clojure} - -Clojure 1.10.1 comes bundled with the default installation of OpenRefine 3.4.1. At this time, not all [variables](expressions#variables) can be used with Clojure expressions: only `value`, `row`, `rowIndex`, `cell`, and `cells` are available. - -For example, functions can take the form -``` -(.. value (toUpperCase) ) -``` - -Or can look like -``` -(-> value (str/split #" ") last ) -``` - -which functions like `value.split(" ")` in GREL. - -For help with syntax, see the [Clojure website's guide to syntax](https://clojure.org/guides/learn/syntax). - -User-contributed Clojure recipes can be found on our wiki at [https://github.com/OpenRefine/OpenRefine/wiki/Recipes#11-clojure](https://github.com/OpenRefine/OpenRefine/wiki/Recipes#11-clojure). - -Full documentation on the Clojure language can be found on its official site: [https://clojure.org/](https://clojure.org/). \ No newline at end of file diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/reconciling.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/reconciling.md deleted file mode 100644 index 0d14e80ad..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/reconciling.md +++ /dev/null @@ -1,242 +0,0 @@ ---- -id: reconciling -title: Reconciling -sidebar_label: Reconciling ---- - -## Overview {#overview} - -Reconciliation is the process of matching your dataset with that of an external source. Datasets for comparison might be produced by libraries, archives, museums, academic organizations, scientific institutions, non-profits, or interest groups. You can also reconcile against user-edited data on [Wikidata or other Wikibase instances](wikibase/reconciling), or reconcile against [a local dataset that you yourself supply](https://github.com/OpenRefine/OpenRefine/wiki/Reconcilable-Data-Sources#local-services). - -To reconcile your OpenRefine project against an external dataset, that dataset must offer a web service that conforms to the [Reconciliation Service API standards](https://reconciliation-api.github.io/specs/0.1/). - -You may wish to reconcile in order to: -* fix spelling or variations in proper names -* clean up manually-entered subject headings against authorities such as the [Library of Congress Subject Headings](https://id.loc.gov/authorities/subjects.html) (LCSH) -* link your data to an existing dataset -* add to an editable platform such as [Wikidata](https://www.wikidata.org) -* or see whether entities in your project appear in some specific list, such as the [Panama Papers](https://aleph.occrp.org/datasets/734). - -Reconciliation is semi-automated: OpenRefine matches your cell values to the reconciliation information as best it can, but human judgment is required to review and approve the results. Reconciling happens by default through string searching, so typos, whitespace, and extraneous characters will have an effect on the results. You may wish to [clean and cluster](cellediting) your data before reconciliaton. - -:::info Working iteratively -We recommend planning your reconciliation operations as iterative: reconcile multiple times with different settings, and with different subgroups of your data. -::: - -## Sources {#sources} - -Start with [this current list of reconcilable authorities](https://reconciliation-api.github.io/testbench/), which includes instructions for adding new services via Wikidata editing if you have one to add. - -OpenRefine maintains a [further list of sources on the wiki](https://github.com/OpenRefine/OpenRefine/wiki/Reconcilable-Data-Sources), which can be edited by anyone. This list includes ways that you can reconcile against a [local dataset](https://github.com/OpenRefine/OpenRefine/wiki/Reconcilable-Data-Sources#local-services). - -Other services may exist that are not yet listed in these two places: for example, the [310 datasets hosted by the Organized Crime and Corruption Reporting Project (OCCRP)](https://aleph.occrp.org/datasets/) each have their own reconciliation URL, or you can reconcile against their entire database with the URL [shared on the reconciliation API list](https://reconciliation-api.github.io/testbench/). For another example, you can reconcile against the entire Virtual International Authority File (VIAF) dataset, or [only the contributions from certain institutions](http://refine.codefork.com/). Search online to see if the authority you wish to reconcile against has an available service, or whether you can download a copy to reconcile against locally. - -OpenRefine includes Wikidata reconciliation in the installation package - see the [Wikibase](wikibase/reconciling) page for more information particular to that service. Extensions can add reconciliation services, and can also add enhanced reconciliation capacities. Check the list of extensions on the [Downloads page](https://openrefine.org/download.html) for more information. - -Each source will have its own documentation on how it provides reconciliation. The table on [the reconciliation API list](https://reconciliation-api.github.io/testbench/) indicates whether your chosen service supports the features described below. Refer to the service's documentation if you have questions about its behaviors and which OpenRefine features it supports. - -In addition to the reconciliation services mentioned above, you may also choose to build your own service. You can either start from scratch using the [API specification](https://reconciliation-api.github.io/specs/latest/) or use one of the frameworks mentioned in the [Reconciliation census](https://reconciliation-api.github.io/census/services/). - -Of particular note is [reconcile-csv](http://okfnlabs.org/reconcile-csv/) which allows you to build a reconciliation service from a simple CSV file. Thus if you wanted to reconcile one OpenRefine project against another, you'd simply need to export the target project as a CSV, point `reconcile-csv` at it and you're good to go. A somewhat newer port of this project written in Python can be found at [csv-reconcile](https://github.com/gitonthescene/csv-reconcile) which is more configurable and defaults to parsing tab separated files for convenience. - -Similiarly, you may choose to export some SPARQL output to a TSV to limit the scope of values you're reconciling against and/or for better peformance. - -## Getting started {#getting-started} - -Choose a column to reconcile and use its dropdown menu to select ReconcileStart reconciling. If you want to reconcile only some cells in that column, first use filters and facets to isolate them. - -In the reconciliation window, you will see Wikidata offered as a default service. To add another service, click Add Standard Service... and paste in the URL of a [service](#sources). You should see the name of the service appear in the list of Services if the URL is correct. - -![The reconciliation window.](/img/reconcilewindow.png) - -Once you select a service, your selected column may be sampled in order to suggest [“types” (categories)](#reconciling-by-type) to reconcile against. Other services will suggest their available types without sampling, and some services have no types. - -For example, if you had a list of artists represented in a gallery collection, you could reconcile their names against the Getty Research Institute’s [Union List of Artist Names (ULAN)](https://www.getty.edu/research/tools/vocabularies/ulan/). The same [Getty reconciliation URL](https://services.getty.edu/vocab/reconcile/) will offer you ULAN, AAT (Art and Architecture Thesaurus), and TGN (Thesaurus of Geographic Names). - -![The reconciliation window with types.](/img/reconcilewindow2.png) - -Refer to the [documentation specific to the reconciliation service](https://reconciliation-api.github.io/testbench/) to learn whether types are offered, which types are offered, and which one is most appropriate for your column. You may wish to facet your data and reconcile batches against different types if available. - -Reconciliation can be a time-consuming process, especially with large datasets. We suggest starting with a small test batch. There is no throttle (delay between requests) to set for the reconciliation process. The amount of time will vary for each service, and vary based on the options you select during the process. - -When the process is done, you will see the reconciliation data in the cells. -If the cell was successfully matched, it displays text as a single dark blue link. In this case, the reconciliation is confident that the match is correct, and you should not have to check it manually. -If there is no clear match, one or more candidates are displayed, together with their reconciliation score, with the text in light blue links. You will need to select the correct one. - -For each matching decision you make, you have two options: match this cell only (one checkmark), or also use the same identifier for all other cells containing the same original string (two checkmarks). - -For services that offer the [“preview entities” feature](https://reconciliation-api.github.io/testbench/), you can hover your mouse over the suggestions to see more information about the candidates or matches. Each participating service (and each type) will deliver different structured data that may help you compare the candidates. - -For example, the Getty ULAN shows an artist’s discipline, nationality, and birth and death years: - -![Hovering over matches.](/img/reconcilehover.png) - -Hovering over the suggestion will also offer the two matching options as buttons. - -For matched values (those appearing as dark blue links), the underlying cell value has not been altered - the cell is storing both the original string and the matched entity link at the same time. If you were to copy your column to a new column at this point using `value`, for example, the reconcilation data would not transfer - only the original strings. You can learn more about how OpenRefine stores different pieces of information in each cell in [the Variables section specific to reconciliation data](expressions#reconciliation). - -For each cell, you can manually “Create new item,” which will take the cell’s original value and apply it, as though it is a match. This will not become a dark blue link, because at this time there is nothing to link to: it is a draft entity stored only in your project. You can use this feature to prepare these entries for eventual upload to an editable service such as [Wikibase](wikibase/overview), but most services do not yet support this feature. - -### Reconciliation facets {#reconciliation-facets} - -Under ReconcileFacets there are a number of reconciliation-specific faceting options. OpenRefine automatically creates two facets when you reconcile some cells. - -One is a numeric facet for “best candidate's score,” the range of reconciliation scores of only the best candidate of each cell. Higher scores mean better matches, although each service calculates scores differently and has a different range. You can facet for higher scores using the numeric facet, and then approve them all in bulk, by using Reconcile[Actions](#reconciliation-actions)Match each cell to its best candidate. - -There is also a “judgment” facet created, which lets you filter for the cells that haven't been matched (pick “none” in the facet). As you process each cell, its judgment changes from “none” to “matched” and it disappears from the view. - -You can add other facets by selecting ReconcileFacets on your reconciled column. You can facet by: - -* your judgments (“matched,” or “none” for unreconciled cells, or “new” for entities you've created) -* the action you’ve performed on that cell (chosen a “single” match, or set a “mass” match, or no action, which appears as “unknown”) -* the timestamps on the edits you’ve made so far (these appear as millisecond counts since an arbitrary point: they can be sorted alphabetically to move forward and back in time). - -You can facet only the best candidates for each cell, based on: -* the score (calculated based on each service's own methods) -* the edit distance (using the [Levenshtein distance](cellediting#nearest-neighbor), a number based on how many single-character edits would be required to get your original value to the candidate value, with a larger value being a greater difference) -* the word similarity. - -Word similarity is calculated as a percentage based on how many words (excluding [stop words](https://en.wikipedia.org/wiki/Stop_word)) in the original value match words in the candidate. For example, the value “Maria Luisa Zuloaga de Tovar” matched to the candidate “Palacios, Luisa Zuloaga de” results in a word similarity value of 0.6, or 60%, or 3 out of 5 words. Cells that are not yet matched to one candidate will show as 0.0). - -You can also look at each best candidate’s: -* type (the ones you have selected in successive reconciliation attempts, or other types returned by the service based on the cell values) -* type match (“true” if you selected a type and it succeeded, “false” if you reconciled against no particular type, and “(no type)” if it didn’t reconcile) -* name match (“true” if you’ve matched, “false” if you haven’t yet chosen from the candidates, or “(unreconciled)” if it didn’t reconcile). - -These facets are useful for doing successive reconciliation attempts, against different types, and with different supplementary information. The information represented by these facets are held in the cells themselves and can be called using the [reconciliation variables](expressions#reconciliation) available in expressions. - -### Reconciliation actions {#reconciliation-actions} - -You can use the ReconcileActions menu options to perform bulk changes (which will apply only to your currently viewed set of rows or records): -* Match each cell to its best candidate (by highest score) -* Create a new item for each cell (discard any suggested matches) -* Create one new item for similar cells (a new entity will be created for each unique string) -* Match all filtered cells to... (a specific item from the chosen service, via a search box; only works with services that support the “suggest entities” property) -* Discard all reconciliation judgments (reverts back to multiple candidates per cell, including cells that may have been auto-matched in the original reconciliation process) -* Clear reconciliation data, reverting all cells back to their original values. - -The other options available under Reconcile are: -* Copy reconciliation data... (to an existing column: if the original values in your reconciliation column are identical to those in your chosen column, the matched and new cells will copy over; unmatched values will not change) -* [Use values as identifiers](#reconciling-with-unique-identifiers) (if you are reconciling with unique identifiers instead of by doing string searches) -* [Add entity identifiers column](#add-entity-identifiers-column). - -## Reconciling with unique identifiers {#reconciling-with-unique-identifiers} - -Reconciliation services use unique identifiers for their entities. For example, the 14th Dalai Lama has the VIAF ID [38242123](https://viaf.org/viaf/38242123/) and the Wikidata ID [Q17293](https://www.wikidata.org/wiki/Q37349). You can supply these identifiers directly to your chosen reconciliation service in order to pull more data, but these strings will not be “reconciled” against the external dataset. - -Select the column with unique identifiers and apply the operation ReconcileUse values as identifiers. This will bring up the list of reconciliation services you have already added (to add a new service, open the Start reconciling... window first). If you use this operation on a column of IDs, you will not have access to the usual reconciliation settings. - -Matching identifiers does not validate them. All cells will appear as dark blue “confirmed” matches. You should check before this operation that the identifiers in the column exist on the target service. - -You may get false positives, which you will need to hover over or click on to identify: - -![Hovering over an error.](/img/reconcileIDerror.png) - -## Reconciling by type {#reconciling-by-type} - -Reconciliation services, once added to OpenRefine, may suggest types from their databases. These types will usually be whatever the service specializes in: people, events, places, buildings, tools, plants, animals, organizations, etc. - -Reconciling against a type may be faster and more accurate, but may result in fewer matches. Some services have hierarchical types (such as “mammal” as a subtype of “animal”). When you reconcile against a more specific type, unmatched values may fall back to the broader type; other services will not do this, so you may need to perform successive reconciliation attempts against different types. Refer to the documentation specific to the reconciliation service to learn more. - -When you select a service from the list, OpenRefine will load some or all available types. Some services will sample the first ten rows of your column to suggest types (check the [“Suggest types” column](https://reconciliation-api.github.io/testbench/)). You will see a service’s types in the reconciliation window: - -![Reconciling using a type.](/img/reconcile-by-type.png) - -In this example, “Person” and “Corporate Name” are potential types offered by the reconciliation API for VIAF. You can also use the Reconcile against type: field to enter in another type that the service offers. When you start typing, this field may search and suggest existing types. For VIAF, you could enter “/book/book” if your column contained publications. You may need to enter the service's own strings precisely instead of attempting to search for a match. - -Types are structured to fit their content: the Wikidata “human” type, for example, can include fields for birth and death dates, nationality, etc. The VIAF “person” type can include nationality and gender. You can use this to [include more properties](#reconciling-with-additional-columns) and find better matches. - -If your column doesn’t fit one specific type offered, you can Reconcile against no particular type. This may take longer. - -We recommend working in batches and reconciling against different types, moving from specific to broad. You can create a facet for Best candidate’s types facet to see which types are being represented. Some candidates may return more than one type, depending on the service. Types may appear in facets by their unique IDs, rather than by their semantic labels (for example, Q5 for “human” in Wikidata). - -## Reconciling with additional columns {#reconciling-with-additional-columns} - -Some of your cells may be ambiguous, in the sense that a string can point to more than one entity: there are dozens of places called “Paris” and many characters, people, and pieces of culture, too. Selecting non-geographic or more localized types can help narrow that down, but if your chosen service doesn't provide a useful type, you can include more properties that make it clear whether you're looking for Paris, France. - -![Reconciling sometimes turns up ambiguous matches.](/img/reconcileParis.gif) - -Including supplementary information can be useful, depending on the service (such as including birthdate information about each person you are trying to reconcile). You can re-reconcile unmatched cells with additional properties, in the right side of the Start reconciling window, under “Also use relevant details from other columns.” The column names in your project will appear in the reconciliation window, with an Include? checkbox next to each one. - -Fill in the As Property field with the type of information you are including. When you start typing, potential fields may pop up (depending on the [“suggest properties” feature](https://reconciliation-api.github.io/testbench/)), such as “birthDate” in the case of ULAN or “Geburtsdatum” in the case of Integrated Authority File (GND). Use the documentation for your chosen service to identify the fields in their terms. - -Some services will not be able to search for the exact name of your desired As Property entry, but you can still manually supply the field name. Refer to the service to choose the most appropriate field, and make sure you enter it correctly. - -![Including a birth-date type.](/img/reconcile-with-property.png) - -## Fetching more data {#fetching-more-data} - -One reason to reconcile to some external service is that it allows you to pull data from that service into your OpenRefine project. There are three ways to do this: - -* Add identifiers for your values -* Add columns from reconciled values -* Add column by fetching URLs. - -### Add entity identifiers column {#add-entity-identifiers-column} - -Once you have selected matches for your cells, you can retrieve the unique identifiers for those cells and create a new column for these, with ReconcileAdd entity identifiers column. You will be asked to supply a column name. New items and other unmatched cells will generate null values in this column. - -### Add columns from reconciled values {#add-columns-from-reconciled-values} - -If the reconciliation service supports [data extension](https://reconciliation-api.github.io/testbench/), then you can augment your reconciled data with new columns using Edit columnAdd columns from reconciled values.... - -For example, if you have a column of chemical elements identified by name, you can fetch categorical information about them such as their atomic number and their element symbol: - -![A screenshare of elements fetching related information.](/img/reconcileelements.gif) - -Once you have chosen reconciliation matches for your cells, selecting Add column from reconciled values... will bring up a window to choose which related information you’d like to import into new columns. You can manually enter desired properties, or select from a list of suggestions. - -The quality of the suggested properties will depend on how you have reconciled your data beforehand: reconciling against a specific type will provide you with the associated properties of that type. For example, GND suggests elements about the “people” type after you've reconciled with it, such as their parents, native languages, children, etc. - -![A screenshot of available properties from GND.](/img/reconcileGND.png) - -If you have left any values unreconciled in your column, you will see “<not reconciled>” in the preview. These will generate blank cells if you continue with the column addition process. - -This process may pull more than one property per row in your data (such as multiple occupations), so you may need to switch into records mode after you've added columns. - -### Add columns by fetching URLs {#add-columns-by-fetching-urls} - -If the reconciliation service cannot extend data, look for a generic web API for that data source, or a structured URL that points to their dataset entities via unique IDs (such as “https://viaf.org/viaf/000000”). You can use the Edit column[Add column by fetching URLs](columnediting#add-column-by-fetching-urls) operation to call this API or URL with the IDs obtained from the reconciliation process. This will require using [expressions](expressions). - -You may not want to pull the entire HTML content of the pages at the ends of these URLs, so look to see whether the service offers a metadata endpoint, such as JSON-formatted data. You can either use a column of IDs, or you can pull the ID from each matched cell during the fetching process. - -For example, if you have reconciled artists to the Getty's ULAN, and [have their unique ULAN IDs as a column](#add-entity-identifiers-column), you can generate a new column of JSON-formatted data by using Add column by fetching URLs and entering the GREL expression `"http://vocab.getty.edu/" + value + ".json"`. For this service, the unique IDs are formatted “ulan/000000” and so the generated URLs look like “http://vocab.getty.edu/ulan/000000.json”. - -Alternatively, you can insert the ID directly from the matched column's reconciliation variables, using a GREL expression like `“http://vocab.getty.edu/” + cell.recon.match.id + “.json”` instead. - -Remember to set an appropriate throttle and to refer to the service documentation to ensure your compliance with their terms. See [the section about this operation](columnediting#add-column-by-fetching-urls) to learn more about the fetching process. - -## Keep all the suggestions made {#keep-all-the-suggestions-made} - -To generate a list of each suggestion made, rather than only the best candidate, you can use a [GREL expression](expressions#GREL). Go to Edit columnAdd column based on this column. To create a list of all the possible matches, use something like - -``` -forEach(cell.recon.candidates,c,c.name).join(", ") -``` - -To get the unique identifiers of these matches instead, use - -``` -forEach(cell.recon.candidates,c,c.id).join(", ") -``` - -This information is stored as a string, without any attached reconciliation information. - -## Writing reconciliation expressions {#writing-reconciliation-expressions} - -OpenRefine supplies a number of variables related specifically to reconciled values. These can be used in GREL and Jython expressions. For example, some of the reconciliation variables are: - -* `cell.recon.match.id` or `cell.recon.match.name` for matched values -* `cell.recon.best.name` or `cell.recon.best.id` for best-candidate values -* `cell.recon.candidates` for all listed candidates of each cell -* `cell.recon.judgment` (the values used in the “judgment” facet) -* `cell.recon.judgmentHistory` (the values used in the “judgment action timestamp” facet) -* `cell.recon.matched` (a “true” or “false” value) - -You can find out more in the [reconciliaton variables](expressions#reconciliaton-variables) section. - -## Exporting reconciled data {#exporting-reconciled-data} - -Once you have data that is reconciled to existing entities online, you may wish to export that data to a user-editable service such as Wikidata. See the section on [uploading your edits to Wikidata or other Wikibase instances](wikibase/uploading) for more information, or the section on [exporting](exporting) to see other formats OpenRefine can produce. - -You can share reconciled data in progress through a [project export or import](exporting#export-a-project), with some preparation. The importing user needs to have the appropriate reconciliation services installed on their OpenRefine instance (by going to Start reconciling and clicking on Add Standard Service...) in advance of opening the project, in order to use candidate and match links. Otherwise, the links will be broken and the user will need to add the reconciliation service and re-reconcile the columns in question. [Wikidata](wikibase/reconciling) reconciliation data can be shared more easily as the service comes bundled with OpenRefine. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/running.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/running.md deleted file mode 100644 index e23317562..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/running.md +++ /dev/null @@ -1,509 +0,0 @@ ---- -id: running -title: Running OpenRefine -sidebar_label: Running ---- - -## Starting and exiting {#starting-and-exiting} - -OpenRefine does not require internet access to run its basic functions. Once you download and install it, it runs as a small web server on your own computer, and you access that local web server by using your browser. - -You will see a command line window open when you run OpenRefine. Ignore that window while you work on datasets in your browser. - -No matter how you start OpenRefine, it will load its interface in your computer’s default browser. If you would like to use another browser instead, start OpenRefine and then point your chosen browser at the home screen: [http://127.0.0.1:3333/](http://127.0.0.1:3333/). - -OpenRefine works best on browsers based on Webkit, such as: -* Google Chrome -* Chromium -* Opera -* Microsoft Edge - -We are aware of some minor rendering and performance issues on other browsers such as Firefox. We don't support Internet Explorer. - -You can view and work on multiple projects at the same time by simply having multiple tabs or browser windows open. From the Open Project screen, you can right-click on project names and open them in new tabs or windows. - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - - -#### With openrefine.exe {#with-openrefineexe} -You can run OpenRefine by double-clicking `openrefine.exe` or calling it from the command line. - -If you want to [modify the way `openrefine.exe` opens](#starting-with-modifications), you can edit the `openrefine.l4j.ini` file. - -#### With refine.bat {#with-refinebat} -On Windows, OpenRefine can also be run by using the file `refine.bat` in the program directory. If you start OpenRefine using `refine.bat`, you can do so by opening the file itself, or by calling it from the command line. - -If you call `refine.bat` from the command line, you can [start OpenRefine with modifications](#starting-with-modifications). -If you want to modify the way `refine.bat` opens through double-clicking or using a shortcut, you can edit the `refine.ini` file. - -#### Exiting {#exiting} - -To exit OpenRefine, close all the browser tabs or windows, then navigate to the command line window. To close this window and ensure OpenRefine exits properly, hold down `Control` and press `C` on your keyboard. This will save any last changes to your projects. - - - - - -You can find OpenRefine in your Applications folder, or you can open it using Terminal. - -To run OpenRefine using Terminal: -* Find the OpenRefine application / icon in Finder -* Control-click on the icon and select “Show Package Contents” from the context menu -* This should open a new Finder menu: navigate into the “MacOS” folder -* Control-click on “JavaAppLauncher” -* Choose “Open With” from the menu, and select “Terminal.” - -To exit, close all your OpenRefine browser tabs, go back to the terminal window and press `Command` and `Q` to close it down. - -:::caution Problems starting? -If you are using an older version of OpenRefine or are on an older version of MacOS, [check our Wiki for solutions to problems with MacOS](https://github.com/OpenRefine/OpenRefine/wiki/Installation-Instructions#macos). -::: - - - - - -Use a terminal to launch OpenRefine. First, navigate to the installation folder. Then call the program: - -``` -cd openrefine-3.4.1 - ./refine -``` - -This will start OpenRefine and open your browser to the home screen. - -To exit, close all the browser tabs, and then press `control` and `C` in the terminal window. - -:::caution Did you get a JAVA_HOME error? -“Error: Could not find the ‘java’ executable at ‘’, are you sure your JAVA_HOME environment variable is pointing to a proper java installation?” - -If you see this error, you need to [install and configure a JDK package](installing#linux), including setting up `JAVA_HOME`. -::: - - - - - ---- - -### Troubleshooting {#troubleshooting} - -If you are having problems connecting to OpenRefine with your browser, [check our Wiki for information about browser settings and operating-system issues](https://github.com/OpenRefine/OpenRefine/wiki/FAQ#i-am-having-trouble-connecting-to-openrefine-with-my-browser). - -### Starting with modifications {#starting-with-modifications} - -When you run OpenRefine from a command line, you can change a number of default settings. - - - - - -On Windows, use a slash: - -``` -C:>refine /i 127.0.0.2 /p 3334 -``` - -Get a list of all the commands with `refine /?`. - -|Command|Use|Syntax example| -|---|---|---| -|/w|Path to the webapp|refine /w /path/to/openrefine| -|/m|Memory maximum heap|refine /m 6000M| -|/p|Port|refine /p 3334| -|/i|Interface (IP address, or IP and port)|refine /i 127.0.0.2:3334| -|/H|HTTP host to expect on incoming requests|refine /H openrefine.internal| -|/d|Enable debugging (on port 8000)|refine /d| -|/x|Enable JMX monitoring for Jconsole and JvisualVM|refine /x| - - - - - -You cannot start the Mac version with modifications using Terminal, but you can modify the way the application starts with [settings within files](#modifications-set-within-files). - - - - - -To see the full list of command-line options, run `./refine -h`. - -|Command|Use|Syntax example| -|---|---|---| -|-w|Path to the webapp|./refine -w /path/to/openrefine| -|-d|Path to the workspace|./refine -d /where/you/want/the/workspace| -|-m|Memory maximum heap|./refine -m 6000M| -|-p|Port|./refine -p 3334| -|-i|Interface (IP address, or IP and port)|./refine -i 127.0.0.2:3334| -|-H|HTTP host to expect on incoming requests|./refine -H openrefine.internal| -|-k|Add a Google API key|./refine -k YOUR_API_KEY| -|-v|Verbosity (from low to high: error,warn,info,debug,trace)|./refine -v info| -|-x|Additional Java configuration parameters (see Java documentation)|| -|--debug|Enable debugging (on port 8000)|./refine --debug| -|--jmx|Enable JMX monitoring for Jconsole and JvisualVM|./refine --jmx| - - - - - ---- - -#### Modifications set within files {#modifications-set-within-files} - -On Windows, you can modify the way `openrefine.exe` runs by editing `openrefine.l4j.ini`; you can modify the way `refine.bat` runs by editing `refine.ini`. - -You can modify the Mac application by editing `info.plist`. - -On Linux, you can edit `refine.ini`. - -Some settings, such as changing memory allocations, are already set inside these files, and all you have to do is change the values. Some lines need to be un-commented to work. - -For example, inside `refine.ini`, you should see: -``` -no_proxy="localhost,127.0.0.1" -#REFINE_PORT=3334 -#REFINE_HOST=127.0.0.1 -#REFINE_WEBAPP=main\webapp - -# Memory and max form size allocations -#REFINE_MAX_FORM_CONTENT_SIZE=1048576 -REFINE_MEMORY=1400M - -# Set initial java heap space (default: 256M) for better performance with large datasets -REFINE_MIN_MEMORY=1400M -... -``` - -##### JVM preferences {#jvm-preferences} - -Further modifications can be performed by using JVM preferences. These JVM preferences are different options and have different syntax than the key/value descriptions used on the command line. - -Some of the most common keys (with their defaults) are: -* The project [autosave](starting#autosaving) frequency: `-Drefine.autosave` (5 [minutes]) -* The workspace director: `-Drefine.data_dir` (/) -* Development mode: `-Drefine.development` (false) -* Headless mode: `-Drefine.headless` (false) -* IP: `-Drefine.host` (127.0.0.1) -* Port: `-Drefine.port` (3333) -* The application folder: `-Drefine.webapp` (main/webapp) - -The syntax is as follows: - - - - - -Locate the `refine.l4j.ini` file, and insert lines in this way: - -``` --Drefine.port=3334 --Drefine.host=127.0.0.2 --Drefine.webapp=broker/core -``` - -In `refine.ini`, use a similar syntax, but set multiple parameters within a single line starting with `JAVA_OPTIONS=`: - -``` -JAVA_OPTIONS=-Drefine.data_dir=C:\Users\user\Documents\OpenRefine\ -Drefine.port=3334 - -``` - - - - -Locate the `info.plist`, and find the `array` element that follows the line - -``` -JVMOptions -``` - -Typically this looks something like: - -``` -JVMOptions - --Xms256M --Xmx1024M --Drefine.version=2.6-beta.1 --Drefine.webapp=$APP_ROOT/Contents/Resource/webapp - -``` - -Add in values such as: - -``` -JVMOptions - --Xms256M --Xmx1024M --Drefine.version=2.6-beta.1 --Drefine.webapp=$APP_ROOT/Contents/Resource/webapp --Drefine.autosave=2 --Drefine.port=3334 - - -``` - - - - - -Locate the `refine.ini` file, and add `JAVA_OPTIONS=` before the `-Drefine.preference` declaration. You can un-comment and edit the existing suggested lines, or add lines: - -``` -JAVA_OPTIONS=-Drefine.autosave=2 -JAVA_OPTIONS=-Drefine.port=3334 -JAVA_OPTIONS=-Drefine.data_dir=usr/lib/OpenRefineWorkspace -``` - - - - - - ---- - -Refer to the [official Java documentation](https://docs.oracle.com/javase/8/docs/technotes/tools/windows/java.html) for more preferences that can be set. - -## The home screen {#the-home-screen} - -When you first launch OpenRefine, you will see a screen with a menu on the left hand side that includes Create Project, Open Project, Import Project, and Language Settings. This is called the “home screen,” where you can manage your projects and general settings. - -In the lower left-hand corner of the screen, you'll see Preferences, Help, and About. - -### Language settings {#language-settings} - -From the home screen, look in the options to the left for Language Settings. You can set your preferred interface language here. This language setting will persist until you change it again in the future. Languages are translated as a community effort; some languages are partially complete and default back to English where unfinished. Currently OpenRefine supports the following languages for 75% or more of the interface: - -* Cebuano -* German -* English (UK) -* English (US) -* Spanish -* Filipino -* French -* Hebrew -* Magyar -* Italian -* Japanese (日本語) -* Portuguese (Brazil) -* Tagalog -* Chinese (简体中文) - -To leave the Language Settings screen, click on the diamond “OpenRefine” logo. - -:::info Help us Translate OpenRefine -We use Weblate to provide translations for the interface. You can check [our profile on Weblate](https://hosted.weblate.org/projects/openrefine/translations/) to see which languages are in the process of being supported. See [our technical reference if you are interested in contributing translation work](../technical-reference/translating-ui) to make OpenRefine accessible to people in other languages. -::: - -### Preferences {#preferences} - -In the bottom left corner of the screen, look for Preferences. At this time you can set preferences using a key/value pair: that is, selecting one of the keys below and setting a value for it. - -|Setting|Key|Value syntax|Default|Example|Version| -|---|---|---|---|---|---| -|Interface language|userLang|[ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) two-digit code|en|fr|—| -|Maximum facets|ui.browsing.listFacet.limit|Number|2000|5000|—| -|Timeout for Google Drive import|googleReadTimeOut|Number (microseconds)|180000|500000|—| -|Timeout for Google Drive authorization|googleConnectTimeOut|Number (microseconds)|180000|500000|—| -|Maximum lag for Wikibase edit retries|wikibase.upload.maxLag|Number (seconds)|5|10|—| -|Display of the reconciliation preview on hover|cell-ui.previewMatchedCells|Boolean|true|false|v3.2| -|Values for the choice of the number of rows to display|ui.browsing.pageSize|Array of number (JSON)|[ 5, 10, 25, 50 ]|[ 100,500,1000 ]|v3.4| -|Width of the panel for facets/history|ui.browsing.facetsHistoryPanelWidth|Number (pixel)|300|500|v3.5| - -To leave the Preferences screen, click on the diamond “OpenRefine” logo. - -If the preference you’re looking for isn’t here, look at the options you can set from the [command line or in an `.ini` file](#starting-with-modifications). - -## The project screen {#the-project-screen} - -The project screen (or work screen) is where you will spend most of your time once you have [begun to work on a project](starting). This is a quick walkthrough of the parts of the interface you should familiarize yourself with. - -![A screenshot of the project screen.](/img/projectscreen.png) - -### The project bar {#the-project-bar} - -The project bar runs across the very top of the project screen. It contains the the OpenRefine logo, the project title, and the project control buttons on the right side. - -At any time you can close your current project and go back to the home screen by clicking on the OpenRefine logo. If you’d like to open another project in a new browser tab or window, you can right-click on the logo and use “Open in a new tab.” You will lose [your current facets and view settings](#facetfilter) if you close your project (but data transformations will be saved in the [History](#history-undoredo) of the project). - -:::caution -Don’t click the “back” button on your browser - it will likely close your current project and you will lose your facets and view settings. -::: - -You can rename a project at any time by clicking inside the project title, which will turn into a text field. Project names don’t have to be unique, as OpenRefine organizes them based on a unique identifier behind the scenes. - -The Permalink allows you to return to a project at a specific view state - that is, with [facets and filters](facets) applied. The Permalink can help you pick up where you left off if you have to close your project while working with facets and filters. It puts view-specific information directly into the URL: clicking on it will load this current-view URL in the existing tab. You can right-click and copy the Permalink URL to copy the current view state to your clipboard, without refreshing the tab you’re using. - -The Open… button will open up a new browser tab showing the Create Project screen. From here you can change settings, start a new project, or open an existing project. - -Export is a dropdown menu that allows you to pick a format for exporting a dataset. Many of the export options will only export rows and records that are currently visible - the currently selected facets and filters, not the total data in the project. - -Help will open up a new browser tab and bring you to this user manual on the web. - -### The grid header {#the-grid-header} - -The grid header sits below the project bar and above the project grid (where the data of your project is displayed). The grid header will tell you the total number of rows or records in your project, and indicate whether you are in [rows or records mode](exploring#rows-vs-records). - -It will also tell you if you’re currently looking at a select number of rows via facets or filtering, rather than the entire dataset, by displaying either, for example, “180 rows” or “67 matching rows (180 total).” - -Directly below the row number, you have the ability to switch between [row mode and records mode](exploring#rows-vs-records). OpenRefine stores projects persistently in one of the two modes, and displays your data as records by default if you are. - -To the right of the rows/records selection is the array of options for how many rows/records to view on screen at one time. At the far right of the screen you can navigate through your entire dataset one page at a time. - -### Extensions {#extensions} - -The Extensions dropdown offers you options for extending your data - most commonly by uploading your edited statements to Wikidata, or by importing or exporting schema. You can learn more about these functions on the [Wikibase section](wikibase/overview). Other extensions may also add functions to this dropdown menu. - -### The grid {#the-grid} - -The area of the project screen that displays your dataset is called the “grid” (or the “data grid,” or the “project grid”). The grid presents data in a tabular format, which may look like a normal spreadsheet program to you. - -Columns widths are automatically set based on their contents; some column headers may be cut off, but can be viewed by mousing over the headers. - -In each column header you will see a small arrow. Clicking on this arrow brings up a dropdown menu containing column-specific data exploration and transformation options. You will learn about each of these options in the [Exploring data](exploring) and [Transforming data](transforming) sections. - -The first column in every project will always be All, which contains options to flag, star, and do non-column-specific operations. The All column is also where rows/records are numbered. Numbering shows the permanent order of rows and records; a temporary sorting or facet may reorder the rows or show a limited set, but numbering will show you the original identifiers unless you make a permanent change. - -The project grid may display with both vertical and horizontal scrolling, depending on the number and width of columns, and the number of rows/records displayed. You can control the display of the project grid by using [Sort and View options](exploring#sort-and-view). - -Mousing over individual cells will allow you to [edit cells individually](cellediting#edit-one-cell-at-a-time). - -### Facet/Filter {#facetfilter} - -The Facet/Filter tab is one of the main ways of exploring your data: displaying the patterns and trends in your data, and helping you narrow your focus and modify that data. [Facets](facets) and [filters](facets#text-filter) are explained more in [Exploring data](exploring). - -![A screenshot of facets and filters in action.](/img/facetfilter.png) - -In the tab, you will see three buttons: Refresh, Reset all, and Remove all. - -Refreshing your facets will ensure you are looking at the latest information about each facet, for example if you have changed the counts or eliminated some options. - -Resetting your facets will remove any inclusion or exclusion you may have set - the facet options will stay in the sidebar, but your view settings will be undone. - -Removing your facets will clear out the sidebar entirely. If you have written custom facets using [expressions](expressions), these will be lost. - -You can preserve your facets and filters for future use by copying a [Permalink](#the-project-bar). - -### History (Undo/Redo) {#history-undoredo} - -In OpenRefine, any activity that changes the data can be undone. Changes are tracked from the very beginning, when a project is first created. The change history of each project is saved with the project's data, so quitting OpenRefine does not erase the steps you've taken. When you restart OpenRefine, you can view and undo changes that you made before you quit OpenRefine. OpenRefine [autosaves](starting#autosaving) your actions every five minutes by default, and when you close OpenRefine properly (using Ctrl + C). You can [change this interval](running#jvm-preferences). - -Project history gets saved when you export a project archive, and restored when you import that archive to a new installation of OpenRefine. - -![A screenshot of the History (Undo/Redo) tab with 13 steps.](/img/history.png "A screenshot of the History (Undo/Redo) tab with 13 steps.") - -When you click on the Undo / Redo tab in the sidebar of any project, that project’s history is shown as a list of changes in order, with the first “change” being the action of creating the project itself. (That first change, indexed as step zero, cannot be undone.) Here is a sample history with 3 changes: - -``` -0. Create project -1. Remove 7 rows -2. Create new column Last Name based on column Name with grel:value.split(" ") -3. Split 230 cell(s) in column Address into several columns by separator -``` - -The current state of the project is highlighted with a dark blue background. If you move back and forth on the timeline you will see the current state become highlighted, while the actions that came after that state will be grayed out. - -To revert your data back to an earlier state, simply click on the last action in the timeline you want to keep. In the example above, if we keep the removal of 7 rows but revert everything we did after that, then click on “Remove 7 rows.” The last 2 changes will be undone, in order to bring the project back to state #1. - -In this example, changes #2 and #3 will now be grayed out. You can redo a change by clicking on it in the history - everything up to and including it will be redone. - -If you have moved back one or more states, and then you perform a new operation on your data, the later actions (everything that’s greyed out) will be erased and cannot be re-applied. - -The Undo/Redo tab will indicate which step you’re on, and if you’re about to risk erasing work - by saying something like “4/5" or “1/7” at the end. - -#### Reusing operations {#reusing-operations} - -Operations that you perform in OpenRefine can be reused. For example, a formula you wrote inside one project can be copied and applied to another project later. - -To reuse one or more operations, first extract it from the project where it was first applied. Click to the Undo/Redo tab and click Extract…. This brings up a box that lists all operations up to the current state (it does not show undone operations). Select the operation or operations you want to extract using the checkboxes on the left, and they will be encoded as JSON on the right. Copy that JSON to the clipboard. - -Move to the second project, go to the Undo/Redo tab, click Apply… and paste in that JSON. - -Not all operations can be extracted. Edits to a single cell, for example, can’t be replicated. - -## Advanced OpenRefine uses {#advanced-openrefine-uses} - -### Running OpenRefine's Linux version on a Mac {#running-openrefines-linux-version-on-a-mac} - -You can run OpenRefine from the command line in Mac by using the Linux installation package. We do not promise support for this method. Follow the instructions in the Linux section. - -### Running as a server {#running-as-a-server} - -:::caution -Please note that if your machine has an external IP (is exposed to the Internet), you should not do this, or should protect it behind a proxy or firewall, such as nginx. Proceed at your own risk. -::: - -By default (and for security reasons), OpenRefine only listens to TCP requests coming from localhost (127.0.0.1) on port 3333. If you want to share your OpenRefine instance with colleagues and respond to TCP requests to any IP address of the machine, start it from the command line like this: -``` -./refine -i 0.0.0.0 -``` - -or set this option in `refine.ini`: -``` -REFINE_HOST=0.0.0.0 -``` - -or set this JVM option: -``` --Drefine.host=0.0.0.0 -``` - -On Mac, you can add a specific entry to the `Info.plist` file located within the app bundle (`/Applications/OpenRefine.app/Contents/Info.plist`): -``` -JVMOptions - - - -Drefine.host=0.0.0.0 - … - -``` - -:::caution -OpenRefine has no built-in security or version control for multi-user scenarios. OpenRefine has a single data model that is not shared, so there is a risk of data operations being overwritten by other users. Care must be taken by users. -::: - -### Automating OpenRefine {#automating-openrefine} - -Some users may wish to employ OpenRefine for batch processing as part of a larger automated pipeline. Not all OpenRefine features can work without human supervision and advancement (such as clustering), but many data transformation tasks can be automated. - -:::caution -The following are all third-party extensions and code; the OpenRefine team does not maintain them and cannot guarantee that any of them work. -::: - -Some examples: - -* This project allows OpenRefine to be run from the command line using [operations saved in a JSON file](#reusing-operations): [OpenRefine batch processing](https://github.com/opencultureconsulting/openrefine-batch) -* A Python project for applying a JSON file of operations to a data file, outputting the new file, and deleting the temporary project, written by David Huynh and Max Ogden: [Python client library for Google Refine](https://github.com/maxogden/refine-python) -* And the same in Ruby: [Refine-Ruby](https://github.com/maxogden/refine-ruby) -* Another Python client library, by Paul Makepeace: [OpenRefine Python Client Library](https://github.com/PaulMakepeace/refine-client-py) - -To look for other instances, search our Google Groups [for users](https://groups.google.com/g/openrefine) and [for developers](https://groups.google.com/g/openrefine-dev), where [these projects were originally posted](https://groups.google.com/g/openrefine/c/GfS1bfCBJow/m/qWYOZo3PKe4J). diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/sortview.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/sortview.md deleted file mode 100644 index 4e6bcb715..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/sortview.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -id: sortview -title: Sort and view -sidebar_label: Sort and view ---- - -## Sort {#sort} - -You can temporarily sort your rows by one column. You can sort based on [data type](exploring#data-types): -* text alphabetically or reverse -* numbers by largest or smallest -* dates by earliest or latest -* boolean values by false first or true first. - -You can also choose where to place errors and blank cells in the sorting. Text can be case-sensitive or not: if so, cells that start with lowercase characters will appear ahead of those that start with uppercase characters. - -![A screenshot of the Sort window.](/img/sort.png) - -After you apply a sorting method, you can make it permanent, remove it, reverse it, or apply a subsequent sorting. When it is applied, you’ll find Sort in the project grid header to the right of the rows-display setting, which will show all current sorting settings. - -If you have multiple sorting methods applied, they will work in the order you applied them (represented in order in the Sort menu). For example, you can sort an “authors” column alphabetically, and then sort their books by publication date, for those authors that have more than one book. If you apply those in a different order - sort all the publication dates in the dataset first, and then alphabetically by author - your dataset will look different. - -![Temporarily sorted rows.](/img/sort2.png) - -When the sorting method you've applied is temporary, you will see that the rows retain their original numbering. When you make that sorting method permanent, by selecting Reorder rows permanently, the row numbers will change and the Sort menu in the project grid header will disappear. This will apply all current sorting methods. - -## View {#view} - -You can control what data you view in the grid. On each column, you will see a View menu option. From there, you can “collapse” (hide) that specific column, all other columns, all columns to the left, and all columns to the right. Using the View option that appears in the All column’s dropdown menu, you can collapse all columns, and expand all the columns that you previously collapsed. - -### Show/hide “null” {#showhide-null} - -You can find, under AllView, the option to show and hide [“null” values](exploring#data-types). A small grey “null” will appear in each applicable cell. Remember that a null cell is not the same thing as an empty cell. - -![A screenshot of what a null value looks like.](/img/null.png) - -## Page navigation {#page-navigation} - -You can go directly to any page by changing the page number on the right-hand side. Using the up and down arrow keys in this input lets you go to the next and previous pages. You can also change the number of rows or records per page on the left-hand side of this view header bar. - -![A screenshot of the Page Navigation Feature.](/img/goto.png) diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/starting.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/starting.md deleted file mode 100644 index f40ec237c..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/starting.md +++ /dev/null @@ -1,194 +0,0 @@ ---- -id: starting -title: Starting a project -sidebar_label: Starting a project ---- - -## Overview {#overview} - -An OpenRefine project is started by importing in some existing data - OpenRefine doesn’t allow you to create a dataset from nothing. - -No matter where your data comes from, OpenRefine won’t modify your original data source. It copies all the information from your input, creates its own project file, and stores it in your [workspace directory](installing#set-where-data-is-stored). - -The data and all of your edits are [automatically saved](#autosaving) inside the project file. When you’re finished modifying the data, you can [export it back out](exporting) into the file format of your choice. - -You can also receive and open other people’s projects, or send them yours, by [exporting a project archive](exporting#export-a-project) and [importing it](#import-a-project). - -## Create a project by importing data {#create-a-project-by-importing-data} - -When you start OpenRefine, you’ll be taken to the Create Project screen. You’ll see on the left side of the screen that your options are to: - -* import data from one or more files on your computer -* import data from one or more links on the web -* import data by pasting in text from your clipboard -* import data from a database (using SQL), and -* import one or more Sheets from Google Drive. - -From these sources, you can load any of the following file formats: - -* comma-separated values (CSV) or text-separated values (TSV) -* Text files -* Fixed-width columns -* JSON -* XML -* OpenDocument spreadsheet (ODS) -* Excel spreadsheet (XLS or XLSX) -* PC-Axis (PX) -* MARC -* RDF data (JSON-LD, N3, N-Triples, Turtle, RDF/XML) -* Wikitext - -More formats can be imported by [adding extensions to provide that functionality](https://openrefine.org/download.html). - -If you supply two or more files for one project, the files’ rows will be loaded in the order that you specify, and OpenRefine will create a column at the beginning of the dataset with the source URL or file name in it to help you identify where each row came from. If the files have columns with identical names, the data will load in those columns; if not, the successive files will append all of their new columns to the end of the dataset: - -|File|Fruit|Quantity|Berry|Berry source| -|---|---|---|---|---| -|fruits.csv|Orange|4| -|fruits.csv|Apple|6| -|berries.csv||9|Mulberry|Greece| -|berries.csv||2|Blueberry|Canada| - -You cannot combine two datasets into one project by appending data within rows. You can, however, combine two projects later using functions such as [cross()](grelfunctions/#crosscell-s-projectname-s-columnname), or [fetch further data](columnediting) using other methods. - -For whichever method you choose to start your project, when you click Next >> you will be given a preview and a chance to configure the way OpenRefine interprets the data you input. - -### Get data from this computer {#get-data-from-this-computer} - -Click on Browse… and select a file (or several) on your hard drive. All files will be shown, not just compatible ones. - -If you import an archive file (something with the extension `.zip`, `.tar.gz`, `.tgz`, `.tar.bz2`, `.gz`, or `.bz2`), OpenRefine detects the files inside it, shows you a preview screen, and allows you to select which ones to load. This does not work with `.rar` files. When importing multiple archives you can store the name of the archive each file was extracted from by ticking the `Store archive file` option upon import. - -### Web addresses (URLs) {#web-addresses-urls} - -Type or paste the URL to a data file into the field provided. You can add as many fields as you want. OpenRefine will download the file and preview the project for you. - -If you supply two or more file URLs, OpenRefine will identify each one and ask you to choose which (or all) to load. - -Do not use this form to load a Google Sheet by its link; use [the Google Data form instead](#google-data). - -### Clipboard {#clipboard} - -You can copy and paste in data from anywhere. OpenRefine will recognize comma-separated, tab-separated, or table-formatted information copied from sources such as word-processing documents, spreadsheets, and tables in PDFs. You can also just paste in a list of items that you want to turn into rows. OpenRefine recognizes each new text line as a row. - -This can be useful if you want to pre-select a specific number of rows from your source data, or paste together rows from different places, rather than delete unwanted rows later in the project interace. - -This can also be useful if you would like to paste in a list of URLs, which you can use later to [fetch more data](columnediting). - -### Database (SQL) {#database-sql} - -If you are an administrator or have SQL access to a database of information, you may want to pull the latest dataset directly from there. This could include an online catalogue, a content management system, or a digital repository or collection management system. You can also load a database (`.db`) file saved locally. You will need to use an [SQL query](https://www.w3schools.com/sql/) to import your intended data. - -There are some publicly-accessible databases you can query, such as [one provided by Rfam](https://docs.rfam.org/en/latest/database.html). The instructions provided by Rfam can help you understand how to connect to and query from other databases. - -OpenRefine can connect to PostgreSQL, MySQL, MariaDB, and SQLite database systems. It will automatically populate the Port field based on which of these you choose, but you can manually edit this if needed. - -If you have a `.db` file, you can supply the path to the file on your computer in the Database field at the bottom of the form. You can leave the rest of the fields blank. - -To import data directly from a database, you will need the database type (such as MySQL), database name, the hostname (either an IP address or the domain that hosts the database), and the port on the host. You will need an account authorized for access, and you may need to add OpenRefine's IP address or host to the "allowable hosts" for that account. You can find that information by pressing Test and getting the IP address from the error message that results. - -You can either connect just once to gather data, or save the connection to use it again later. If you press Connect without saving, OpenRefine will forget all the information you just entered. If you’d like to save the connection, name your connection in a way you will recognize later. Click Save and it will appear in the Saved Connections list on the left. From now on, you can click on the ... ellipsis to the right of the connection you’ve saved, and click Connect. - -If your connection is successful, you will see a Query Editor where you can run your SQL query. OpenRefine will give you an error if you write a statement that tries to modify the source database in any way. - -### Google data {#google-data} - -You have two ways to load in data from Google Sheets: -* providing a link to an accessible Google Sheet (that is, one with link-sharing turned on), and -* selecting a Google Sheet in your Google Drive. - -#### Google Sheet by URL {#google-sheet-by-url} - -You can import data from any Google Sheet that has link-sharing turned on. Paste in a URL that looks something like - -``` -https://docs.google.com/spreadsheets/………/edit?usp=sharing -``` - -This will only work with Sheets, not with any other Google Drive file that might have an available link, including `.xls` and other valid files that are hosted in Google Drive. These links will not work when attempting to start a project [by URL](#web-addresses-urls) either, so you need to download those files to your computer. - -#### Google Sheet from Drive {#google-sheet-from-drive} - -You can authorize OpenRefine to access your Google Drive data and import data from any Google Sheet it finds there. This will include Sheets that belong to you and Sheets that are shared with you, as well as Sheets that are in your trash. - -When you select a Google option (either here, or [when exporting project data to Google Drive or Google Sheets](exporting), you will see a pop-up window that asks you to select a Google account to authorize with. You may see an error message when you authorize: if so, try your import or export operation again and it should succeed. - -OpenRefine will not show spreadsheets that are in your email inbox or stored in any other Google property - only in Drive. It also won’t show all compatible file formats, only Sheets files. - -OpenRefine will generate a list of all Sheets it finds, with the most recently modified Sheets at the top. If a file you’ve just added isn’t showing in this list, you can close and restart OpenRefine, or simply navigate to an existing project, open it, then head back to the Create Project window and check again. - -When you click Preview the Sheet will open in a new browser tab. When you click the Sheet title, OpenRefine will begin to process the data. - - -## Project preview {#project-preview} - -Once OpenRefine is ready to import the data, you will see a screen with Configure Parsing Options at the top. You’ll see a preview of the first 100 rows and all identified columns. - -At the bottom of the screen you will find options for telling OpenRefine how to process what it has found. You can tell it which row(s) to parse as column headers, as well as to ignore any number of rows at the top. You can also select a specific range of rows to work with, by discarding some rows at the top (excluding the header) and limiting the total number of rows it loads. - -OpenRefine tries to guess how to parse your data based on the file extension. For example, `.xml` files are going to be parsed as though they are formatted in XML. An unknown file extension (or your clipboard copy-paste) is assumed to be either tab-separated or comma-separated. OpenRefine looks for a tab character, and if one is found, it assumes you have imported tab-separated data. - -If OpenRefine isn’t certain what format you imported, it will provide a list of possibilities under Parse data as and some settings. You can specify a custom separator now, or split columns later while [transforming your data](transforming). - -If you imported a spreadsheet with multiple worksheets, they will be listed along with the number of rows they contain. You can only select data from one worksheet. - -Note that OpenRefine does not preserve any formatting, such as cell or text colour, that my have been in the original data file. Hyperlinked text will be input as plain text, but OpenRefine will recognize links and make them clickable inside the project interface. - -:::info Encoding issues? -Look for character encoding issues at this stage. You may want to manually select an encoding, such as UTF-8, UTF-16, or ASCII, if OpenRefine does not display some characters correctly in the preview. Once your project is created, you can specify another encoding for specific columns using the [reinterpret() function](grelfunctions#reinterprets-s-encoder). -::: - -You should create a project name at this stage. You can also supply tags to keep your projects organized. When you’re happy with the preview, click Create Project. - - -## Import a project {#import-a-project} - -Because OpenRefine only runs locally on your computer, you can’t have a project accessible to more than one person at the same time. - -The best way to collaborate with another person is to export and import projects that save all your changes, so that you can pick up where someone else left off. You can also [export projects](exporting#export-a-project) and import them to other computers, such as for working on the same project from the office and from home. - -An exported project will include all of the [history](running#history-undoredo), so you can see (and undo) all the changes from the previous user. It is essentially a point-in-time snapshot of their work. OpenRefine only exports projects as `.tar.gz` files at this time. -:::caution -If you wish to hide the original state of your data and your history of edits (for example, if you are using OpenRefine to anonymize information), export your cleaned dataset only and do not share your project archive. -::: - -Once someone has sent you a project archive file from their computer, you can save it anywhere. OpenRefine will import it like a new project and save its information to your workspace directory. - -In the left-hand menu of the home screen, click Import Project. Click Browse… and navigate to wherever you saved the file you were sent (for example, your Downloads folder). - -You can rename the project if you’d like - we recommend adding your name, a date, or a version number, if you’re planning to continue collaborating with another person (or working from multiple computers). - -Then, click Import Project. Your project should appear with a step count beside Undo/Redo if steps were saved by the exporter. - -OpenRefine will store the project in its own workspace directory, so you can now delete the original file that was sent to you. - - -## Project management {#project-management} - -You can access all of your created projects by clicking on Open Project. Your project list can be organized by modification date, title, row count, and other metadata you can supply (such as subject, descripton, tags, or creator). To edit the fields you see here, click About to the left of each project. There you can edit a number of available fields. You can also see the project ID that corresponds to the name of the folder in your work directory. - -### Naming projects {#naming-projects} - -You may have multiple projects from the same dataset, or multiple versions from sharing a project with another person. OpenRefine automatically generates a project name from the imported file, or “clipboard” when you use Clipboard importing. Project names don’t have to be unique, and OpenRefine will create many projects with the same name unless you intervene. - -You can edit a project's name when you create it or import it, and you can rename a project later by opening it and clicking on the project name at the top of the screen. - -### Autosaving {#autosaving} - -OpenRefine [saves all of your actions](running#history-undoredo) (everything you can see in the Undo/Redo panel). That includes flagging and starring rows. - -It doesn’t, however, save your facets, filters, or any kind of view you may have in place while you work. This includes the number of rows showing, and any sorting or column collapsing you may have done. A good rule of thumb is: if it’s not showing in Undo/Redo, you will lose it when you leave the project workspace. - -Autosaving happens by default every five minutes. You can [change this preference by following these directions](running#jvm-preferences). - -You can only save and share facets and filters, not any other type of view. To save current facets and filters, click Permalink. The project will reload with a different URL, which you can then copy and save elsewhere. This permalink will save both the facets and filters you’ve set, and the settings for each one (such as sorting by count rather than by name). - -### Deleting projects {#deleting-projects} - -You can delete projects, which will erase the project files from the workspace directory on your computer. This is immediate and cannot be undone. - -Go to Open Project and find the project you want to delete. Click on the X to the left of the project name. There will be a confirmation dialog. - -### Project files {#project-files} - -You can find all of your raw project files in your work directory. They will be named according to the unique “Project ID” that OpenRefine has assigned them, which you can find on the Open Project screen, under the “About” link for each project. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/transforming.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/transforming.md deleted file mode 100644 index 53a436370..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/transforming.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -id: transforming -title: Transforming data -sidebar_label: Overview ---- - -## Overview {#overview} - -OpenRefine gives you powerful ways to clean, correct, codify, and extend your data. Without ever needing to type inside a single cell, you can automatically fix typos, convert things to the right format, and add structured categories from trusted sources. - -This section of ways to improve data are organized by their appearance in the menu options in OpenRefine. You can: - -* change the order of [rows](#edit-rows) or [columns](columnediting#rename-remove-and-move) -* edit [cell contents](cellediting) within a particular column -* [transform](transposing) rows into columns, and columns into rows -* [split or join columns](columnediting#split-or-join) -* [add new columns](columnediting) based on existing data, with fetching new information, or through [reconciliation](reconciling) -* convert your rows of data into [multi-row records](exploring#rows-vs-records). - -## Edit rows {#edit-rows} - -Moving rows around is a permanent change to your data. - -You can [sort your data](sortview#sort) based on the values in one column, but that change is a temporary view setting. With that setting applied, you can make that new order permanent. - -![A screenshot of where to find the Sort menu with a sorting applied.](/img/sortPermanent.png) - -In the project grid header, the word “Sort” will appear when a sort operation is applied. Click on it to show the dropdown menu, and select Reorder rows permanently. You will see the numbering of the rows change under the All column. - -:::info Reordering all rows -Reordering rows permanently will affect all rows in the dataset, not just those currently viewed through [facets and filters](facets). -::: - -You can undo this action using the [History tab](running#history-undoredo). \ No newline at end of file diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/transposing.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/transposing.md deleted file mode 100644 index 6b5ec0dc8..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/transposing.md +++ /dev/null @@ -1,234 +0,0 @@ ---- -id: transposing -title: Transposing -sidebar_label: Transposing ---- - -## Overview {#overview} - -These functions were created to solve common problems with reshaping your data: pivoting cells from a row into a column, or pivoting cells from a column into a row. You can also transpose from a repeated set of values into multiple columns. - -## Transpose cells across columns into rows {#transpose-cells-across-columns-into-rows} - -Imagine personal data with addresses in this format: - -|Name|Street|City|State/Province|Country|Postal code| -|---|---|---|---|---|---| -|Jacques Cousteau|23, quai de Conti|Paris||France|75270| -|Emmy Noether|010 N Merion Avenue|Bryn Mawr|Pennsylvania|USA|19010| - -You can transpose the address information from this format into multiple rows. Go to the “Street” column and select TransposeTranspose cells across columns into rows. From there you can select all of the five columns, starting with “Street” and ending with “Postal code,” that correspond to address information. Once you begin, you should put your project into [records mode](exploring#rows-vs-records) to associate the subsequent rows with “Name” as the key column. - -![A screenshot of the transpose across columns window.](/img/transpose1.png) - -### One column {#one-column} - -You can transpose the multiple address columns into a series of rows: - -|Name|Address| -|---|---| -|Jacques Cousteau|23, quai de Conti| -| |Paris| -| |France| -| |75270| -|Emmy Noether|010 N Merion Avenue| -||Bryn Mawr| -||Pennsylvania| -||USA| -||19010| - -You can choose one column and include the column-name information in each cell by prepending it to the value, with or without a separator: - -|Name|Address| -|---|---| -|Jacques Cousteau|Street: 23, quai de Conti| -| |City: Paris| -| |Country: France| -| |Postal code: 75270| -|Emmy Noether|Street: 010 N Merion Avenue| -||City: Bryn Mawr| -||State/Province: Pennsylvania| -||Country: USA| -||Postal code: 19010| - -### Two columns {#two-columns} - -You can retain the column names as separate cell values, by selecting Two new columns and naming the key and value columns. - -|Name|Address part|Address| -|---|---|---| -|Jacques Cousteau|Street|23, quai de Conti| -| |City|Paris| -| |Country|France| -| |Postal code|75270| -|Emmy Noether|Street|010 N Merion Avenue| -||City|Bryn Mawr| -||State/Province|Pennsylvania| -||Country|USA| -||Postal code|19010| - -## Transpose cells in rows into columns {#transpose-cells-in-rows-into-columns} - -Imagine employee data in this format: - -|Column| -|---| -|Employee: Karen Chiu| -|Job title: Senior analyst| -|Office: New York| -|Employee: Joe Khoury| -|Job title: Junior analyst| -|Office: Beirut| -|Employee: Samantha Martinez| -|Job title: CTO| -|Office: Tokyo| - -The goal is to sort out all of the information contained in one column into separate columns, but keep it organized by the person it represents: - -|Name |Job title |Office| -|---|---|---| -|Karen Chiu |Senior analyst |New York| -|Joe Khoury |Junior analyst |Beirut| -|Samantha Martinez |CTO |Tokyo| - -By selecting TransposeTranspose cells in rows into columns... a window will appear that simply asks how many rows to transpose. In this case, each employee record has three rows, so input “3” (do not subtract one for the original column). The original column will disappear and be replaced with three columns, with the name of the original column plus a number appended. - -|Column 1 |Column 2 |Column 3| -|---|---|---| -|Employee: Karen Chiu |Job title: Senior analyst |Office: New York| -|Employee: Joe Khoury |Job title: Junior analyst |Office: Beirut| -|Employee: Samantha Martinez |Job title: CTO |Office: Tokyo| - -From here you can use Cell editingReplace to remove “Employee: ”, “Job title: ”, and “Office: ” if you wish, or use [expressions](expressions) with Edit cellsTransform... to clean out the extraneous characters: - -``` -value.replace("Employee: ", "") -``` - -If your dataset doesn't have a predictable number of cells per intended row, such that you cannot specify easily how many columns to create, try Columnize by key/value columns. - -## Columnize by key/value columns {#columnize-by-keyvalue-columns} - -This operation can be used to reshape a dataset that contains key and value columns: the repeating strings in the key column become new column names, and the contents of the value column are moved to new columns. This operation can be found at TransposeColumnize by key/value columns. - -![A screenshot of the Columnize window.](/img/transpose2.png) - -Consider the following example, with flowers, their colours, and their International Union for Conservation of Nature (IUCN) identifiers: - -|Field |Data | -|--------|----------------------| -|Name |Galanthus nivalis | -|Color |White | -|IUCN ID |162168 | -|Name |Narcissus cyclamineus | -|Color |Yellow | -|IUCN ID |161899 | - -In this format, each flower species is described by multiple attributes on consecutive rows. The “Field” column contains the keys and the “Data” column contains the values. In the Columnize by key/value columns window you can select each of these from the available columns. It transforms the table as follows: - -| Name | Color | IUCN ID | -|-----------------------|----------|---------| -| Galanthus nivalis | White | 162168 | -| Narcissus cyclamineus | Yellow | 161899 | - -### Entries with multiple values in the same column {#entries-with-multiple-values-in-the-same-column} - -If a new row would have multiple values for a given key, then these values will be grouped on consecutive rows, to form a [record structure](exploring#rows-vs-records). - -For instance, flowers can have multiple colors: - -| Field | Data | -|-------------|-----------------------| -| Name | Galanthus nivalis | -| _Color_ | _White_ | -| _Color_ | _Green_ | -| IUCN ID | 162168 | -| Name | Narcissus cyclamineus | -| Color | Yellow | -| IUCN ID | 161899 | - -This table is transformed by the Columnize operation to: - -| Name | Color | IUCN ID | -|-----------------------|----------|---------| -| Galanthus nivalis | White | 162168 | -| | Green | | -| Narcissus cyclamineus | Yellow | 161899 | - -The first key encountered by the operation serves as the record key, so the “Green” value is attached to the “Galanthus nivalis” name. See the [Row order](#row-order) section for more details about the influence of row order on the results of the operation. - -### Notes column {#notes-column} - -In addition to the key and value columns, you can optionally add a column for notes. This can be used to store extra metadata associated to a key/value pair. - -Consider the following example: - -| Field | Data | Source | -|---------|---------------------|-----------------------| -| Name | Galanthus nivalis | IUCN | -| Color | White | Contributed by Martha | -| IUCN ID | 162168 | | -| Name | Narcissus cyclamineus | Legacy | -| Color | Yellow | 2009 survey | -| IUCN ID | 161899 | | - -If the “Source” column is selected as the notes column, this table is transformed to: - -| Name | Color | IUCN ID | Source: Name | Source: Color | -|-----------------------|----------|---------|---------------|-----------------------| -| Galanthus nivalis | White | 162168 | IUCN | Contributed by Martha | -| Narcissus cyclamineus | Yellow | 161899 | Legacy | 2009 survey | - -Notes columns can therefore be used to preserve provenance or other context about a particular key/value pair. - -### Row order {#row-order} - -The order in which the key/value pairs appear matters. The Columnize operation will use the first key it encounters as the delimiter for entries: every time it encounters this key again, it will produce a new row, and add the following key/value pairs to that row. - -Consider for instance the following table: - -| Field | Data | -|----------|-----------------------| -| _Name_ | Galanthus nivalis | -| Color | White | -| IUCN ID | 162168 | -| _Name_ | Crinum variabile | -| _Name_ | Narcissus cyclamineus | -| Color | Yellow | -| IUCN ID | 161899 | - -The occurrences of the “Name” value in the “Field” column define the boundaries of the entries. Because there is no other row between the “Crinum variabile” and the “Narcissus cyclamineus” rows, the “Color” and “IUCN ID” columns for the “Crinum variabile” entry will be empty: - -| Name | Color | IUCN ID | -|-----------------------|----------|---------| -| Galanthus nivalis | White | 162168 | -| Crinum variabile | | | -| Narcissus cyclamineus | Yellow | 161899 | - -This sensitivity to order is removed if there are extra columns: in that case, the first extra column will serve as the key for the new rows. - -### Extra columns {#extra-columns} - -If your dataset contains extra columns, that are not being used as the key, value, or notes columns, they can be preserved by the operation. For this to work, they must have the same value in all old rows corresponding to a new row. - -In the following example, the “Field” and “Data” columns are used as key and value columns respectively, and the “Wikidata ID” column is not selected: - -| Field | Data | Wikidata ID | -|---------|-----------------------|-------------| -| Name | Galanthus nivalis | Q109995 | -| Color | White | Q109995 | -| IUCN ID | 162168 | Q109995 | -| Name | Narcissus cyclamineus | Q1727024 | -| Color | Yellow | Q1727024 | -| IUCN ID | 161899 | Q1727024 | - -This will be transformed to: - -| Wikidata ID | Name | Color | IUCN ID | -|-------------|-----------------------|----------|---------| -| Q109995 | Galanthus nivalis | White | 162168 | -| Q1727024 | Narcissus cyclamineus | Yellow | 161899 | - -This actually changes the operation: OpenRefine no longer looks for the first key (“Name”) but simply pivots all information based on the first extra column's values. Every old row with the same value gets transposed into one new row. If you have more than one extra column, they are pivoted as well but not used as the new key. - -You can use [Fill down](cellediting#fill-down-and-blank-down) to put identical values in the extra columns if you need to. \ No newline at end of file diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/troubleshooting.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/troubleshooting.md deleted file mode 100644 index 5388fc351..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/troubleshooting.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -id: troubleshooting -title: Troubleshooting -sidebar_label: Troubleshooting ---- - -## Frequently asked questions {#frequently-asked-questions} - -We collect and share FAQs and responses on Github at [https://github.com/OpenRefine/OpenRefine/wiki/FAQ](https://github.com/OpenRefine/OpenRefine/wiki/FAQ). - -If you don’t find your problem and solution there, continue on to the resources in the Community section below to see more conversations and look for solutions. - -## Community {#community} - -### If you’re having a problem: {#if-youre-having-a-problem} -* Search the [User forum](https://groups.google.com/g/openrefine) to see if the problem is already reported -* Search [Github issues](https://github.com/OpenRefine/OpenRefine/issues) to see if the problem is already reported -* Read [Stack Overflow](https://stackoverflow.com/questions/tagged/openrefine) to see if others had a similar problem -* Check [Twitter](https://twitter.com/search?f=tweets&vertical=default&q=OpenRefine%20OR%20%22Open%20Refine%22%20OR%20%23OpenRefine&src=typd) to see if others are discussing the problem -* Report an issue: - * First as a new thread (conversation) in the [User forum](https://groups.google.com/g/openrefine). - * Then, if you wish, you can create a Github issue. - -### If you want to contribute: {#if-you-want-to-contribute} -* [Help us translate the tool into more languages](../technical-reference/translating-ui), using Weblate -* [We have a guide to contributing](../technical-reference/contributing) in the [Technical Reference](../technical-reference/technical-reference-index) section -* Contribute your feature requests in the [User forum](https://groups.google.com/g/openrefine) or as [Github issues](https://github.com/OpenRefine/OpenRefine/issues/new/choose) -* Join the User Forum and/or the [Developer Forum](https://groups.google.com/g/openrefine-dev) -* Share your successes and use cases with us, in the User forum -* Add your [blog posts, guides, tips, tricks, tutorials to our list](https://github.com/OpenRefine/OpenRefine/wiki/External-Resources) -* Keep an eye out for and respond to our biennial user survey. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/advanced-schemas.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/advanced-schemas.md deleted file mode 100644 index 45f91800c..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/advanced-schemas.md +++ /dev/null @@ -1,72 +0,0 @@ -Sometimes your data is not as simple as a normal table, or the sort of -statements that you want to do varies on each row. This document -explains how to work around these cases. - -## Hierarchical data {#hierarchical-data} - -Sometimes your source provides data in a structured format, such as XML, -JSON or RDF. OpenRefine can import these files and will convert them to -tables. These tables will reflect some of the hierarchy in the file by -means of null cells, using the [records mode](/manual/exploring#rows-vs-records). - -The Wikibase extension always works in rows mode, so if we want to add -statements which reference both the artist and the song, we need to fill -the null cells with the corresponding artist. You can do this with the -**Fill down** operation (in the **Edit cells** menu for this column). -This function will copy not just cell values but also reconciliation -results. - -## Conditional additions {#conditional-additions} - -Sometimes you want to add a statement only in some conditions. - -The workflow to achieve this looks like this: -- Use facets to select the rows where you do not want to add any - information; -- Blank out the cells in the column that contain the information you - want to add. If you do not want to lose this information, you can - create a copy of the column beforehand; -- Remove your facets to see all rows again; -- Create a schema using the column you partially blanked out as - statement value. - -## Varying properties {#varying-properties} - -Sometimes you wish you could use column variables for properties in your -schema. It is currently not possible, first because we do not have a -reconciliation service for properties yet, but also because allowing -varying properties in a statement would mean that these properties could -potentially have different datatypes, which would break the structure of -the schema. - -If you only want to use a few properties, there is a way to go around -this problem. For instance, say you have a first column of altitudes and a -second column that indicates whether you should add it as -[operating altitude (P2254)](https://www.wikidata.org/wiki/Property:P2254) or as -[elevation above sea level (P2044)](https://www.wikidata.org/wiki/Property:P2044). - -Create a text facet on the first column. Filter to keep only the -*altitude* values. Add a new column based on the second column, by -keeping the default expression (`value`) which just copies the existing -values. Then, select the *maximum operating altitude* value in the facet -and do the same. Reset the facet, you should have obtained two new columns -which partition the original column. You can now create a schema which adds -two statements, with values taken from those columns. Since blank values are -ignored, exactly one statement will be added for each item, with the desired property. - -## Adapting to existing data on Wikibase {#adapting-to-existing-data-on-wikibase} - -Sometimes you want to create statements only if there are no such -statements on the item yet. Here is one way to achieve this: - -- first, retrieve the existing values from Wikidata first, using the - **Edit columns** → **Add columns from reconciled values** action; -- second, create a *facet by null* on the newly created column that - contains the information you want to control against; -- select the non-null rows (value **false**); -- clear the contents of the column where your source values are - (**Edit cells** → **Common transformations** → **To null**). - -You can now construct your schema as usual - null values will be ignored -when generating the statements. - diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/configuration.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/configuration.md deleted file mode 100644 index aa7fd1d98..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/configuration.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -id: configuration -title: Connecting OpenRefine to a Wikibase instance -sidebar_label: Connecting to Wikibase ---- - -This page explains how to connect OpenRefine to any Wikibase instance. If you just want to work with [Wikidata](https://www.wikidata.org/), you can ignore this page as Wikidata is configured out of the box in OpenRefine. - -## For Wikibase end users {#for-wikibase-end-users} - -All you need to configure OpenRefine to work with a Wikibase instance is a *manifest* for that instance, which provides some metadata and links required for the integration to work. - -We offer some off-the-shelf manifests for some public Wikibase instances in the [wikibase-manifests](https://github.com/OpenRefine/wikibase-manifests) repository. But the administrators of your Wikibase instance should provide one that is potentially more -up to date, so it makes sense to request it to them first. - -## For Wikibase administrators {#for-wikibase-administrators} - -To let your users contribute to your Wikibase instance with OpenRefine, you will need to write a manifest as described above. There is currently no canonical location where this manifest should be hosted - just make sure can be found easily by your users. This section explains the format of the manifest. - -### Requirements {#requirements} - -To work with OpenRefine, your Wikibase instance needs an associated reconciliation service. For instance you can use [a Python wrapper](https://github.com/wetneb/openrefine-wikibase) for this. Also, in addition to Wikibase, the [UniversalLanguageSelector extension](https://www.mediawiki.org/wiki/Special:MyLanguage/Extension:UniversalLanguageSelector) should be installed. - - -### The format of the manifest {#the-format-of-the-manifest} - -Here is the manifest of Wikidata: - -```json -{ - "version": "1.0", - "mediawiki": { - "name": "Wikidata", - "root": "https://www.wikidata.org/wiki/", - "main_page": "https://www.wikidata.org/wiki/Wikidata:Main_Page", - "api": "https://www.wikidata.org/w/api.php" - }, - "wikibase": { - "site_iri": "http://www.wikidata.org/entity/", - "maxlag": 5, - "properties": { - "instance_of": "P31", - "subclass_of": "P279" - }, - "constraints": { - "property_constraint_pid": "P2302", - "exception_to_constraint_pid": "P2303", - "constraint_status_pid": "P2316", - "mandatory_constraint_qid": "Q21502408", - "suggestion_constraint_qid": "Q62026391", - "distinct_values_constraint_qid": "Q21502410", - // ... - } - }, - "oauth": { - "registration_page": "https://meta.wikimedia.org/wiki/Special:OAuthConsumerRegistration/propose" - }, - "reconciliation": { - "endpoint": "https://wikidata.reconci.link/${lang}/api" - }, - "editgroups": { - "url_schema": "([[:toollabs:editgroups/b/OR/${batch_id}|details]])" - } -} -``` - -In general, there are several parts of the manifest: version, mediawiki, wikibase, oauth, reconciliation and editgroups. - -#### version {#version} - -The version should in the format "1.x". The minor version should be increased when you update the manifest in a backward-compatible manner. The major version should be "1" if the manifest is in the format specified by [wikibase-manifest-schema-v1.json](https://github.com/afkbrb/wikibase-manifest/blob/master/wikibase-manifest-schema-v1.json). - -#### mediawiki {#mediawiki} - -This part contains some basic information of the Wikibase. - -##### name {#name} - -The name of the Wikibase, should be unique for different Wikibase instances. - -##### root {#root} - -The root of the Wikibase. Typically in the form "https://foo.bar/wiki/". The trailing slash cannot be omitted. - -##### main_page {#main_page} - -The main page of the Wikibase. Typically in the form "https://foo.bar/wiki/Main_Page". - -##### api {#api} - -The MediaWiki API endpoint of the Wikibase. Typically in the form "https://foo.bar/w/api.php". - -#### wikibase {#wikibase} - -This part contains configurations of the Wikibase extension. - -##### site_iri {#site_iri} - -The IRI of the Wikibase, in the form 'http://foo.bar/entity/'. This should match the IRI prefixes used in RDF serialization. Be careful about using "http" or "https", because any variation will break comparisons at various places. The trailing slash cannot be omitted. - -##### maxlag {#maxlag} - -Maxlag is a parameter that controls how aggressive a mass-editing tool should be when uploading edits to a Wikibase instance. See https://www.mediawiki.org/wiki/Manual:Maxlag_parameter for more details. The value should be adapted according to the actual traffic of the Wikibase. - -##### properties {#properties} - -Some special properties of the Wikibase. - -###### instance_of {#instance_of} - -The ID of the property "instance of". - -###### subclass_of {#subclass_of} - -The ID of the property "subclass of". - -##### constraints {#constraints} - -Not required. Should be configured if the Wikibase has the [WikibaseQualityConstraints extension](https://www.mediawiki.org/wiki/Extension:WikibaseQualityConstraints) installed. Configurations of constraints consists of IDs of constraints related properties and items. For Wikidata, these IDs are retrieved from [extension.json](https://github.com/wikimedia/mediawiki-extensions-WikibaseQualityConstraints/blob/master/extension.json). To configure this for another Wikibase instance, you should contact an admin of the Wikibase instance to get the content of `extension.json`. - -#### oauth {#oauth} - -Not required. Should be configured if the Wikibase has the [OAuth extension](https://www.mediawiki.org/wiki/Extension:OAuth) installed. - -##### registration_page {#registration_page} - -The page to register an OAuth consumer of the Wikibase. Typically in the form "https://foo.bar/wiki/Special:OAuthConsumerRegistration/propose". - -#### reconciliation {#reconciliation} - -The Wikibase instance must have at least a reconciliation service endpoint linked to it. If there is no reconciliation service for the Wikibase, you can run one with [openrefine-wikibase](https://github.com/wetneb/openrefine-wikibase). - -##### endpoint {#endpoint} - -The default reconciliation service endpoint of the Wikibase instance. The endpoint must contain the "${lang}" variable such as "https://wikidata.reconci.link/${lang}/api", since the reconciliation service is expected to work for different languages. - -#### editgroups {#editgroups} - -Not required. Should be configured if the Wikibase instance has [EditGroups](https://github.com/Wikidata/editgroups) service(s). - -##### url_schema {#url_schema} - -The URL schema used in edits summary. This is used for EditGroups to extract the batch id from a batch of edits and for linking to the EditGroups page of the batch. The URL schema must contains the variable '${batch_id}', such as '([[:toollabs:editgroups/b/OR/${batch_id}|details]])' for Wikidata. - -#### Check the format of the manifest {#check-the-format-of-the-manifest} - -As mentioned above, the manifest should be in the format specified by [wikibase-manifest-schema-v1.json](https://github.com/afkbrb/wikibase-manifest/blob/master/wikibase-manifest-schema-v1.json). You can check the format by adding the manifest directly to OpenRefine, and OpenRefine will complain if there is anything wrong with the format. - -![test-validate-manifest-format](https://user-images.githubusercontent.com/29347603/90506110-52d85d00-e186-11ea-8077-683d2f234c46.gif) diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/new-entities.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/new-entities.md deleted file mode 100644 index 8dbc4f362..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/new-entities.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -id: new-entities -title: Creating new items -sidebar_label: New items ---- - -OpenRefine can create new items. This page explains how they are -generated. - -## Words of caution {#words-of-caution} - -- The fact that OpenRefine does not propose any item when reconciling - a cell does not mean that the item is not present in the Wikibase instance: - it can be missed for all sorts of reasons. Please make - sure that you are not creating any duplicates! - -- Make sure that the items that you want to create are admissible in - the Wikibase instance. For Wikidata, see the [notability guidelines](https://www.wikidata.org/wiki/Wikidata:Notability); - -- Deleting items generally requires special rights: if you want to revert an - edit group that includes new items in Wikidata, you will need to ask an - administrator to do it. - -## Workflow overview {#workflow-overview} - -Here is how you would typically create new items with OpenRefine: - -- Reconcile a column; -- Mark some of its cells as new items. This will not create items yet. - If you need to mark many rows as new items, use the **Reconcile** → - **Actions** → **Create a new item for each cell** operation. -- Create a Wikibase schema as usual, using the column where your new - items are marked; -- Perform the edits: the new items will be created on Wikidata at this - point; -- The cells that you had marked as new items will now be reconciled to - the newly-created items. - -It is often useful (but not mandatory) to treat new items in isolation -and use a dedicated schema for them. This helps you add many statements -on the new items (including labels and descriptions) without risking to -clutter existing items with redundant edits. Use a facet on the judgment -status of the reconciled column to isolate new items and perform their -edits separately. As always in OpenRefine, only the rows covered by your -facets will be considered when uploading the edits to Wikidata: if a -cell is reconciled to a new item but is excluded by the facet, no new -item will be created for it.[^1] - -Note that even if you know that all items in your column are new, you -will still need to make a first reconciliation pass by selecting the -Wikidata reconciliation service, and then setting all reconciliation -statuses to \"new\". If you skip the first part, OpenRefine will not -know that this column is reconciled against your Wikibase instance (it could be -reconciled to other services) so it will not let you use it in place of -an item in a Wikibase schema. - -You can also perform the edits with QuickStatements - in this case, your -OpenRefine project will not be updated with the newly created Qids. - -## Adding labels to new items {#adding-labels-to-new-items} - -The text that is in a cell reconciled to \"new\" is not automatically -used as label for the newly-created item. This is because OpenRefine has -no way to guess in which language this label should be. When adding new -items, you need to explicitly add a label in the schema. This label can -use the reconciled column as source, but if you have other cells matched -to existing items, be careful not to override the labels of these items -(if it is not your intention). - -OpenRefine will refuse to perform edits where new items are created -without any labels (as this is considered a critical issue). Other -issues will be raised if insufficient basic information is added on the -items (but these other warnings will not prevent you from performing the -edits). - -## Marking multiple cells as identical items {#marking-multiple-cells-as-identical-items} - -If you mark individual cells as new items, one new item per cell will be -created. Sometimes multiple rows refer to the same item. OpenRefine -makes it possible to mark all the corresponding cells as the *same* new -item. Two conditions have to be met: -- the reconciled cells must be in the same column (it is not possible - to mark two cells in different colums as the same new item); -- the cells must contain the same initial text value. - -If these two conditions are met, then isolate these cells with facets -and go to **Reconcile** → **Actions** → **Create one item for similar -cells**. This will mark the cells as new and referring to the same item. - -## Retrieving the Qids of the newly-created items {#retrieving-the-qids-of-the-newly-created-items} - -Once you have performed your edits with OpenRefine, any new cells -covered by the facet will be updated with their new Qids. You can -retrieve these Qids with the **Edit column** → **Add column based on -this column** action and using the `cell.recon.match.id` expression. -Note that you will no longer be able to isolate new items with a -judgment facet at this stage (because the judgment will be updated to -**matched**) so it can be worth marking these rows (for instance with a -star or flag) before performing the edits. - -[^1]: The only exception to this rule is when marking multiple cells as - identical items: in this case, if one of such cells are included in - the facet, then all the others will be updated with the newly - created Qid once the edits are made. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/overview.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/overview.md deleted file mode 100644 index a202101fd..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/overview.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -id: overview -title: Overview of Wikibase support -sidebar_label: Overview ---- - -[Wikibase](https://wikiba.se/) is a platform for collaborative knowledge base editing. Its flagship instance [Wikidata](https://www.wikidata.org/) offers structured data about the world and can be edited by anyone. OpenRefine provides powerful ways to both pull data from Wikibase and add data to it. - -OpenRefine's Wikibase integration is provided by an extension which is available by default in OpenRefine. In this page, we present the functionalities for Wikidata, but [any Wikibase instance can be connected to OpenRefine](./configuration) to obtain a similar integration. - -## Editing Wikidata with OpenRefine {#editing-wikidata-with-openrefine} - -As a user-maintained data source, Wikidata can be edited by anyone. OpenRefine makes it simple to upload information in bulk. You simply need to get your information into the correct format, and ensure that it is new (not redundant to information already on Wikidata) and does not conflict with existing Wikidata information. - -You do not need a Wikidata account to reconcile your local OpenRefine project to Wikidata, but to upload your cleaned dataset to Wikidata, you will need an [autoconfirmed](https://www.wikidata.org/wiki/Wikidata:Autoconfirmed_users) account, and you must [authorize OpenRefine with that account](#manage-wikidata-account). - -Wikidata is built by creating entities (such as people, organizations, or places, identified with unique numbers starting with Q), defining properties (unique numbers starting with P), and using properties to define relationships between entities (a Q has a property P, with a value of another Q). - -For example, you may wish to create entities for local authors and the books they've set in your community. Each writer will be an entity with the occupation [author (Q482980)](https://www.wikidata.org/wiki/Q482980), each book will be an entity with the property “instance of” ([P31](https://www.wikidata.org/wiki/Property:P31)) linking it to a class such as [literary work (Q7725634)](https://www.wikidata.org/wiki/Q7725634), and books will be related to authors through a property [author (P50)](https://www.wikidata.org/wiki/Property:P50). Books can have places where they are set, with the property [narrative location (P840)](https://www.wikidata.org/wiki/Property:P840). - -To do this with OpenRefine, you'll need a column of publication titles that you have reconciled (and create new items where needed); each publication will have one or more locations in a “setting” column, which is also reconciled to municipalities or regions where they exist (and create new items where needed). Then you can add those new relationships, and create new entities for authors, books, and places where needed. You do not need columns for properties; those are defined later, in the creation of your [schema](#edit-wikidata-schema). - -There is a list of [tutorials and walkthroughs on Wikidata](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing) that will allow you to see the full process. You can save your schemas and drafts in OpenRefine, and your progress stays in draft until you are ready to upload it to Wikidata. - -Batches of edits to Wikidata that are created with OpenRefine can be undone. You can test out the uploading process by reconciling to several “sandbox” entities created specifically for drafting edits and learning about Wikidata: -* https://www.wikidata.org/wiki/Q4115189 -* https://www.wikidata.org/wiki/Q13406268 -* https://www.wikidata.org/wiki/Q15397819 -* https://www.wikidata.org/wiki/Q64768399 - -If you upload edits that are redundant (that is, all the statements you want to make have already been made), nothing will happen. If you upload edits that conflict with existing information (such as a different birthdate than one already in Wikidata), it will be added as a second statement. OpenRefine produces no warnings as to whether your data replicates or conflicts with existing Wikidata elements. - -You can use OpenRefine's reconciliation preview to look at the target Wikidata elements and see what information they already have, and whether the elements' histories have had similar edits reverted in the past. - -### Wikidata schema {#wikidata-schema} - -A [schema](https://en.wikipedia.org/wiki/Database_schema) is a plan for how to structure information in a database. In OpenRefine, the schema operates as a template for how Wikidata edits should be applied: how to translate your tabular data into statements. With a schema, you can: -* preview the Wikidata edits and inspect them manually; -* analyze and fix any issues highlighted by OpenRefine; -* upload your changes to Wikidata by logging in with your own account; -* export the changes to the QuickStatements v1 format. - -For example, if your dataset has columns for authors, publication titles, and publication years, your schema can be conceptualized as: [publication title] has the author [author], and was published in [publication year]. To establish these facts, you need to establish one or more columns as “items,” for which you will make “statements” that relate them to other columns. - -You can export any schema you create, and import an existing schema for use with a new dataset. This can help you work in batches on a large amount of data while minimizing redundant labor. - -Once you select Edit Wikidata schema under the Extensions dropdown menu, your project interface will change. You’ll see new tabs added to the right of “X rows/records" in the grid header: “Schema,” “Issues,” and “Preview.” You can now switch between the tabular grid format of your dataset and the screens that allow you to prepare data for uploading. - -OpenRefine presents you with an easy visual way to map out the relationships in your dataset. Each of the columns of your project will appear at the top of the sceren, and you can simply drag and drop them into the appropriate slots. To get start, select one column as an item. - -![A screenshot of the schema construction window in OpenRefine.](/img/wikidata-schema.png) - -You may wish to refer to [this Wikidata tutorial on how OpenRefine handles Wikidata schema](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Tutorials/Basic_editing). For details about how each data type is handled in the Wikibase schema, see [Schema alignment](./schema-alignment). - -#### Editing terms with your schema {#editing-terms-with-your-schema} - -With OpenRefine, you can edit the terms (labels, aliases, descriptions, or sitelinks) of Wikidata entities as well as establish relationships between entities. For example, you may wish to upload pseudonyms, pen names, maiden names, or married names for authors. - -![An author with a number of aliases indicating pseudonyms.](/img/wikidata-terms.png) - -You can do so by putting the preferred names in one column of your dataset and alternative names in another column. In the schema interface, add an item for the preferred values, then click “Add term” on the right-hand side of the screen. Select “Alias” from the dropdown, enter in “English” in the language field, and drop your alternative names column into the space. For this example, you should also consider adding those alternative names to the authors' entries using the property [pseudonym (P742)](https://www.wikidata.org/wiki/Property:P742). The "description" and "label" terms can only contain one value, so there is an option to override existing values if needed. Aliases can be potentially infinite. - -![The schema window showing a term being edited.](/img/wikidata-terms2.png) - -Terms must always have an associated language. You can select the term's language by typing in the “lang” field, which will auto-complete for you. You cannot edit multiple languages at once, unless you supply a suitable column instead. For example, suppose you had translated publication titles, with data in the following format: - -|English title|Translated title|Translation language| -|---|---|---| -|Possession|Besessen|German| -||Обладать|Russian| -|Disgrace|Disgrâce|French| -||Vergogna|Italian| -|Wolf Hall|En la corte del lobo|Spanish| -||ウルフ・ホール|Japanese| - -You could upload the “Translated titles” to “Label” with the language specified by “Translation language.” You may wish to fetch the two-letter language code and use that instead for better language matches. - -![Constructing a schema with aliases and languages.](/img/wikidata-translated.png) - -### Manage Wikidata account {#manage-wikidata-account} - -To edit Wikidata directly from OpenRefine, you must log in with a Wikidata account. OpenRefine can only upload edits with Wikidata user accounts that are “[autoconfirmed](https://www.wikidata.org/wiki/Wikidata:Autoconfirmed_users)” - at this time, that means accounts that have more than 50 edits and have existed for longer than four days. - -Use the Extensions menu to select Manage Wikidata account and you will be presented with the following window: - -![The Wikidata authorization window in OpenRefine.](/img/wikidata-login.png) - -For security reasons, you should not use your main account authorization with OpenRefine. Wikidata allows you to set special passwords to access your account through software. You can find [this setting for your account here](https://www.wikidata.org/wiki/Special:BotPasswords) once logged in. Creating bot access will prompt you for a unique name. You should then enable the following required settings: -* High-volume editing -* Edit existing pages -* Create, edit, and move pages - -It will then generate a username (in the form of “yourwikidatausername@yourbotname”) and password for you to use with OpenRefine. - -If your account or your bot is not properly authorized, OpenRefine will not display a warning or error when you try to upload your edits. - -You can store your unencrypted username and password in OpenRefine, saved locally to your computer and available for future use. For security reasons, you may wish to leave this box unchecked. You can also save your OpenRefine-specific bot password in your browser or with a password management tool. - -### Import and export schema {#import-and-export-schema} - -You can save time on repetitive processes by defining a schema on one project, then exporting it and importing for use on new datasets in the future. Or you and your colleagues can share a schema with each other to coordinate your work. - -You can export a schema from a project using ExportWikidata schema, or by using ExtensionsExport schema. OpenRefine will generate a JSON file for you to save and share. You may experience issues with pop-up windows in your browser: consider allowing pop-ups from the OpenRefine URL (`127.0.0.1`) from now on. - -You can import a schema using ExtensionsImport schema. You can upload a JSON file, or paste JSON statements directly into a field in the window. An imported schema will look for columns with the same names, and you will see an error message if your project doesn't contain matching columns. - -### Upload edits to Wikidata {#upload-edits-to-wikidata} - -There are two menu options in OpenRefine for applying your edits to Wikidata, and the details of the differences between the two can be found in the [Uploading page](./uploading). Under Export you will see Wikidata edits... and under Extensions you will see Upload edits to Wikidata. Both will bring up the same window for you to [log in with a Wikidata account](#manage-wikidata-account). - -Once you are authorized, you will see a window with any outstanding issues. You can ignore these issues, but we recommend you resolve them. - -If you are ready to upload your edits, you can provide an “Edit summary” - a short message describing the batch of edits you are making. It can be helpful to leave notes for yourself, such as “batch 1: authors A-G” or other indicators of your workflow progress. OpenRefine will show the progress of the upload as it is happening, but does not show a confirmaton window. - -If your edits have been successful, you will see them listed on [your Wikidata user contributions page](https://www.wikidata.org/wiki/Special:Contributions/), and on the [Edit groups page](https://editgroups.toolforge.org/). All edits can be undone from this second interface. - -### QuickStatements export {#quickstatements-export} - -Your OpenRefine data can be exported in a format recognized by [QuickStatements](https://www.wikidata.org/wiki/Help:QuickStatements), a tool that creates Wikidata edits using text commands. OpenRefine generates “version 1” QuickStatements commands. - -There are advantages to using QuickStatements rather than uploading your edits directly to Wikidata, including the way QuickStatements resolves duplicates and redundancies. You can learn more on QuickStatements' [Help page](https://www.wikidata.org/wiki/Help:QuickStatements), and on OpenRefine's [Uploading page](https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Uploading). - -In order to use QuickStatements, you must authorize it with a Wikidata account that is [autoconfirmed](https://www.wikidata.org/wiki/Wikidata:Autoconfirmed_users) (it may appear as “MediaWiki” when you authorize). - -Follow the [steps listed on this page](https://www.wikidata.org/wiki/Help:QuickStatements#Running_QuickStatements). -To prepare your OpenRefine data into QuickStatements, select ExportQuickStatements file, or ExtensionsExport to QuickStatements. Exporting your schema from OpenRefine will generate a text file called `statements.txt` by default. Paste the contents of the text file into a new QuickStatements batch using version 1. You can find [version 1 of the tool (no longer maintained) here](https://wikidata-todo.toolforge.org/quick_statements.php). The text commands will be processed into Wikidata edits and previewed for you to review before submitting. - -### Issue detection {#issue-detection} - -This section is an overview of the [Quality assurance page](./quality-assurance). - -OpenRefine will analyze your schema and make suggestions. It does not check for conflicts in your proposed edits, or tell you about redundancies. - -One of the most common suggestions is to attach [a reference to your edits](https://www.wikidata.org/wiki/Help:Sources) - a citation for where the information can be found. This can be a book or newspaper citation, a URL to an online page, a reference to a physical source in an archival or special collection, or another source. If the source is itself an item on Wikidata, use the relationship [stated in (P248)](https://www.wikidata.org/wiki/Property:P248); otherwise, use [reference URL (P854)](https://www.wikidata.org/wiki/Property:P854) to identify an external source. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/quality-assurance.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/quality-assurance.md deleted file mode 100644 index a8ead43e3..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/quality-assurance.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -id: quality-assurance -title: Quality assurance for Wikibase uploads -sidebar_label: Quality assurance ---- - -This page explains how the Wikidata extension of OpenRefine analyzes edits before they are uploaded to the Wikibase -instance. Most of these checks rely on the use of the [Wikibase Quality Constraints](https://gerrit.wikimedia.org/g/mediawiki/extensions/WikibaseQualityConstraints) extension and the configuration of the property and item identifiers in the [Wikibase manifest](./configuration). - -## Overview {#overview} - -Changes are scrutinized before they are uploaded, but also before the current content of the corresponding items is retrieved and merged with the updates. This means that some constraint violations cannot be predicted by the software (for instance, adding a new statement that conflicts with an existing statement on the item). However, this makes it possible to run the checks quickly, even for relatively large batches of edits. Issues are therefore refreshed in real time while the user builds the schema. - -As a consequence, not all constraint violations can be detected: the ones that are supported are listed in the [Constraint violations](#constraint-violations) section. Conversely, not all issues reported will be flagged as constraint violations on the Wikibase site: see [Generic issues](#generic-issues) for these. - -## Reconciliation {#reconciliation} - -You should always assess the quality of your reconciliation results first. OpenRefine has various tools for quality assurance of reconciliation results. For instance: - -* you can analyze the string similarity between your original names and those of the reconciled items (for instance with ReconcileFacetsBest candidate's name edit distance); -* you can compare the values in your table with those on the items (via a text facet defined by a custom expression); -* you can facet by type on the reconciled items (add a new column with the types and use a text facet ordered by counts to get a sense of the distribution of types in your reconciled items). - -## Constraint violations {#constraint-violations} - -Constraints are retrieved as defined on the properties, using [ (P2302)](https://www.wikidata.org/wiki/Property:P2302). - -The following constraints are supported: -* [format constraint (Q21502404)](https://www.wikidata.org/wiki/Q21502404), checked on all values -* [inverse constraint (Q21510855)](https://www.wikidata.org/wiki/Q21510855): OpenRefine assumes that the inverses of the candidate statements are not in Wikidata yet. If you know that the inverse statements are already in Wikidata, you can safely ignore this issue. -* [used for values only constraint (Q21528958)](https://www.wikidata.org/wiki/Q21528958), [used as qualifier constraint (Q21510863)](https://www.wikidata.org/wiki/Q21510863) and [used as reference constraint (Q21528959)](https://www.wikidata.org/wiki/Q21528959) -* [allowed qualifiers constraint (Q21510851)](https://www.wikidata.org/wiki/Q21510851) -* [required qualifier constraint (Q21510856)](https://www.wikidata.org/wiki/Q21510856) -* [single-value constraint (Q19474404)](https://www.wikidata.org/wiki/Q19474404): this will only trigger if you are adding more than one statement with the property on the same item, but will not detect any existing statement with this property. -* [distinct values constraint (Q21502410)](https://www.wikidata.org/wiki/Q21502410): similarly, this only checks for conflicts inside your edit batch. - -A comparison of the supported constraints with respect to other implementations is available [here](https://www.wikidata.org/wiki/Wikidata:WikiProject_property_constraints/reports/implementations). - -## Generic issues {#generic-issues} - -OpenRefine also detects issues that are not flagged (yet) by constraint violations on Wikidata: -* Statements without references. This does not rely on [citation needed constraint (Q54554025)](https://www.wikidata.org/wiki/Q54554025): all statements are expected to have references. (The idea is that when importing a dataset, every statement you add -* should link to this dataset - it does not hurt to do it even for generic properties such as [instance of (P31)](https://www.wikidata.org/wiki/Property:P31).) -* Spurious whitespace and non-printable characters in strings (including labels, descriptions and aliases); -* Self-referential statements (statements which mention the item they belong to); -* New items created without any label; -* New items created without any description; -* New items created without any [instance of (P31)](https://www.wikidata.org/wiki/Property:P31) or [subclass of (P279)](https://www.wikidata.org/wiki/Property:P279) statement. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/reconciling.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/reconciling.md deleted file mode 100644 index 0e5625c9f..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/reconciling.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -id: reconciling -title: Reconciling with Wikibase -sidebar_label: Reconciling with Wikibase ---- - -The Wikidata [reconciliation service](reconciling) for OpenRefine [supports](https://reconciliation-api.github.io/testbench/): -* A large number of potential types to reconcile against -* Previewing and viewing entities -* Suggesting entities, types, and properties -* Augmenting your project with more information pulled from Wikidata. - -You can find documentation and further resources on the reconciliation API [here](https://wikidata.reconci.link/). - -For the most part, Wikidata reconciliation behaves the same way other reconciliation services do, but there are a few processes and features specific to Wikidata. - -## Language settings {#language-settings} - -You can install a version of the Wikidata reconciliation service that uses your language. First, you need the language code: this is the [two-letter code found on this list](https://en.wikipedia.org/wiki/List_of_Wikipedias), or in the domain name of the desired Wikipedia/Wikidata (for instance, “fr” if your Wikipedia is https://fr.wikipedia.org/wiki/). - -Then, open the reconciliation window (under ReconcileStart reconciling...) and click Add Standard Service. The URL to enter is `https://wikidata.reconci.link/fr/api`, where “fr” is your desired language code. - -When reconciling using this interface, items and properties will be displayed in your chosen language if the label is available. The matching score of the reconciliation is not influenced by your choice of language for the service: items are matched by considering all labels and returning the best possible match. The language of your dataset is also irrelevant to your choice of language for the reconciliation service; it simply determines which language labels to return based on the entity chosen. - -## Restricting matches by type {#restricting-matches-by-type} - -In Wikidata, types are items themselves. For instance, the [university of Ljubljana (Q1377)](https://www.wikidata.org/wiki/Q1377) has the type [public university (Q875538)](https://www.wikidata.org/wiki/Q875538), using the [instance of (P31)](https://www.wikidata.org/wiki/Property:P31) property. Types can be subclasses of other types, using the [subclass of (P279)](https://www.wikidata.org/wiki/Property:P279) property. For instance, [public university (Q875538)](https://www.wikidata.org/wiki/Q875538) is a subclass of [university (Q3918)](https://www.wikidata.org/wiki/Q3918). You can visualize these structures with the [Wikidata Graph Builder](https://angryloki.github.io/wikidata-graph-builder/). - -When you select or enter a type for reconciliation, OpenRefine will include that type and all of its subtypes. For instance, if you select [university (Q3918)](https://www.wikidata.org/wiki/Q3918), then [university of Ljubljana (Q1377)](https://www.wikidata.org/wiki/Q1377) will be a possible match, though that item isn't directly linked to Q3918 - because it is directly linked to Q875538, the subclass of Q3918. - -Some items and types may not yet be set as an instance or subclass of anything (because Wikidata is crowdsourced). If you restrict reconciliation to a type, items without the chosen type will not appear in the results, except as a fallback, and will have a lower score. - -## Reconciling via unique identifiers {#reconciling-via-unique-identifiers} - -You can supply a column of unique identifiers (in the form "Q###" for entities) directly to Wikidata in order to pull more data, but [these strings will not be “reconciled” against the external dataset](reconciling#reconciling-with-unique-identifiers). Apply the operation ReconcileUse values as identifiers on your column of QIDs. All cells will appear as dark blue “confirmed” matches. Some of the “matches” may be errors, which you will need to hover over or click on to identify. You cannot use this to reconcile properties (in the form "P###"). - -If the identifier you submit is assigned to multiple Wikidata items (because Wikidata is crowdsourced), all of the items are returned as candidates, with none automatically matched. - -## Property paths, special properties, and subfields {#property-paths-special-properties-and-subfields} - -Wikidata's hierarchical property structure can be called by using property paths (using |, /, and . symbols). Labels, aliases, descriptions, and sitelinks can also be accessed. You can also match values against subfields, such as latitude and longitude subfields of a geographical coordinate. - -For information on how to do this, read the [documentation and further resources here](https://wikidata.reconci.link/#documentation). - - diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/schema-alignment.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/schema-alignment.md deleted file mode 100644 index 112c1a45c..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/schema-alignment.md +++ /dev/null @@ -1,253 +0,0 @@ ---- -id: schema-alignment -title: Schema alignment -sidebar_label: Schema alignment ---- - -A Wikibase schema is a template of Wikidata edits that is applied -to each row in the project. This page describes how each part of this -template works, and how it generates edits depending on the contents of -the table cells. - -## Items {#items} - -An item in the schema represents a set of changes on a particular -Wikidata item, generated by a single row. This item can contain changes -in [terms](#terms) (labels, descriptions and aliases) or -[statements](#statements). - -It is possible to make edits on different items for each row of your -table: just add multiple items in your schema. Each item has a subject, -which can be either entered manually (when the item on which the edits -should be made is the same for all rows), or any reconciled column can -be dropped in this field. In this case, the edits will depend on the -reconciliation status of each cell: - -- If the cell is matched to an item, edits will be made on that item; -- If the cell is marked as corresponding to a new item, a new item - will be created for it. See [New items](./new-entities) for more - details about how this works; -- If the cell has reconciliation candidates but has not been matched - to any of them, the edit will be skipped (even if there is only one - candidate with a high reconciliation score); -- If the cell is not reconciled or blank, the edit will be skipped. - -Do not worry about the ordering of items in the schema or the order of -your rows, as OpenRefine will rearrange your edits to optimize their -upload. If your project makes edits on the same item across multiple -rows, these edits will be merged together and performed in one edit. See -[Uploading your changes](./uploading) about that. - -## Terms {#terms} - -**Terms** are the language-specific strings that you find at the top of -Wikidata items: labels, descriptions and aliases. OpenRefine lets you -edit these terms via the Wikidata schema. - -### Languages {#languages} - -Each term belongs to a particular language. Wikidata supports [hundreds -of languages](https://www.wikidata.org/wiki/Help:Wikimedia_language_codes/lists/all), which -are designated by language codes. For each term that you want to add to -an item, you will need to specify the language for this term. There are -two cases: - -- Either the language is constant across your dataset: you know that - all the names in a given column are spelled in the same language. In - this case, type the name of the language in the input and select the - language in the drop-down suggestion dialog. This will place the - appropriate language code in the input. -- Or the language varies across your dataset. In this case, you need - to provide a column of Wikimedia language codes that indicates the - language for each term that you want to add. Just drag and drop this - column to the language field. If there are any invalid language - codes in this column, the corresponding terms will be ignored. - OpenRefine will translate any deprecated language codes to their - preferred values silently. - -### Labels {#labels} - -This is because Wikidata items can have at most one label per language, -so you need to choose whether to override any existing label (default -behaviour before 3.2) or only insert your label if there is no such -label in the given language (default behaviour starting from 3.2). When -the content of the cell providing the label is blank, nothing will be -changed (so, it is not possible to remove labels). - -### Descriptions {#descriptions} - -Descriptions work like labels: there is at most one description per -language, and OpenRefine can override existing descriptions or leave -them unchanged. It is not possible to remove descriptions either. - -### Aliases {#aliases} - -Aliases are added to the list of existing aliases in the given language. -When adding an alias in a language where no label has been added yet, -the alias is automatically promoted to a label for this language. It is -not possible to remove aliases or to override any existing aliases. - -## Statements {#statements} - -You can add statements in the schema: this will generate new statements -on the corresponding items. These statements will be merged with any -existing statements on the actual Wikidata items and [this merging process depends on the upload medium](./uploading#Merging-strategies-for-statements). -It is forecast to give more control over the merging strategy in the -near future. - -### Main values {#main-values} - -Statements must have main values: \"novalue\" or \"somevalue\" -statements are not supported yet. The main value of a statement is a -data value whose type depends on the property used for the statement. If -the main value cannot be evaluated (for instance because one of the -cells it depends on is empty), then the entire statement will be -skipped. - -See the [data values](#data-values) section for more details -about how to specify each type of data value and when they are skipped. - -### Qualifiers {#qualifiers} - -Qualifiers can be added on each statement. When their values are -skipped, only the qualifier will be discarded: the rest of the statement -will still be added. - -### References {#references} - -References can (and should) be added to back each statement. If values -inside the reference are skipped, the corresponding part of the -reference will be discarded but the reference will still be added -(unless the reference becomes empty). - -### Ranks {#ranks} - -All statements ranks are set to **Normal**. It is currently not possible -to set a different rank. - -## Data values {#data-values} - -Data values are the data that you can find as target of a statement (or -qualifier, or part of a reference). Each property dictates a particular -type of data value. In each case, OpenRefine uses a particular process -to translate cell contents to a data value of the appropriate type. This -section explains the process for all data types. - -### Items {#items-1} - -Items are evaluated in the same way as the subjects of items in the -schema. They can be input directly using the auto-suggest service -provided, or any column reconciled against Wikidata can be used. Refer to -[the first Items section](#items) to see how they are -evaluated. - -### Strings and external identifiers {#strings-and-external-identifiers} - -Bare strings and external identifiers can be input directly as constants -(if they do not change across rows) or using any column. If a reconciled -column is used for a string value, it is the value of the cell that is -going to be used, not the name of the reconciled item (which is what -OpenRefine displays). Values are skipped when the column is blank or -null. - -### Monolingual texts {#monolingual-texts} - -Monolingual texts consist of two parts: - -- the language: see [Languages](#languages) for their - structure; -- the value of the text: see [the section above](#strings-and-external-identifiers). - -A monolingual text is skipped when any of its parts is skipped (that is, -if the language or the text are invalid). - -### Dates {#dates} - -Dates are parsed from cell contents (or from any constant provided in -the schema) and the precision of the date is inferred from its format. -Here are the valid formats: - -- `YYYYM`, such as `2001M` (millenium precision) -- `YYYYC`, such as `1901C` (century precision) -- `YYYYD`, such as `1981D` (decade precision) -- `YYYY`, such as `1984` (year precision) -- `YYYY-MM`, such as `2019-03` (month precision) -- `YYYY-MM-DD`, such as `1897-08-14` (day precision) - -Any value that does not match any of these formats will be ignored. All -dates are represented in UTC, Gregorian calendar. - -In OpenRefine 3.3, the following new formats have been introduced: - -- `TODAY` returns today's date with day precision. This will be - evaluated when performing the edits (or exporting to - QuickStatements); -- `YYYY-MM-DD_QID` can be used to specify a date in a particular - calendar (such as the [proleptic Julian calendar (Q1985786)](https://www.wikidata.org/wiki/Q1985786). - -In OpenRefine 3.5, the following new format has been introduced: - -- `-234` represents the year 234 [BCE](https://en.wikipedia.org/wiki/Common_Era) - -### Quantities {#quantities} - -Quantities consist of two parts: the amount and the unit. - -- the amount is mandatory and must be a string, such as `18,229.1020`. - The precision that is displayed will be respected (the same number - of trailing zeros will be shown in Wikidata). By default, no upper - and lower bounds will be set. To define these, one needs to use the - engineering notation, such as `3.45E+3`, which will be interpreted - as `3,450±5`. As usual, the amount can be provided as a constant or - as a column variable. In the latter case, the values in the column - must be strings. -- the unit is optional. It is an item, so it can be provided either - with the auto-suggest dialog or as a reconciled column. It is - important to note that if a reconciled column is used, any - unreconciled cells will discard the entire quantity value. So a - template for a quantity value is either always unit-less, or always - has a unit. - -### Globe coordinates {#globe-coordinates} - -Geographic coordinates are specified as strings with the following -formats, where all components are floating point numbers in degrees: - -- `latitude,longitude` for a default precision of ten micro degrees - (for instance: - [`49.265278,4.028611`](https://tools.wmflabs.org/geohack/geohack.php?params=49.265277777778_N_4.0286111111111_E_globe:earth&language=en) - can be used indicate the position of Reims, France. - - -- `latitude,longitude,precision` when specifying an explicit precision - (for instance: `49.265278,4.028611,0.1` can be used indicate the - position of Reims within a tenth of a degree). - -All globe coordinates are on Earth ([Q2](https://www.wikidata.org/wiki/Q2)). - -If your coordinates are in a different format, such as -`49° 15′ 55″ N, 4° 1′ 43″ E`, you will need to convert them to decimal -format first. - -### Media on Commons {#media-on-commons} - -Media on Wikimedia Commons is treated like strings, whose values must -exactly match filenames on Commons. These values are not checked during -schema evaluations: if they are wrong, uploading the statements will -fail. - -Tabular data and Geoshapes must be prefixed with the `Data:` namespace. -This is indicated by the placeholder in the field that appears when -constructing the schema. - -### Properties {#properties} - -Properties are always constants: there is currently no way to reconcile -a column against properties. They have to be selected with the -auto-suggest dialog. - -### Other data types {#other-data-types} - -URLs, mathematical expressions and other textual datatypes are supported -and treated as strings. At the time of writing, all datatypes supported -by Wikidata are supported by OpenRefine. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/uploading.md b/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/uploading.md deleted file mode 100644 index 99da1259d..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/manual/wikibase/uploading.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -id: uploading -title: Uploading edits to Wikibase -sidebar_label: Uploading edits ---- - -This page explains how to upload your edits to the target Wikibase. It assumes you already have a created a Wikibase schema in your OpenRefine project. - -## Uploading with OpenRefine {#uploading-with-openrefine} - -* Click WikidataUpload edits to Wikidata. -* Log in with your personal account or your bot account depending on which account you want to use to make the edits. It is a good practice to use a [bot password](https://www.mediawiki.org/wiki/Manual:Bot_passwords). -* Supply a meaningful edit summary. This is especially important because OpenRefine condenses all your changes on the same item as one edit: if you are making multiple changes, the edit summary generated by Wikibase will not indicate clearly what sort of change you made. If you are making atomic changes, such as adding a single alias or statement, the automatic edit summaries will be more meaningful. If supported by your Wikibase instance, OpenRefine will append a link to the [EditGroups](https://editgroups.toolforge.org/) tool, which lets you track and analyze your edit batch after upload. -* Click Perform edits and wait for the operation to complete. You can watch your edits being made by checking your wiki contributions or the EditGroups tool. - -Because performing edits in OpenRefine counts as an operation, you can extract this operation and reapply it to other projects. If you do so, you should also include the operation that saves the schema (only the last one is required), and make sure that the column names in the schema match those of the OpenRefine project where you are applying the operation. - -## Uploading with QuickStatements {#uploading-with-quickstatements} - -This requires that the Wikibase site has an associated [QuickStatements](https://meta.wikimedia.org/wiki/QuickStatements) tool. - -* Click WikibaseExport to QuickStatements and copy the contents of the file; -* Go to QuickStatements (for Wikidata it can be found at https://quickstatements.toolforge.org/) and login to authorize the tool to use your account; -* Click Version 1 format; -* Paste the generated changes in the text area; -* Perform the edits with Run or Run in background. - -## Notable differences between the two methods {#notable-differences-between-the-two-methods} - -### Merging strategy for statements {#merging-strategy-for-statements} - -OpenRefine checks for existing statements which match not only the property and the target value, but also the qualifiers. On the other hand, QuickStatements ignores qualifiers when matching statements. -Both merging strategies can be useful depending on the properties. It is forecast to let the user configure the matching method in OpenRefine. - -If references are provided, both tools merge references in matching statements. - -### New item creation {#new-item-creation} - -OpenRefine supports creating new items with arbitrary relations between them. - -QuickStatements supports creating new items with the CREATE instruction, and subsequent instructions can use the LAST placeholder to use the Qid of the last created item. When generating QuickStatements instructions, OpenRefine reorders your edits so that this syntax can be used. In rare cases, such as when a statement links two newly-created items, it is impossible to use QuickStatements to perform the edit. In this case, no QuickStatements script will be generated. - -### Speed and number of edits {#speed-and-number-of-edits} - -OpenRefine generally performs one edit per item touched by an edit batch and at most two in general (in the case where new items contain links between them). This was chosen to minimize server load, speed up the upload and keep item histories compact. The downside is that the edit summaries can be less meaningful - it is therefore important that users supply informative summaries when uploading their batches. OpenRefine asymptotically edits at the rate of 60 edits per minute (so, usually 60 items per minute). The first edits are made more quickly, which is convenient for small batches. - -QuickStatements performs incremental edits (for instance, when adding a statement with a qualifier and a reference, it will make three edits). That generally means lower speed, but more explicit item histories. - diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/architecture.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/architecture.md deleted file mode 100644 index e80d7015a..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/architecture.md +++ /dev/null @@ -1,283 +0,0 @@ ---- -id: architecture -title: Architecture -sidebar_label: Architecture ---- - -OpenRefine is a web application, but is designed to be run locally on your own machine. The server-side maintains states of the data (undo/redo history, long-running processes, etc.) while the client-side maintains states of the user interface (facets and their selections, view pagination, etc.). The client-side makes GET and POST ajax calls to cause changes to the data and to fetch data and data-related states from the server-side. - -This architecture provides a good separation of concerns (data vs. UI); allows the use of familiar web technologies (HTML, CSS, Javascript) to implement user interface features; and enables the server side to be called by third-party software through standard GET and POST operations. - -## Technology stack {#technology-stack} - -The server-side part of OpenRefine is implemented in Java as one single servlet which is executed by the [Jetty](http://jetty.codehaus.org/jetty/) web server + servlet container. The use of Java strikes a balance between performance and portability across operating systems (there is very little OS-specific code and has mostly to do with starting the application). - -OpenRefine has no database. It uses its own in-memory data-store that is built up-front to be optimized for the operations required by faceted browsing and infinite undo. - -The client-side part of OpenRefine is implemented in HTML, CSS and Javascript and uses the following libraries: -* [jQuery](http://jquery.com/) -* [jQueryUI](http:jqueryui.com/) -* [Recurser jquery-i18n](https://github.com/recurser/jquery-i18n) - -The functional extensibility of OpenRefine is provided by a fork of the [SIMILE Butterfly](https://github.com/OpenRefine/simile-butterfly) modular web application framework. - -Several projects provide the functionality to read and write custom format files (POI, opencsv, JENA, marc4j). - -String clustering is provided by the [SIMILE Vicino](http://code.google.com/p/simile-vicino/) project. - -OAuth functionality is provided by the [Signpost](https://github.com/mttkay/signpost) project. - -## Server-side architecture {#server-side-architecture} - -OpenRefine's server-side is written entirely in Java (`main/src/`) and its entry point is the Java servlet `com.google.refine.RefineServlet`. By default, the servlet is hosted in the lightweight Jetty web server instantiated by `server/src/com.google.refine.Refine`. Note that the server class itself is under `server/src/`, not `main/src/`; this separation leaves the possibility of hosting `RefineServlet` in a different servlet container. - -The web server configuration is in `main/webapp/WEB-INF/web.xml`; that's where `RefineServlet` is hooked up. `RefineServlet` itself is simple: it just reacts to requests from the client-side by routing them to the right `Command` class in the packages `com.google.refine.commands.**`. - -As mentioned before, the server-side maintains states of the data, and the primary class involved is `com.google.refine.ProjectManager`. - -### Projects {#projects} - -In OpenRefine there's the concept of a workspace similar to that in Eclipse. When you run OpenRefine it manages projects within a single workspace, and the workspace is embodied in a file directory with sub-directories. The default workspace directories are listed in the [FAQs](https://github.com/OpenRefine/OpenRefine/wiki/FAQ-Where-Is-Data-Stored). You can get OpenRefine to use a different directory by specifying a -d parameter at the command line. - -The class `ProjectManager` is what manages the workspace. It keeps in memory the metadata of every project (in the class `ProjectMetadata`). This metadata includes the project's name and last modified date, and any other information necessary to present and let the user interact with the project as a whole. Only when the user decides to look at the project's data would `ProjectManager` load the project's actual data. The separation of project metadata and data is to minimize the amount of stuff loaded into memory. - -A project's _actual_ data includes the columns, rows, cells, reconciliation records, and history entries. - -A project is loaded into memory when it needs to be displayed or modified, and it remains in memory until 1 hour after the last time it gets modified. Periodically the project manager tries to save modified projects, and it saves as many modified projects as possible within 30 seconds. - -### Data Model {#data-model} - -A project's data consists of - -- _raw data_: a list of rows, each row consisting of a list of cells -- _models_ on top of that raw data that give high-level presentation or interpretation of that data. This design lets the same raw data be viewed in different ways by different models, and let the models be changed without costly changes to the raw data. - -#### Column Model {#column-model} - -Cells in rows are not named and can only be addressed by their list position indices. So, a _column model_ is needed to give a name to each list position. The column model also stores other metadata for each column, including the type that cells in the column have been reconciled to and the overall reconciliation statistics of those cells. - -Each column also acts as a cache for data computed from the raw data related to that column. - -Columns in the column model can be removed and re-ordered without changing the raw data--the cells in the rows. This makes column removal and ordering operations really quick. - -##### Column Groups {#column-groups} - -Consider the following data: - -![Illustration of row groups in OpenRefine](https://raw.github.com/OpenRefine/OpenRefine/2.0/graphics/row-groups.png) - -Although the data is in a grid, we humans can understand that it is a tree. First of all, all rows contain data ultimately linked to the movie Austin Powers, although only one row contains the text "Austin Powers" in the "movie title" column. We also know that "USA" and "Germany" are not related to Elizabeth Hurley and Mike Myers respectively (say, as their nationality), but rather, "USA" and "Germany" are related to the movie (where it was released). We know that Mike Myers played both the character "Austin Powers" and the character "Dr. Evil"; and for the latter he received 2 awards. We humans can understand how to interpret the grid as a tree based on its visual layout as well as some knowledge we have about the movie domain but is not encoded in the table. - -OpenRefine can capture our knowledge of this transformation from grid to tree using _column groups_, also stored in the column model. Each column group illustrated as a blue bracket above specifies which columns are grouped together, as well as which of those columns is the key column in that group (blue triangle). One column group can span over columns grouped by another column group, and in this way, column groups form a hierarchy determined by which column group envelopes another. This hierarchy of column groups allows the 2-dimensional (grid-shaped) table of rows and cells to be interpreted as a list of hierarchical (tree-shaped) data records. - -Blank cells play a very important role. The blank cell in a key column of a row (e.g., cell "character" on row 4) makes that row (row 4) _depend_ on the first preceding row with that column filled in (row 3). This means that "Best Comedy Perf" on row 4 applies to "Dr. Evil" on row 3. Row 3 is said to be a _context row_ for row 4. Similarly, since rows 2 - 6 all have blank cells in the first column, they all depend on row 1, and all their data ultimately applies to the movie Austin Powers. Row 1 depends on no other row and is said to be a _record row_. Rows 1 - 6 together form one _record_. - -Currently (as of 12th December 2017) only the XML and JSON importers create column groups, and while the data table view does display column groups but it doesn't support modifying them. - -### Changes, History, Processes, and Operations {#changes-history-processes-and-operations} - -All changes to the project's data are tracked (N.B. this does not include changes to a project's metadata - such as the project name.) - -Changes are stored as `com.google.refine.history.Change` objects. `com.google.refine.history.Change` is an interface, and implementing classes are in `com.google.refine.model.changes.**`. Each change object stores enough data to modify the project's data when its `apply()` method is called, and enough data to revert its effect when its `revert()` method is called. It's only supposed to _store_ data, not to actually _compute_ the change. In this way, it's like a .diff patch file for a code base. - -Some change objects can be huge, as huge as the project itself. So change objects are not kept in memory except when they are to be applied or reverted. However, since we still need to show the user some information about changes (as displayed in the History panel in the UI), we keep metadata of changes separate from the change objects. For each change object there is one corresponding `com.google.refine.history.HistoryEntry` for storing its metadata, such as the change's human-friendly description and timestamp. - -Each project has a `com.google.refine.history.History` object that contains an ordered list of all `HistoryEntry` objects storing metadata for all changes that have been done since after the project was created. Actually, there are 2 ordered lists: one for done changes that can be reverted (undone), an done for undone changes that can be re-applied (redone). Changes must be done or redone in their exact orders in these lists because each change makes certain assumptions about the state of the project before and after it is applied. As changes cannot be undone/redone out of order, when one change fails to revert, it blocks the whole history from being reverted to any state preceding that change (as happened in [Issue #2](https://github.com/OpenRefine/OpenRefine/issues/2)). - -As mentioned before, a change contains only the diff and does not actually compute that diff. The computation is performed by a `com.google.refine.process.Process` object--every change object is created by a process object. A process can be immediate, producing its change object synchronously within a very short period of time (e.g., starring one row); or a process can be long-running, producing its change object after a long time and a lot of computation, including network calls (e.g., reconciling a column). - -As the user interacts with the UI on the client-side, their interactions trigger ajax calls to the server-side. Some calls are meant to modify the project. Those are handled by commands that instantiates processes. Processes are queued in a first-in-first-out basis. The first-in process gets run and until it is done all the other processes are stuck in the queue. - -A process can effect a change in one thing in the project (e.g., edit one particular cell, star one particular row), or a process can effect changes in _potentially_ many things in the project (e.g., edit zero or more cells sharing the same content, starring all rows filtered by some facets). The latter kind of process is generalizable: it is meaningful to apply them on another similar project. Such a process is associated with an _abstract operation_ `com.google.refine.model.AbstractOperation` that encodes the information necessary to create another instance of that process, but potentially for a different project. When you click "extract" in the History panel, these abstract operations are called to serialize their information to JSON; and when you click "apply" in the History panel, the JSON you paste in is used to re-construct these abstract operations, which in turn create processes, which get run sequentially in a queue to generate change object and history entry pairs. - -In summary, - -- change objects store diffs -- history entries store metadata of change objects -- processes compute diffs and create change object and history entry pairs -- some processes are long-running and some are immediate; processes are run sequentially in a queue -- generalizable processes can be re-constructed from abstract operations - -## Client-side architecture {#client-side-architecture} -The client-side part of OpenRefine is implemented in HTML, CSS and Javascript and uses the following Javascript libraries: -* [jQuery](http://jquery.com/) -* [jQueryUI](http:jqueryui.com/) -* [Recurser jquery-i18n](https://github.com/recurser/jquery-i18n) - -### Importing architecture {#importing-architecture} - -OpenRefine has a sophisticated architecture for accommodating a diverse and extensible set of importable file formats and work flows. The formats range from simple CSV, TSV to fixed-width fields to line-based records to hierarchical XML and JSON. The work flows allow the user to preview and tweak many different import settings before creating the project. In some cases, such as XML and JSON, the user also has to select which elements in the data file to import. Additionally, a data file can also be an archive file (e.g., .zip) that contains many files inside; the user can select which of those files to import. Finally, extensions to OpenRefine can inject functionalities into any part of this architecture. - -### The Index Page and Action Areas {#the-index-page-and-action-areas} - -The opening screen of OpenRefine is implemented by the file refine/main/webapp/modules/core/index.vt and will be referred to here as the index page. Its default implementation contains 3 finger tabs labeled Create Project, Open Project, and Import Project. Each tab selects an "action area". The 3 default action areas are for, obviously, creating a new project, opening an existing project, and importing a project .tar file. - -Extensions can add more action areas in Javascript. For example, this is how the Create Project action area is added (refine/main/webapp/modules/core/scripts/index/create-project-ui.js): - -```javascript -Refine.actionAreas.push({ - id: "create-project", - label: "Create Project", - uiClass: Refine.CreateProjectUI -}); -``` - -The UI class is a constructor function that takes one argument, a jQuery-wrapped HTML element where the tab body of the action area should be rendered. - -If your extension requires a very unique importing work flow, or a very novel feature that should be exposed on the index page, then add a new action area. Otherwise, try to use the existing work flows as much as possible. - -### The Create Project Action Area {#the-create-project-action-area} - -The Create Project action area is itself extensible. Initially, it embeds a set of finger tabs corresponding to a variety of "source selection UIs": you can select a source of data by specifying a file on your computer, or you can specify the URL to a publicly accessible data file or data feed, or you can paste in from the clipboard a chunk of data. - -There are actually 3 points of extension in the Create Project action area, and the first is invisible. - -#### Importing Controllers {#importing-controllers} - -The Create Project action area manages a list of "importing controllers". Each controller follows a particular work flow (in UI terms, think "wizard"). Refine comes with a "default importing controller" (refine/main/webapp/modules/core/scripts/index/default-importing-controller/controller.js) and its work flow assumes that the data can be retrieved and cached in whole before getting processed in order to generate a preview for the user to inspect. (If the data cannot be retrieved and cached in whole before previewing, then another importing controller is needed.) - -An importing controller is just programming logic, but it can manifest itself visually by registering one or more data source UIs and one or more custom panels in the Create Project action area. The default importing controller registers 3 such custom panels, which act like pages of a wizard. - -An extension can register any number of importing controller. Each controller has a client-side part and a server-side part. Its client-side part is just a constructor function that takes an object representing the Create Project action area (usually named `createProjectUI`). The controller (client-side) is expected to use that object to register data source UIs and/or create custom panels. The controller is not expected to have any particular interface method. The default importing controller's client-side code looks like this (refine/main/webapp/modules/core/scripts/index/default-importing-controller/controller.js): - -```javascript -Refine.DefaultImportingController = function(createProjectUI) { - this._createProjectUI = createProjectUI; // save a reference to the create project action area - - this._progressPanel = createProjectUI.addCustomPanel(); // create a custom panel - this._progressPanel.html('...'); // render the custom panel - ... do other stuff ... -}; -Refine.CreateProjectUI.controllers.push(Refine.DefaultImportingController); // register the controller -``` - -We will cover the server-side code below. - -#### Data Source Selection UIs {#data-source-selection-uis} - -Data source selection UIs are another point of extensibility in the Create Project action area. As mentioned previously, by default there are 3 data source UIs. Those are added by the default importing controller. - -Extensions can also add their own data source UIs. A data source selection UI object can be registered like so - -```javascript -createProjectUI.addSourceSelectionUI({ - label: "This Computer", - id: "local-computer-source", - ui: theDataSourceSelectionUIObject -}); -``` - -`theDataSourceSelectionUIObject` is an object that has the following member methods: - -- `attachUI(bodyDiv)` -- `focus()` - -If you want to install a data source selection UI that is managed by the default importing controller, then register its UI class with the default importing controller, like so (refine/main/webapp/modules/core/scripts/index/default-importing-sources/sources.js): - -```javascript -Refine.DefaultImportingController.sources.push({ - "label": "This Computer", - "id": "upload", - "uiClass": ThisComputerImportingSourceUI -}); -``` - -The default importing controller will assume that the `uiClass` field is a constructor function and call it with one argument--the controller object itself. That constructor function should save the controller object for later use. More specifically, for data source UIs that use the default importing controller, they can call the controller to kickstart the process that retrieves and caches the data to import: - -```javascript -controller.startImportJob(form, "... status message ..."); -``` - -The argument `form` is a jQuery-wrapped FORM element that will get submitted to the server side at the command /command/core/create-importing-job. That command and the default importing controller will take care of uploading or downloading the data, caching it, updating the client side's progress display, and then showing the next importing step when the data is fully cached. - -See refine/main/webapp/modules/core/scripts/index/default-importing-sources/sources.js for examples of such source selection UIs. While we write about source selection UIs managed by the default importing controller here, chances are your own extension will not be adding such a new source selection UI. Your extension probably adds with a new importing controller as well as a new source selection UI that work together. - -#### File Selection Panel {#file-selection-panel} -Documentation not currently available - -#### Parsing UI Panel {#parsing-ui-panel} -Documentation not currently available - -### Server-side Components {#server-side-components} - -#### ImportingController {#importingcontroller} -Documentation not currently available - -#### UrlRewriter {#urlrewriter} -Documentation not currently available - -#### FormatGuesser {#formatguesser} -Documentation not currently available - -#### ImportingParser {#importingparser} -Documentation not currently available - - -## Faceted browsing architecture {#faceted-browsing-architecture} - -Faceted browsing support is core to OpenRefine as it is the primary and only mechanism for filtering to a subset of rows on which to do something _en masse_ (ie in bulk). Without faceted browsing or an equivalent querying/browsing mechanism, you can only change one thing at a time (one cell or row) or else change everything all at once; both kinds of editing are practically useless when dealing with large data sets. - -In OpenRefine, different components of the code need to know which rows to process from the faceted browsing state (how the facets are constrained). For example, when the user applies some facet selections and then exports the data, the exporter serializes only the matching rows, not all rows in the project. Thus, faceted browsing isn't only hooked up to the data view for displaying data to the user, but it is also hooked up to almost all other parts of the system. - -### Engine Configuration {#engine-configuration} - -As OpenRefine is a web app, there might be several browser windows opened on the same project, each in a different faceted browsing state. It is best to maintain the faceted browsing state in each browser window while keeping the server side completely stateless with regard to faceted browsing. Whenever the client-side needs something done by the server, it transfers the entire faceted browsing state over to the server-side. The faceted browsing state behaves much like the `WHERE` clause in a SQL query, telling the server-side how to select the rows to process. - -In fact, it is best to think of the faceted browsing state as just a database query much like a SQL query. It can be passed around the whole system, to any component needing to know which rows to process. It is serialized into JSON to pass between the client-side and the server side, or to save in an abstract operation's specification. The job of the faceted browsing subsystem on the client-side is to let the user interactively modify this "faceted browsing query", and the job of the faceted browsing subsystem on the server side is to resolve that query. - -In the code, the faceted browsing state, or faceted browsing query, is actually called the *engine configuration* or *engine config* for short. It consists mostly of an array facet configurations. For each facet, it stores the name of the column on which the facet is based (or an empty string if there is no base column). Each type of facet has different configuration. Text search facets have queries and flags for case-sensitivity mode and regular expression mode. Text facets (aka list facets) and numeric range facets have expressions. Each list facet also has an array of selected choices, an invert flag, and flags for whether blank and error cells are selected. Each numeric range facet has, among other things, a "from" and a "to" values. If you trace the AJAX calls, you'd see the engine configs being shuttled, e.g., - -```json -{ - "facets" : [ - { - "type": "text", - "name": "Shrt_Desc", - "columnName": "Shrt_Desc", - "mode": "text", - "caseSensitive": false, - "query": "cheese" - }, - { - "type": "list", - "name": "Shrt_Desc", - "columnName": "Shrt_Desc", - "expression": "grel:value.toLowercase().split(\",\")", - "omitBlank": false, - "omitError": false, - "selection": [], - "selectBlank":false, - "selectError":false, - "invert":false - }, - { - "type": "range", - "name": "Water", - "expression": "value", - "columnName": "Water", - "selectNumeric": true, - "selectNonNumeric": true, - "selectBlank": true, - "selectError": true, - "from": 0, - "to": 53 - } - ], - "includeDependent": false - } -``` - -### Server-Side Subsystem {#server-side-subsystem} - -From an engine configuration like the one above, the server-side faceted browsing subsystem is capable of producing: - -- an iteration over the rows matching the facets' constraints -- information on how to render the facets (e.g., choice and count pairs for a list facet, histogram for a numeric range facet) - -When the engine config JSON arrives in an HTTP request on the server-side, a `com.google.refine.browsing.Engine` object is constructed and initialized with that JSON. It in turns constructs zero or more `com.google.refine.browsing.facets.Facet` objects. Then for each facet, the engine calls its `getRowFilter()` method, which returns `null` if the facet isn't constrained in anyway, or a `com.google.refine.browsing.filters.RowFilter` object. Then, to when iterating over a project's rows, the engine calls on all row filters' `filterRow()` method. If and only if all row filters return `true` the row is considered to match the facets' constraints. How each row filter works depends on the corresponding type of facet. - -To produce information on how to render a particular facet in the UI, the engine follows the same procedure described in the previous except it skips over the facet in question. In other words, it produces an iteration over all rows constrained by the other facets. Then it feeds that iteration to the facet in question by calling the facet's `computeChoices()` method. This gives the method a chance to compute the rendering information for its UI counterpart on the client-side. When all facets have been given a chance to compute their rendering information, the engine calls all facets to serialize their information as JSON and returns the JSON to the client-side. Only one HTTP call is needed to compute all facets. - -### Client-side subsystem {#client-side-subsystem} - -On the client-side there is also an engine object (implemented in Javascript rather than Java) and zero or more facet objects (also in Javascript, obviously). The engine is responsible for distributing the rendering information computed on the server-side to the right facets, and when the user interacts with a facet, the facet tells the engine to update the whole UI. To do so, the engine gathers the configuration of each facet and composes the whole engine config as a single JSON object. Two separate AJAX calls are made with that engine config, one to retrieve the rows to render, and one to re-compute the rendering information for the facets because changing one facet does affect all the other facets. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/build-test-run.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/build-test-run.md deleted file mode 100644 index 300784ac8..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/build-test-run.md +++ /dev/null @@ -1,283 +0,0 @@ ---- -id: build-test-run -title: How to build, test and run -sidebar_label: How to build, test and run ---- - -import useBaseUrl from '@docusaurus/useBaseUrl'; - - -You will need: -* [OpenRefine source code](https://github.com/OpenRefine/OpenRefine) -* [Java JDK](http://java.sun.com/javase/downloads/index.jsp) (Get [OpenJDK from here](https://jdk.java.net/15/).) -* [Apache Maven](https://maven.apache.org) (OPTIONAL) -* A Unix/Linux shell environment OR the Windows command line - -From the top level directory in the OpenRefine application you can build, test and run OpenRefine using the `./refine` shell script (if you are working in a \*nix shell), or using the `refine.bat` script from the Windows command line. Note that the `refine.bat` on Windows only supports a subset of the functionality, supported by the `refine` shell script. The example commands below are using the `./refine` shell script, and you will need to use `refine.bat` if you are working from the Windows command line. - -### Get OpenRefine source code - -With Git installed, use the `git clone` command to download the [project's repo](https://github.com/OpenRefine/OpenRefine) to a directory of your choice. - -### Set up JDK {#set-up-jdk} - -You must [install JDK](https://jdk.java.net/15/) and set the JAVA_HOME environment variable (please ensure it points to the JDK, and not the JRE). - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - - - - - -1. On Windows 10, click the Start Menu button, type `env`, and look at the search results. Click Edit the system environment variables. (If you are using an earlier version of Windows, use the “Search” or “Search programs and files” box in the Start Menu.) - -![A screenshot of the search results for 'env'.](/img/env.png "A screenshot of the search results for 'env'.") - -2. Click Environment Variables… at the bottom of the Advanced window. -3. In the Environment Variables window that appears, click New… and create a variable with the key `JAVA_HOME`. You can set the variable for only your user account, as in the screenshot below, or set it as a system variable - it will work either way. - -![A screenshot of 'Environment Variables'.](/img/javahome.png "A screenshot of 'Environment Variables'.") - -4. Set the `Value` to the folder where you installed JDK, in the format `D:\Programs\OpenJDK`. You can locate this folder with the Browse directory... button. - - - - - -First, find where Java is on your computer with this command: - -``` -which java -``` - -Check the environment variable `JAVA_HOME` with: - -``` -$JAVA_HOME/bin/java --version -``` - -To set the environment variable for the current Java version of your MacOS: - -``` -export JAVA_HOME="$(/usr/libexec/java_home)" -``` - -Or, for Java 13.x: - -``` -export JAVA_HOME="$(/usr/libexec/java_home -v 13)" -``` - - - - - -##### With the terminal {#with-the-terminal} - -Enter the following: - -``` -sudo apt install default-jre -``` - -This probably won’t install the latest JDK package available on the Java website, but it is faster and more straightforward. (At the time of writing, it installs OpenJDK 11.0.7.) - -##### Manually {#manually} - -First, [extract the JDK package](https://openjdk.java.net/install/) to the new directory `usr/lib/jvm`: - -``` -sudo mkdir -p /usr/lib/jvm -sudo tar -x -C /usr/lib/jvm -f /tmp/openjdk-14.0.1_linux-x64_bin.tar.gz -``` - -Then, navigate to this folder and confirm the final path (in this case, `usr/lib/jvm/jdk-14.0.1`). Open a terminal and type - -``` -sudo gedit /etc/profile -``` - -In the text window that opens, insert the following lines at the end of the `profile` file, using the path above: - -``` -JAVA_HOME=/usr/lib/jvm/jdk-14.0.1 -PATH=$PATH:$HOME/bin:$JAVA_HOME/bin -export JAVA_HOME -export PATH -``` - -Note: OpenRefine on Linux currently supports jdk versions 8 to 15. Reference: [Issue 4106](https://github.com/OpenRefine/OpenRefine/issues/4106). - -Save and close the file. When you are back in the terminal, type - -``` -source /etc/environment -``` - -Exit the terminal and restart your system. You can then check that `JAVA_HOME` is set properly by opening another terminal and typing -``` -echo $JAVA_HOME -``` - -It should show the path you set above. - - - - - ---- - - - -### Maven (Optional) {#maven-optional} -OpenRefine's build script will download Maven for you and use it, if not found already locally installed. - -If you will be using your Maven installation instead of OpenRefine's build script download installation, then set the `MVN_HOME` environment variable. You may need to reboot your machine after setting these environment variables. If you receive a message `Could not find the main class: com.google.refine.Refine. Program will exit.` it is likely `JAVA_HOME` is not set correctly. - -Ensure that you set your `MAVEN_HOME` environment variable, for example: - -```shell -MAVEN_HOME=E:\Downloads\apache-maven-3.5.4-bin\apache-maven-3.5.4\ -``` - -NOTE: You can use Maven commands directly, but running some goals in isolation might fail (try adding the `compile test-compile` goals in your invocation if that is the case). - -### Building {#building} - -To see what functions are supported by OpenRefine's build system, type -```shell -./refine -h -``` - -To build the OpenRefine application from source type: -```shell -./refine clean -./refine build -``` - -### Testing {#testing} -Since OpenRefine is composed of two parts, a server and a in-browser UI, the testing system reflects that: - -* on the server side, it's powered by [TestNG](http://testng.org/) and the unit tests are written in Java; -* on the client side, we use [Cypress](https://www.cypress.io/) and the tests are written in Javascript - -To run all tests, use: -```shell -./refine test -``` -**this option is not available when using refine.bat** - - -If you want to run only the server side portion of the tests, use: -```shell -./refine server_test -``` - -If you are running the UI tests for the first time, [you must go through the installation process.](functional-tests) -If you want to run only the client side portion of the tests, use: -```shell -./refine ui_test chrome -``` - -## Running {#running} -To run OpenRefine from the command line (assuming you have been able to build from the source code successfully) -```shell -./refine -``` -By default, OpenRefine will use [refine.ini](https://github.com/OpenRefine/OpenRefine/blob/master/refine.ini) for configuration. You can copy it and rename it to `refine-dev.ini`, which will be used for configuration instead. `refine-dev.ini` won't be tracked by Git, so feel free to put your custom configurations into it. - -## Building Distributions (Kits) {#building-distributions-kits} - -The Refine build system uses Apache Ant to automate the creation of the installation packages for the different operating systems. The packages are currently optimized to run on Mac OS X which is the only platform capable of creating the packages for all three OS that we support. - -To build the distributions type - -```shell -./refine dist -``` -where 'version' is the release version. - -## Building, Testing and Running OpenRefine from Eclipse {#building-testing-and-running-openrefine-from-eclipse} -OpenRefine' source comes with Maven configuration files which are recognized by [Eclipse](http://www.eclipse.org/) if the Eclipse Maven plugin (m2e) is installed. - -At the command line, go to a directory **not** under your Eclipse workspace directory and check out the source: - -```shell -git clone https://github.com/OpenRefine/OpenRefine.git -``` - -In Eclipse, invoke the `Import...` command and select `Existing Maven Projects`. - -![Screenshot of Import a Maven project option](/img/eclipse-import-maven-project-1.png) - -Choose the root directory of your clone of the repository. You get to choose which modules of the project will be imported. You can safely leave out the `packaging` module which is only used to generate the Linux, Windows and MacOS distributions. - -Screenshot of Select maven projects to import - -To run and debug OpenRefine from Eclipse, you will need to add an execution configuration on the `server` sub-project. -Right click on the `server` subproject, click `Run as...` and `Run configurations...` and create a new `Maven Build` run configuration. Rename the run configuration `OpenRefine`. Enter the root directory of the project as `Base directory` and use `exec:java` as a Maven goal. - -![Screenshot of Add a run configuration with the exec:java goal](/img/eclipse-exec-config.png) - -This will add a run configuration that you can then use to run OpenRefine from Eclipse. - -## Code style in Eclipse - -You can apply the supplied Eclipse code style (in `IDEs/eclipse/Refine.style.xml`) to make sure Eclipse lints your files according to the existing style. -Pull requests deviating from this style will fail in the CI. - -You can manually apply the code style (regardless of your IDE) with the `mvn formatter:format` command. - -## Testing in Eclipse {#testing-in-eclipse} - -You can run the server tests directly from Eclipse. To do that you need to have the TestNG launcher plugin installed, as well as the TestNG M2E plugin (for integration with Maven). If you don't have it, you can get it by [installing new software](https://help.eclipse.org/2020-03/index.jsp?topic=/org.eclipse.platform.doc.user/tasks/tasks-129.htm) from this update URL http://dl.bintray.com/testng-team/testng-eclipse-release/ - -Once the TestNG launching plugin is installed in your Eclipse, right click on the source folder "main/tests/server/src", select `Run As` -> `TestNG Test`. This should open a new tab with the TestNG launcher running the OpenRefine tests. - -### Test coverage in Eclipse {#test-coverage-in-eclipse} - -It is possible to analyze test coverage in Eclipse with the `EclEmma Java Code Coverage` plugin. It will add a `Coverage as…` menu similar to the `Run as…` and `Debug as…` menus which will then display the covered and missed lines in the source editor. - -### Debug with Eclipse {#debug-with-eclipse} -Here's an example of putting configuration in Eclipse for debugging, like putting values for the Google Data extension. Other type of configurations that can be set are memory, Wikidata login information and more. - -![Screenshot of Eclipse debug configuration](/img/eclipse-debug-config.png) - -## Building, Testing and Running OpenRefine from IntelliJ idea {#building-testing-and-running-openrefine-from-intellij-idea} - -At the command line, go to a directory you want to save the OpenRefine project and execute the following command to clone the repository: - -```shell -git clone https://github.com/OpenRefine/OpenRefine.git -``` - -Then, open the IntelliJ idea and go to `file -> open` and select the location of the cloned repository. - -![Screenshot of Open option on the IntelliJ File menu](/img/intellij-setup-1.png) - -It will prompt you to add as a maven project as the source code contains a pom.xml file in it. Allow `auto-import` so that it can add it as a maven project. -If it doesn't prompt something like this then you can go on the right side of the IDE and click on maven then, click on `reimport all the maven projects` that will add all the dependencies and jar files required for the project. - -![Screenshot of Maven project controls in IntelliJ](/img/intellij-maven.png) - -After this, you will be able to properly build, test, and run the OpenRefine project from the terminal. -But if you will go to any of the test folders and open some file it will show you some import errors because the project isn't yet set up at the module level. - -For removing those errors, and enjoying the features of the IDE like ctrl + click, etc you need to set up the project at the module level too. Open the different modules like `extensions/wikidata`, `main` as a project in the IDE. Then, right-click on the project folder and open the module settings. - -![Screenshot of open module settings menu in IntelliJ](/img/intellij-open-module-settings.png) - -In the module settings, add the source folder and test source folders of that module. - -![Screenshot of module settings in IntelliJ](/img/intellij-module-settings.png) - -Then, do the same thing for the main OpenRefine project and now you are good to go. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/contributing.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/contributing.md deleted file mode 100644 index a9151abf7..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/contributing.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -id: contributing -title: Contributing -sidebar_label: Contributing ---- - -Please read the general [guidelines on contributing to OpenRefine](https://github.com/OpenRefine/OpenRefine/blob/master/CONTRIBUTING.md) first, then review the information on [reporting and tracking issues](#reporting-and-tracking-issues), and on making your [first pull request](#your-first-pull-request) below) - -## Reporting and tracking issues {#reporting-and-tracking-issues} - -If you need to file a bug or request a feature, [create an Issue in the OpenRefine Github repository](https://github.com/OpenRefine/OpenRefine/issues). Github issues should be used for reporting specific bugs and requesting specific features. If you just don't know how to do something using OpenRefine, or want to discuss some ideas, please: - -- [Try the user manual](/) -- [post to our OpenRefine mailing list](http://groups.google.com/group/openrefine/) - -## Contributing to the documentation {#contributing-to-the-documentation} - -We use [Docusaurus](https://docusaurus.io/) for our docs. For small documentation changes, you should be able to edit the Markdown files directly and submit them as a pull request. A preview of the docs will be generated automatically. But it is also -possible to preview your changes locally. Assuming you have [Node.js](https://nodejs.org/en/download/) installed (which includes npm), you can install Docusaurus with: - -You will need to install [Yarn](https://yarnpkg.com/getting-started/install) before you can build the site. -```sh -npm install -g yarn -``` - -Once you have installed yarn, navigate to docs directory & set-up the dependencies. - -```sh -cd docs -yarn -``` - -Once this is done, generate the docs with: - -```sh -yarn build -``` - -You can also spin a local web server to serve the docs for you, with auto-refresh when you edit the source files, with: -```sh -yarn start -``` - -## Your first code pull request {#your-first-code-pull-request} - -This describes the overall steps to your first code contribution in OpenRefine. If you have trouble with any of these steps feel free to reach out on the [developer mailing list](https://groups.google.com/forum/#!forum/openrefine-dev) or the [Gitter channel](https://gitter.im/OpenRefine/OpenRefine). - -- Install OpenRefine, learn to use it by following some tutorials or watching [some videos](http://openrefine.org/). That will ensure you understand the user workflows and get familiar with the terminology used in the tool. - -- Fork the GitHub repository, clone it on your machine and set up your IDE to work on it. We have [instructions for this](https://github.com/OpenRefine/OpenRefine/wiki/Building-OpenRefine-From-Source). - -- Browse through the list of issues to find an issue that you find interesting. You should pick one where you understand what the problem is as a user, you can see why fixing it would be an improvement to the tool. It is also a good idea to pick an issue that matches your technical skills: some require work on the backend (in Java) or in the frontend (Javascript), often both. We try to maintain a list of [good first issues](https://github.com/OpenRefine/OpenRefine/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) which should be easier than others and should not require any difficult design decision. - -- Reproduce the issue locally, by following the steps described in the issue. You might need to locate a particular dialog, use a specific importer on a sample file, or follow any other user workflow. If you have followed all the steps described in the issue and cannot observe the issue mentioned, write a comment on the issue explaining that you are not able to reproduce it (perhaps it was fixed by another change). - -- Locate the code that is relevant for the issue you want to solve. Text search across files is often useful for that. For instance, if the issue you want to solve is about a dialog entitled "Columnize by key/values", you can search for "Columnize" in the entire source code. For more details about this technique, see [this comment](https://github.com/OpenRefine/OpenRefine/issues/3137#issuecomment-691649962). - -- Study how the current code works. You might want to use a debugger to put breakpoints at the relevant locations (for inspecting the backend, use your IDE's debugger, for the frontend, use your browser's developer tools). - -- Create a git branch for your fix. The name of your branch should contain the issue number, and a few words to describe the topic of the fix, for instance "issue-1234-columnize-layout". - -- Make changes to the code to fix the issue. If you are changing backend code, it would be great if you could also write a test in Java to demonstrate the fix. You can imitate existing tests for that. We currently do not have frontend tests. - -- If you made Java changes, run linting to make sure they conform to our code style, with `mvn formatter:format`. - -- commit your changes, using a message that contains one of the special words "closes" and "fixes" which are detected by Github, followed by the issue number, e.g. "closes #1234" or "fixes #1234", this will link the commit to the issue you are working on. - -- push your branch to your fork and create a pull request for it, explaining the approach you have used, any design decisions you have made. - -Thank you! diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/development-roadmap.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/development-roadmap.md deleted file mode 100644 index da35f2f77..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/development-roadmap.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -id: development-roadmap -title: Development roadmap -sidebar_label: Development roadmap ---- - -Please be aware that the OpenRefine roadmap is subject to change at any time, so please check back regularly, and monitor [milestones](https://github.com/OpenRefine/OpenRefine/milestones), [projects](https://github.com/OpenRefine/OpenRefine/projects) and [issues](https://github.com/OpenRefine/OpenRefine/issues) in Github to keep up to date with current plans. - -If there are features you would like to see that are not currently listed here or in current [milestones](https://github.com/OpenRefine/OpenRefine/milestones), [projects](https://github.com/OpenRefine/OpenRefine/projects) and [issues](https://github.com/OpenRefine/OpenRefine/issues), please add them to the [issue tracker](https://github.com/OpenRefine/OpenRefine/issues). - - -## Planned releases {#planned-releases} - -### 4.0 {#40} -[New backend storage option to allow using much bigger datasets at the expense of real-time feedback.](https://github.com/OpenRefine/OpenRefine/milestone/7) - -New UI (possibly Vue or React based) - -## Work in progress {#work-in-progress} -Alongside the planned releases there are often smaller pieces of work in progress. Check for [recently updated issues](https://github.com/OpenRefine/OpenRefine/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc) and [pull requests](https://github.com/OpenRefine/OpenRefine/pulls?q=is%3Apr+is%3Aopen+sort%3Aupdated-desc) to see what is currently in the works. - -## On the back burner {#on-the-back-burner} -Some aspects of OpenRefine have previously been targeted for release, but have not made it into a release and have not been worked on recently. If you would like to see features in these areas, please create an issue the describes what development you would like to see: - -- Streamlining traditional features -- Views: map, timeline, protovis (D3.js) charts -- Better machinery to guess and re-encode cell values (useful for fixing encoding issues) -- Collaborative editing support (see documentation on the '[broker protocol](https://github.com/OpenRefine/OpenRefine/wiki/Broker-Protocol)' to see where this work was going) -- Column groups diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/functional-tests.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/functional-tests.md deleted file mode 100644 index afdfbe43c..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/functional-tests.md +++ /dev/null @@ -1,241 +0,0 @@ ---- -id: functional-tests -title: Functional tests -sidebar_label: Functional tests ---- - -import useBaseUrl from '@docusaurus/useBaseUrl'; - -## Introduction {#introduction} - -OpenRefine interface is tested with the [Cypress framework](https://www.cypress.io/). -With Cypress, tests are performing assertions using a real browser, the same way a real user would use the software. - -Cypress tests can be ran - -- using the Cypress test runner (development mode) -- using a command line (CI/CD mode) - -If you are writing tests, the Cypress test runner is good enough, and the command-line is mainly used by the CI/CD platform (Github actions) - -## Cypress brief overview {#cypress-brief-overview} - -Cypress operates insides a browser, it's internally using NodeJS. -That's a key difference with tools such as Selenium. - -**From the Cypress documentation:** - -> But what this also means is that your test code **is being evaluated inside the browser**. Test code is not evaluated in Node, or any other server side language. The **only** language we will ever support is the language of the web: JavaScript. - -Good starting points with Cypress are the [Getting started guide](https://docs.cypress.io/guides/getting-started/writing-your-first-test.html#Write-your-first-test), and the [Trade-offs](https://docs.cypress.io/guides/references/trade-offs.html#Permanent-trade-offs-1) - -The general workflow of a Cypress test is to - -- Start a browser (yarn run cypress open) -- Visit a URL -- Trigger user actions -- Assert that the DOM contains expected texts and elements using selectors - -## Getting started {#getting-started} - -If this is the first time you use Cypress, it is recommended for you to get familiar with the tool. - -- [Cypress overview](https://docs.cypress.io/guides/overview/why-cypress.html) -- [Cypress examples of tests and syntax](https://example.cypress.io/) - -### 1. Install Cypress {#1-install-cypress} - -You will need: - -- [Node.js 10 or 12 and above](https://nodejs.org) -- [Yarn or NPM](https://yarnpkg.com/) -- A Unix/Linux shell environment or the Windows command line - -To install Cypress and dependencies, run : - -```shell -cd ./main/tests/cypress -yarn install -``` - -### 2. Start the test runner {#2-start-the-test-runner} - -The test runner assumes that OpenRefine is up and running on the local machine, the tests themselves do not launch OpenRefine, nor restarts it. - -Start OpenRefine with - -```shell -./refine -``` - -Then start Cypress - -```shell -yarn --cwd ./main/tests/cypress run cypress open -``` - -### 3. Run the existing tests {#3-run-the-existing-tests} - -Once the test runner is up, you can choose to run one or several tests by selecting them from the interface. -Click on one of them and the test will start. - -### 4. Add your first test {#4-add-your-first-test} - -- Add a `test.spec.js` into the `main/tests/cypress/cypress/integration` folder. -- The test is instantly available in the list -- Click on the test -- Start to add some code - -## Tests technical documentation {#tests-technical-documentation} - -### A typical test {#a-typical-test} - -A typical OpenRefine test starts with the following code - -```javascript -it('Ensure cells are blanked down', function () { - cy.loadAndVisitProject('food.mini'); - cy.get('.viewpanel-sorting a').contains('Sort').click(); - cy.get('.viewpanel').should('to.contain', 'Something'); -}); -``` - -The first noticeable thing about a test is the description (`Ensure cells are blanked down`), which describes what the test is doing. -Lines usually starts with `cy.something...`, which is the main way to interact with the Cypress framework. - -A few examples: - -- `cy.get('a.my-class')` will retrieve the `
` element -- `cy.click()` will click on the element -- eventually, `cy.should()` will perform an assertion, for example that the element contains an expected text with `cy.should('to.contains', 'my text')` - -On top of that, OpenRefine contributors have added some functions for common OpenRefine interactions. -For example - -- `cy.loadAndVisitProject` will create a fresh project in OpenRefine -- `cy.assertCellEquals` will ensure that a cell contains a given value - -See below on the dedicated section 'Testing utilities' - -### Testing guidelines {#testing-guidelines} - -- `cy.wait` should be used in the last resort scenario. It's considered a bad practice, though sometimes there is no other choice -- Tests should remain isolated from each other. It's best to try one feature at the time -- A test should always start with a fresh project -- The name of the files should mirror the OpenRefine UI organization - -### Testing utilities {#testing-utilities} - -OpenRefine contributors have added some utility methods on the top of the Cypress framework. -Those methods perform some common actions or assertions on OpenRefine, to avoid code duplication. - -Utilities can be found in `cypress/support/commands.js`. - -The most important utility method is `loadAndVisitProject`. -This method will create a fresh OpenRefine project based on a dataset given as a parameter. -The fixture parameter can be - -- An arbitrary array, the first row is for the column names, other rows are for the values - Use an arbitrary array **only** if the test requires some specific grid values - **Example:** - - ```javascript - const fixture = [ - ['Column A', 'Column B', 'Column C'], - ['0A', '0B', '0C'], - ['1A', '1B', '1C'], - ['2A', '2B', '2C'], - ]; - cy.loadAndVisitProject(fixture); - ``` - -- A referenced dataset: `food.small` or `food.mini` - Most of the time, tests does not require any specific grid values - Use food.mini as much as possible, it loads 2 rows and very few columns in the grid - Use food.small if the test requires a few hundred rows in the grid - - Those datasets live in `cypress/fixtures` - -### Browsers {#browsers} - -In terms of browsers, Cypress is using what is installed on your operating system. -See the [Cypress documentation](https://docs.cypress.io/guides/guides/launching-browsers.html#Browsers) for a list of supported browsers - -### Folder organization {#folder-organization} - -Tests are located in `main/tests/cypress/cypress` folder. -The test should not use any file outside the cypress folder. - -- `/fixtures` contains CSVs and OpenRefine project files used by the tests -- `/integration` contains the tests -- `/plugins` contains custom plugins for the OR project -- `/screenshots` and `/videos` contains the recording of the tests, Git ignored -- `/support` is a custom library of assertion and common user actions, to avoid code duplication in the tests themselves - -### Configuration {#configuration} - -Cypress execution can be configured with environment variables, they can be declared at the OS level, or when running the test - -Available variables are - -- OPENREFINE_URL, determine on which scheme://url:port to access OpenRefine, default to http://localhost:333 -- DISABLE_PROJECT_CLEANUP, If set to 1, projects will not be deleted after each run. Default to 0 to keep the OpenRefine instance clean - - -Cypress contains [exaustive documentation](https://docs.cypress.io/guides/guides/environment-variables.html#Setting) about configuration, but here are two simple ways to configure the execution of the tests: - -#### Overriding with a cypress.env.json file {#overriding-with-a-cypressenvjson-file} - -This file is ignored by Git, and you can use it to configure Cypress locally - -#### Command-line {#command-line} - -You can pass variables at the command-line level - -```shell -yarn --cwd ./main/tests/cypress run cypress open --env OPENREFINE_URL="http://localhost:1234" -``` - -### Visual testing {#visual-testing} - -Tests generally ensure application behavior by making assertions against the DOM, to ensure specific texts or css attributes are present in the document body. -Visual testing, on the contrary, is a way to test applications by comparing images. -A reference screenshot is taken the first time the test runs, and subsequent runs will compare a new screenshot against the reference, at the pixel level. - -Here is an [introduction to visual testing by Cypress](https://docs.cypress.io/plugins/directory#visual-testing). - -In some cases, we are using visual testing. -We are using [Cypress Image Snapshot](https://github.com/jaredpalmer/cypress-image-snapshot) - -Identified cases are so far: - -- testing images created by OpenRefine backend (scatterplots for example) - -Reference screenshots (Called snapshots), are stored in /cypress/snapshots. -And a snapshot can be taken for the whole page, or just a single part of the page. - -#### When a visual test fails {#when-a-visual-test-fails} - -First, Cypress will display the following error message: - -![Diff image when a visual test fails](/img/visual-test-cypress-failure.png) - -Then, a diff image will be created in /cypress/snapshots, this directory is ignored by Git. -The diff images shows the reference image on the left, the image that was taken during the test run on the right, and the diff in the middle. - -![Diff image when a visual test fails](/img/failed-visual-test.png) - -## CI/CD {#cicd} - -In CI/CD, tests are run headless, with the following command-line - -```shell -./refine ui_test chrome -``` - -Results are displayed in the standard output - -## Resources {#resources} - -[Cypress command line options](https://docs.cypress.io/guides/guides/command-line.html#Installation) -[Lots of good Cypress examples](https://example.cypress.io/) diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/homebrew-cask-process.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/homebrew-cask-process.md deleted file mode 100644 index 650af5ccf..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/homebrew-cask-process.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -id: homebrew-cask-process -title: Maintaining OpenRefine's Homebrew cask -sidebar_label: Maintaining OpenRefine's Homebrew cask ---- diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/maintainer-guidelines.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/maintainer-guidelines.md deleted file mode 100644 index b4ca7b20f..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/maintainer-guidelines.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -id: maintainer-guidelines -title: Guidelines for maintaining OpenRefine -sidebar_label: Maintainer guidelines ---- - -This page describes our practices to review issues and pull requests in the OpenRefine project. - -## Reviewing issues {#reviewing-issues} - -When people create new issues, they automatically get assigned [the "to be reviewed" tag](https://github.com/OpenRefine/OpenRefine/issues?q=is%3Aissue+is%3Aopen+label%3A%22to+be+reviewed%22). - -Ideally, for each of these issues, someone familiar with OpenRefine (not necessarily a developer!) should read the issue and try to determine if there is a genuine bug to fix, or if the enhancement request is legitimate. In those cases, we can remove the "to be reviewed" tag and leave the issue open. In the others, the issue should be politely closed. - -### Bugs {#bugs} - -For a bug, we should first check if it is a real unexpected behaviour or if just comes from a misunderstanding of the intended behaviour of the tool (which could suggest an improvement to the documentation). Then, if it sounds like a genuine problem, we need to check if it can be reproduced independently on the master branch. If the issue does not give enough details about the bug to reproduce it on master, mark it as "not reproducible" and ask the reporter for more information. After some time without any information from the reporter, we can close the issue. - -### Enhancement requests {#enhancement-requests} - -For an enhancement, we need to make a judgment call of whether the proposed functionality is in the scope of the project. There is no universal rule for this of course, so just use your own intuition: do you think this would improve the tool? Would it be consistent with the spirit of the project? Trust your own opinion - if people disagree, they can have a discussion on the issue. - -### Tagging good first issues {#tagging-good-first-issues} - -Adding [the "good first issue" tag](https://github.com/OpenRefine/OpenRefine/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) is something that requires a bit more familiarity with the development process. This tag is used by GitHub to showcase issues in some project lists and we point interested potential contributors to it. It is therefore important that tackling these issues gives them a nice onboarding experience, with as few hurdles as possible. - -Develepers should add the "good first issue" tag when they are confident that they can provide a good pull request for the issue with at most a few hours of work. Also, solving the issue should not require any difficult design decision. The issue should be uncontentious: it should be clear that the proposed solution should be accepted by the team. - -## Reviewing pull requests {#reviewing-pull-requests} - -### Process {#process} - -1. A committer reviews the PR to check for the requirements below and tests it. Each PR should be linked to one or more corresponding issues and the reviewer should check that those are correctly addressed by the PR. The reviewer should be someone else than the PR author. For PRs with an important impact or contentious issues, it is important to leave enough time for other contributors to give their opinion. - -2. The reviewer merges the pull request by squashing its commits into one (except for Weblate PRs which should be merged without squashing). - -3. The reviewer adds the linked issues to the milestone for the next release (such as [the 3.5 milestone](https://github.com/OpenRefine/OpenRefine/milestone/17)) - -4. If the change is worth noting for users or developers, the reviewer adds an entry in the changelog for the next release (such as [Changes for 3.5](https://github.com/OpenRefine/OpenRefine/wiki/Changes-for-3.5)) - -### Requirements {#requirements} - -#### Code style {#code-style} - -Currently, only our code style for integration tests (using Cypress) is codified and enforced by the CI. -For the rest, we rely on imitating the surrounding code. [We should decide on a code style and check it in the CI for other areas of the tool](https://github.com/OpenRefine/OpenRefine/issues/2338). - -#### Testing {#testing} - -We currently rely have two sorts of tests: -* Backend tests, in Java, written with the TestNG framework. Their granularity varies, but generally speaking they are unit tests which test components in isolation. -* UI tests, in Javascript, written with the Cypress framework. They are integration tests which test both the frontend and the backend at the same time. - -Changes to the backend should generally come with the accompanying TestNG tests. -Functional changes to the UI should ideally come with corresponding Cypress tests as well. - -Those tests should be supplied in the same PR as the one that touches the product code. - -#### Documentation {#documentation} - -Changes to user-facing functionality should be reflected in the docs. Those documentation changes should happen in the same PR as the one that touches the product code. - -#### UI style {#ui-style} - -We do not have formally defined UI style guidelines. Contributors are invited to imitate the existing style. - -#### Licensing and dependencies {#licensing-and-dependencies} - -Dependencies can only be added if they are released under a license that is compatible with our BSD Clause-3 license. -One should pay attention to the size of the dependencies since they inflate the size of the release bundles. - -#### Continuous integration {#continuous-integration} - -The various check statuses reported by our continuous integration suite should be green. - -### Special pull requests {#special-pull-requests} - -#### Weblate PRs {#weblate-prs} - -Weblate PRs should not be squashed as it prevents Weblate from recognizing that the corresponding changes have been made in master. They should be merged without squashing. - -Reviewing Weblate PRs only amonuts to a quick visual sanity check as maintainers are not expected to master the languages involved. If corrections need to be made, they should be done in Weblate itself. - -#### Dependabot PRs {#dependabot-prs} - -When reviewing a Dependabot PR it is generally useful to pay attention to: -* the type of version change: most libraries follow the "semver" versioning convention, which indicates the nature of the change. -* the library's changelog, especially if the version change is more significant than a patch release - diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/migrating-older-extensions.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/migrating-older-extensions.md deleted file mode 100644 index 75f40015e..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/migrating-older-extensions.md +++ /dev/null @@ -1,169 +0,0 @@ ---- -id: migrating-older-extensions -title: Migrating older Extensions -sidebar_label: Migrating older Extensions ---- - -## Migrating from Ant to Maven {#migrating-from-ant-to-maven} - -### Why are we doing this change? {#why-are-we-doing-this-change} - -Ant is a fairly old (antique?) build system that does not incorporate any dependency management. -By migrating to Maven we are making it easier for developers to extend OpenRefine with new libraries, and stop having to ship dozens of .jar files in the repository. Using the Maven repository also encourages developers to add dependencies to released versions of libraries instead of custom snapshots that are hard to update. - -### When was this change made? {#when-was-this-change-made} - -The migration was done between 3.0 and 3.1-beta with this commit: -https://github.com/OpenRefine/OpenRefine/commit/47323a9e750a3bc9d43af606006b5eb20ca397b8 - -### How to migrate an extension {#how-to-migrate-an-extension} - -You will need to write a `pom.xml` in the root folder of your extension to configure the compilation process with Maven. Sample `pom.xml` files for extensions can be found in the extensions that are shipped with OpenRefine (`gdata`, `database`, `jython`, `pc-axis` and `wikidata`). A sample extension (`sample`) is also provided, with a minimal build file. - -For any library that your extension depends on, you should try to find a matching artifact in the Maven Central repository. If you can find such an artifact, delete the `.jar` file from your extension and add the dependency in your `pom.xml` file. If you cannot find such an artifact, it is still possible to incorporate your own `.jar` file using `maven-install-plugin` that you can configure in your `pom.xml` file as follows: - - - - org.apache.maven.plugins - maven-install-plugin - 2.5.2 - - - install-wdtk-datamodel - process-resources - - ${basedir}/lib/my-proprietary-library.jar - default - com.my.company - my-library - 0.5.3-SNAPSHOT - jar - true - - - install-file - - - - - - -And add the dependency to the `` section as usual: - - - com.my.company - my-library - 0.5.3-SNAPSHOT - - -## Migrating to Wikimedia's i18n jQuery plugin {#migrating-to-wikimedias-i18n-jquery-plugin} - -### Why are we doing this change? {#why-are-we-doing-this-change-1} - -This adds various important localization features, such as the ability to handle plurals or interpolation. This also restores the language fallback (displaying strings in English if they are not available in the target language) which did not work with the previous set up. - -### When was the migration made? {#when-was-the-migration-made} - -The migration was made between 3.1-beta and 3.1, with this commit: https://github.com/OpenRefine/OpenRefine/commit/22322bd0272e99869ab8381b1f28696cc7a26721 - -### How to migrate an extension {#how-to-migrate-an-extension-1} - -You will need to update your translation files, merging nested objets in one global object, concatenating keys. You can do this by running the following Python script on all your JSON translation files: - - import json - import sys - - with open(sys.argv[1], 'r') as f: - j = json.loads(f.read()) - - result = {} - def translate(obj, path): - res = {} - if type(obj) == str: - result['/'.join(path)] = obj - else: - for k, v in obj.items(): - new_path = path + [k] - translate(v, new_path) - - translate(j, []) - - with open(sys.argv[1], 'w') as f: - f.write(json.dumps(result, ensure_ascii=False, indent=4)) - -Then your javascript files which retrieve the translated strings should be updated: `$.i18n._('core-dialogs')['cancel']` becomes `$.i18n('core-dialogs/cancel')`. You can do this with the following `sed` script: - - sed -i "s/\$\.i18n._(['\"]\([A-Za-z0-9/_\\-]*\)['\"])\[['\"]\([A-Za-z0-9\-\_]*\)[\"']\]/$.i18n('\1\/\2')/g" my_javascript_file.js - -You can then chase down the places where you are concatenating translated strings, and replace that with more flexible patterns using [the plugin's features](https://github.com/wikimedia/jquery.i18n#jqueryi18n-plugin). - -## Migrating from org.json to Jackson {#migrating-from-orgjson-to-jackson} - -### Why are we doing this change? {#why-are-we-doing-this-change-2} - -The org.json (or json-java) library has multiple drawbacks. -* First, it has limited functionality - all the serialization and deserialization has to be done explicitly - an important proportion of OpenRefine's code was dedicated to implementing these; -* Second, its implementation is not optimized for speed - multiple projects have reported speedups when migrating to more modern JSON libraries; -* Third, and this was the decisive factor to initiate the migration: [its license](https://json.org/license) is the MIT license with an additional condition which makes it non-free. Getting rid of this dependency was required by the Software Freedom Conservancy as a prerequisite to become a fiscal sponsor for the project. - -### When was the migration made? {#when-was-the-migration-made-1} - -This change was made between 3.1 and 3.2-beta, with this commit: https://github.com/OpenRefine/OpenRefine/commit/5639f1b2f17303b03026629d763dcb6fef98550b - -### How to migrate an extension or fork {#how-to-migrate-an-extension-or-fork} - -You will need to use the Jackson library to serialize the classes that implement interfaces or extend classes exposed by OpenRefine. -The interface `Jsonizable` was deleted. Any class that used to implement this now needs to be serializable by Jackson, producing the same format as the previous serialization code. This applies to any operation, facet, overlay model or GREL function. If you are new to Jackson, have a look at [this tutorial](https://www.baeldung.com/jackson) to learn how to annotate your class for serialization. Once this is done, you can remove the `void write(JSONWriter writer, Properties options)` method from your class. Note that it is important that you do this migration for all classes implementing the `Jsonizable` interface that are exposed to OpenRefine's core. - -We encourage you to migrate out of org.json completely, but this is only required for the classes that interact with OpenRefine's core. - -#### General notes about migrating {#general-notes-about-migrating} - -OpenRefine's ObjectMapper is available at `ParsingUtilities.mapper`. It is configured to only serialize the fields and getters that have been explicitly marked with `@JsonProperty` (to avoid accidental JSON format changes due to refactoring). On deserialization it will ignore any field in the JSON payload that does not correspond to a field in the Java class. It has serializers and deserializers for `OffsetDateTime` and `LocalDateTime`. - -Useful snippets to use in tests: -* deserialize an instance: `MyClass instance = ParsingUtilities.mapper.readValue(jsonString, MyClass.class);` (replaces calls to `Jsonizable.write`); -* serialize an instance: `String json = ParsingUtilities.mapper.writeValueAsString(myInstance);` (replaces calls to static methods such as `load`, `loadStreaming` or `reconstruct`); -* the equivalent of `JSONObject` is `ObjectNode`, the equivalent of `JSONArray` is `ArrayNode`; -* create an empty JSON object: `ParsingUtilities.mapper.createObjectNode()` (replaces `new JSONObject()`); -* create an empty JSON array: `ParsingUtilities.mapper.createArrayNode()` (replaces `new JSONArray()`). - -Before undertaking the migration, we recommend that you write some tests which serialize and deserialize your objects. This will help you make sure that the JSON format is preserved during the migration. One way to do this is to collect some sample JSON representations of your objects, and check in your tests that deserializing these JSON payloads and serializing them back to JSON preserves the JSON payload. Some utilities are available to help you with that in [`TestUtils`](https://github.com/OpenRefine/OpenRefine/blob/master/main/tests/server/src/com/google/refine/tests/util/TestUtils.java) (we had [some to test org.json serialization](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/tests/server/src/com/google/refine/tests/util/TestUtils.java) before we got rid of the dependency, feel free to copy them). - -#### For functions {#for-functions} - -Before the migration, you had to explicitly define JSON serialization of functions with a `write` method. You should now override the getters returning the various documentation fields. - -Example: `Cos` function [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/expr/functions/math/Cos.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/expr/functions/math/Cos.java). - -#### For operations {#for-operations} - -Before the JSON migration we refactored engine-dependent operations so that the engine configuration is represented by an `EngineConfig` object instead of a `JSONObject`. Therefore the constructor for your operation should be updated to use this new class. Your constructor should also be annotated to be used during deserialization. - -Note that you do not need to explicitly serialize the operation type, this is already done for you by `AbstractOperation`. - -Example: `ColumnRemovalOperation` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java). - -#### For changes {#for-changes} - -Changes are serialized in plain text but often relies on JSON serialization for parts of the data. Just use the methods above with `ParsingUtilities.mapper` to maintain this behaviour. - -Example: `ReconChange` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/model/changes/ReconChange.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java). - -#### For importers {#for-importers} - -The importing options have been migrated from `JSONObject` to `ObjectNode`. Your compiler should help you propagate this change. Utility functions in `JSONUtilities` have been migrated to Jackson so you should have minimal changes if you used them. - -Example: `TabularImportingParserBase` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/importers/TabularImportingParserBase.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/importers/TabularImportingParserBase.java). - -#### For overlay models {#for-overlay-models} - -Migrate serialization and deserialization as for other objects. - -Example: `WikibaseSchema` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java#L203) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java#L60) - -#### For preference values {#for-preference-values} - -Any class that is stored in OpenRefine's preference now needs to implement the `com.google.refine.preferences.PreferenceValue` interface. The static `load` method and the `write` method used previously for deserialization should be deleted and regular Jackson serialization and deserialization should be implemented instead. Note that you do not need to explicitly serialize the class name, this is already done for you by the interface. - -Example: `TopList` [before](https://github.com/OpenRefine/OpenRefine/blob/3.1/main/src/com/google/refine/preference/TopList.java) and [after](https://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/preference/TopList.java) diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/openrefine-api.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/openrefine-api.md deleted file mode 100644 index 7d1a4c402..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/openrefine-api.md +++ /dev/null @@ -1,285 +0,0 @@ ---- -id: openrefine-api -title: OpenRefine API -sidebar_label: OpenRefine API ---- - -This is a generic API reference for interacting with OpenRefine's HTTP API. - -**NOTE:** This protocol is subject to change without warning at any time (and has in the past) and is not versioned. Use at your own risk! - -For OpenRefine 3.3 and later, all POST requests need to include a CSRF token as described here: https://github.com/OpenRefine/OpenRefine/wiki/Changes-for-3.3#csrf-protection-changes - -## Create project: {#create-project} - -> **Command:** _POST /command/core/create-project-from-upload_ - -When uploading files you will need to send the data as `multipart/form-data`. This is different to all other API calls which use a mixture of query string and POST parameters. - -multipart form-data: - - 'project-file' : file contents - 'project-name' : project name - 'format' : format of data in project-file (e.g. 'text/line-based/*sv') [optional] - 'options' : json object containing options relevant to the file format [optional - however, some importers may have required options, such as `recordPath` for the JSON & XML importers]. - -The formats supported will depend on the version of OpenRefine you are using and any Extensions you have installed. The common formats include: - -* 'text/line-based': Line-based text files -* 'text/line-based/*sv': CSV / TSV / separator-based files [separator to be used in specified in the json submitted to the options parameter] -* 'text/line-based/fixed-width': Fixed-width field text files -* 'binary/text/xml/xls/xlsx': Excel files -* 'text/json': JSON files -* 'text/xml': XML files - -If the format is omitted OpenRefine will try to guess the format based on the file extension and MIME type. -The values which can be specified in the JSON object submitted to the 'options' parameter will vary depending on the format being imported. If not specified the options will either be guessed at by OpenRefine (e.g. separator being used in a separated values file) or a default value used. The import options for each file format are not currently documented, but can be seen in the OpenRefine GUI interface when importing a file of the relevant format. - -If the project creation is successful, you will be redirected to a URL of the form: - - http://127.0.0.1:3333/project?project= - -From the project parameter you can extract the project id for use in future API calls. The content of the response is the HTML for the OpenRefine interface for viewing the project. - -### Get project models: {#get-project-models} - -> **Command:** _GET /command/core/get-models?_ - - 'project' : project id - -Recovers the models for the specific project. This includes columns, records, overlay models, scripting. In the columnModel a list of the columns is displayed, key index and name, and column groupings. - -### Response: {#response} -**On success:** -```JSON -{ - "columnModel":{ - "columns":[ - { - "cellIndex":0, - "originalName":"email", - "name":"email" - }, - { - "cellIndex":1, - "originalName":"name", - "name":"name" - }, - { - "cellIndex":2, - "originalName":"state", - "name":"state" - }, - { - "cellIndex":3, - "originalName":"gender", - "name":"gender" - }, - { - "cellIndex":4, - "originalName":"purchase", - "name":"purchase" - } - ], - "keyCellIndex":0, - "keyColumnName":"email", - "columnGroups":[ - - ] - }, - "recordModel":{ - "hasRecords":false - }, - "overlayModels":{ - - }, - "scripting":{ - "grel":{ - "name":"General Refine Expression Language (GREL)", - "defaultExpression":"value" - }, - "jython":{ - "name":"Python / Jython", - "defaultExpression":"return value" - }, - "clojure":{ - "name":"Clojure", - "defaultExpression":"value" - } - } -} -``` - - - -## Apply operations {#apply-operations} - -> **Command:** _POST /command/core/apply-operations?_ - -In the parameter - - 'project' : project id - -In the form data - - 'operations' : Valid JSON **Array** of OpenRefine operations - -Example of a Valid JSON **Array** -```JSON -'[ - { - "op":"core/column-addition", - "description":"Create column zip type at index 15 based on column Zip Code 2 using expression grel:value.type()", - "engineConfig":{ - "mode":"row-based", - "facets":[] - }, - "newColumnName":"zip type", - "columnInsertIndex":15, - "baseColumnName":"Zip Code 2", - "expression":"grel:value.type()", - "onError":"set-to-blank" - }, - { - "op":"core/column-addition", - "description":"Create column testing at index 15 based on column Zip Code 2 using expression grel:value.toString()0,5]", - "engineConfig":{ - "mode":"row-based", - "facets":[] - }, - "newColumnName":"testing", - "columnInsertIndex":15, - "baseColumnName":"Zip Code 2", - "expression":"grel:value.toString()[0,5]", - "onError":"set-to-blank" - } -] -``` - -On success returns JSON response -`{ "code" : "ok" }` - -## Export rows {#export-rows} - -> **Command:** _POST /command/core/export-rows_ - -In the parameter - - 'project' : project id - 'format' : format... (e.g 'tsv', 'csv') - -In the form data - - 'engine' : JSON string... (e.g. '{"facets":[],"mode":"row-based"}') - -Returns exported row data in the specified format. The formats supported will depend on the version of OpenRefine you are using and any Extensions you have installed. The common formats include: - -* csv -* tsv -* xls -* xlsx -* ods -* html - -## Delete project {#delete-project} - -> **Command:** _POST /command/core/delete-project_ - - 'project' : project id... - -Returns JSON response - -## Check status of async processes {#check-status-of-async-processes} - -> **Command:** _GET /command/core/get-processes_ - - 'project' : project id... - -Returns JSON response - -## Get all projects metadata: {#get-all-projects-metadata} - -> **Command:** _GET /command/core/get-all-project-metadata_ - -Recovers the meta data for all projects. This includes the project's id, name, time of creation and last time of modification. - -### Response: {#response-1} -```json -{ - "projects":{ - "[project_id]":{ - "name":"[project_name]", - "created":"[project_creation_time]", - "modified":"[project_modification_time]" - }, - ...[More projects]... - } -} -``` - -## Expression Preview {#expression-preview} -> **Command:** _POST /command/core/preview-expression_ - -Pass some expression (GREL or otherwise) to the server where it will be executed on selected columns and the result returned. - -### Parameters: {#parameters} -* **cellIndex**: _[column]_ -The cell/column you wish to execute the expression on. -* **rowIndices**: _[rows]_ -The rows to execute the expression on as JSON array. Example: `[0,1]` -* **expression**: _[language]_:_[expression]_ -The expression to execute. The language can either be grel, jython or clojure. Example: grel:value.toLowercase() -* **project**: _[project_id]_ -The project id to execute the expression on. -* **repeat**: _[repeat]_ -A boolean value (true/false) indicating whether or not this command should be repeated multiple times. A repeated command will be executed until the result of the current iteration equals the result of the previous iteration. -* **repeatCount**: _[repeatCount]_ -The maximum amount of times a command will be repeated. - -### Response: {#response-2} -**On success:** -```json -{ - "code": "ok", - "results" : [result_array] -} -``` - -The result array will hold up to ten results, depending on how many rows there are in the project that was specified by the [project_id] parameter. Each result is the string that would be put in the cell if the GREL command was executed on that cell. Note that any expression that would return an array or JSon object will be jsonized, although the output can differ slightly from the jsonize() function. - -**On error:** -```JSON -{ - "code": "error", - "type": "[error_type]", - "message": "[error message]" -} -``` - -## Third-party software libraries {#third-party-software-libraries} -Libraries using the [OpenRefine API](openrefine-api): - -### Python {#python} -* [refine-client-py](https://github.com/PaulMakepeace/refine-client-py/) - * Or this fork of the above with an extended CLI [openrefine-client](https://github.com/felixlohmeier/openrefine-client) -* [refine-python](https://github.com/maxogden/refine-python) - -### Ruby {#ruby} -* [refine-ruby](https://github.com/distillytics/refine-ruby) - * The above is a maintained fork of [google-refine](https://github.com/maxogden/refine-ruby) -* [google_refine](https://github.com/chengguangnan/google_refine) - -### NodeJS {#nodejs} -* [node-openrefine](https://github.com/pm5/node-openrefine) - -### R {#r} -* [rrefine](https://cran.r-project.org/web/packages/rrefine/index.html) - -### PHP {#php} -* [openrefine-php-client](https://github.com/keboola/openrefine-php-client) - -### Java {#java} -* [refine-java](https://github.com/dtap-gmbh/refine-java) - -### Bash {#bash} -* [bash-refine.sh](https://gist.github.com/felixlohmeier/d76bd27fbc4b8ab6d683822cdf61f81d) (templates for shell scripts) diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/reconciliation-api.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/reconciliation-api.md deleted file mode 100644 index 7649a6ebb..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/reconciliation-api.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -id: reconciliation-api -title: Reconciliation API -sidebar_label: Reconciliation API ---- - -_This is a technical description of the mechanisms behind the reconciliation system in OpenRefine. For usage instructions, see [Reconciliation](/manual/reconciling)._ - -A reconciliation service is a web service that, given some text which is a name or label for something, and optionally some additional details, returns a ranked list of potential entities matching the criteria. The candidate text does not have to match each entity's official name perfectly, and that's the whole point of reconciliation--to get from ambiguous text name to precisely identified entities. For instance, given the text "apple", a reconciliation service probably should return the fruit apple, the Apple Inc. company, and New York city (also known as the Big Apple). - -Entities are identified by strong identifiers in some particular identifier space. In the same identifier space, identifiers follow the same syntax. For example, given the string "apple", a reconciliation service might return entities identified by the strings " [Q89](https://www.wikidata.org/wiki/Q89)", "[Q312](https://www.wikidata.org/wiki/Q312)", and "[Q60](https://www.wikidata.org/wiki/Q60)", in the Wikidata ID space. Each reconciliation service can only reconcile to one single identifier space, but several reconciliation services can reconcile to the same identifier space. - -OpenRefine can connect to any reconciliation service which follows the [reconciliation API v0.1](https://reconciliation-api.github.io/specs/0.1/). This was formerly a specification edited by the OpenRefine project, which has now transitioned to its own -[W3C Entity Reconciliation Community Group](https://www.w3.org/community/reconciliation/). - -Informally, the main function of any reconciliation service is to find good candidates in the underlying database, given the following data: - -* A string, which is normally the name or title of the entity, in some language. -* Optionally, a type which can be used to narrow down the search to entities of this type. OpenRefine does not define a particular set of acceptable types: this choice is left to the reconciliation service (see the suggest API for that). -* Optionally, a list of properties and their values, which can be used to refine the search. For instance, when reconciling a database of books, the author name or the publication date are useful bits of information that can be transferred to the reconciliation service. This information will be sent to the reconciliation service if the user binds columns to properties. Again, the notion of property is not predefined in OpenRefine: its definition depends on the reconciliation service. - -See [the specifications of the protocol](https://reconciliation-api.github.io/specs/0.1) for more details about the protocol. You can suggest changes on its [issues tracker](https://github.com/reconciliation-api/specs/issues) or on the [group mailing -list](https://lists.w3.org/Archives/Public/public-reconciliation/). - diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/technical-reference-index.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/technical-reference-index.md deleted file mode 100644 index 170b60958..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/technical-reference-index.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -id: technical-reference-index -title: OpenRefine technical reference -sidebar_label: Technical Reference Index ---- - -Technical reference index diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/translating-docs.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/translating-docs.md deleted file mode 100644 index eb8482e5f..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/translating-docs.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -id: translating-docs -title: Translate OpenRefine's documentation -sidebar_label: Translate OpenRefine's documentation ---- - -Our user manual can be translated using [Crowdin](https://crowdin.com/project/openrefine). - -## Getting access to Crowdin - -You need to request access to Crowdin on [our developers mailing list](https://groups.google.com/forum/#!forum/openrefine-dev). You will then be granted translator access on the platform. - -## Publication of translations - -Only the user manual for the current development version is being translated. When we publish a new stable version, we take a snapshot of this and publish it on the website. - -In the meantime the pages you translate will be visible under https://docs.openrefine.org/next/ after a few days. - - diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/translating-ui.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/translating-ui.md deleted file mode 100644 index 2ca3a811e..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/translating-ui.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -id: translating-ui -title: Translate OpenRefine's interface -sidebar_label: Translate OpenRefine's interface ---- - -Currently supported languages include English, Spanish, Chinese, French, Hebrew, Italian and Japanese. - -![Translation status](https://hosted.weblate.org/widgets/openrefine/-/287x66-grey.png) - -You can help translate OpenRefine into your language by visiting [Weblate](https://hosted.weblate.org/engage/openrefine/?utm_source=widget) which provides a web based UI to edit and add translations and sends automatic pull requests back to our project. - -Click to help translate --> [Weblate](https://hosted.weblate.org/engage/openrefine/?utm_source=widget) - -## Manual translation process {#manual-translation-process} - -Localized strings are entered in a .json file, one per language. They are located in the folder `main/webapp/modules/core/langs/` in a file named `translation-xx`.json, where xx is the language code (i.e. fr for French). - -### Simple case of localized string {#simple-case-of-localized-string} -This is an example of a simple string, with the start of the JSON file. This example is for French. -``` -{ - "name": "Français", - "core-index/help": "Aide", - (… more lines) -} -``` - -So the key `core-index/help` will render as `"Aide"` in French. - -### Localization with a parameterized value {#localization-with-a-parameterized-value} -In this example, the name of the column (represented by `$1` in this example), will be substituted with the string of the name of the column. - -`"core-facets/edit-facet-title": "Cliquez ici pour éditer le nom de la facette\nColonne : $1",` - -### Localization with a singular/plural value {#localization-with-a-singularplural-value} -In this example, one of the parameter will have a different string depending if the value is 1 or another value. -In this example, the string for page, the second parameter, `$2`, will have an « s » or not depending on the value of `$2`. - -`"core-views/goto-page": "$1 de $2 {{plural:$2|page|pages}}"` - -## Front-end development {#front-end-development} - -The OpenRefine front end has been localized using the [Wikidata jquery.i18n library](https://github.com/OpenRefine/OpenRefine/pull/1285. The localized text is stored in a JSON dictionary on the server and retrieved with a new OpenRefine command. - -### Adding a new string {#adding-a-new-string} - -There should be no hard-coded language strings in the HTML or JSON used for the front end. If you need a new string, first check the existing strings to make sure there isn't an equivalent string, **in an equivalent context**, that you can reuse. Context is important because it can affect how the same literal English text is translated. This cuts down on the amount of text which needs to be translated. - -Strings should be entire sentences or phrases and should include substitution variables for any parameters. Do not concatenate strings in either Java or Javascript (or implicitly by laying them out in a specific order). So, instead of `"You have " + count + " row(s)"` (or worse `count != 1 ? " rows" : " row"`), internationalize everything together so that it can be translated taking into account word ordering and plurals for different languages, ie `"You have $1 {{plural $1: row|rows}}"`, passing the parameter(s) into the `$.i18n` call. - -If there's no string you can reuse, allocate an available key in the appropriate translation dictionary and add the default string, e.g. - -```json -..., -"section/newkey": "new default string for this key", -... -``` - -and then set the text (or HTML) of your HTML element using i18n helper method. So given an HTML fragment like: -```html - -``` -we could set its text using: -``` -$('#new-html-element-id').text($.i18n('section/newkey'])); -``` -or, if you need to embed HTML tags: -``` -$('#new-html-element-id').html($.i18n('section/newkey']); -``` - -### Adding a new language {#adding-a-new-language} - -The language dictionaries are stored in the `langs` subdirectory for the module e.g. - -* https://github.com/OpenRefine/OpenRefine/tree/master/main/webapp/modules/core/langs for the main interface -* https://github.com/OpenRefine/OpenRefine/tree/master/extensions/gdata/module/langs for google spreadsheet connection -* https://github.com/OpenRefine/OpenRefine/tree/master/extensions/database/module/langs for database via JDBC -* https://github.com/OpenRefine/OpenRefine/tree/master/extensions/wikidata/module/langs for Wikidata - -To add support for a new language, the easiest way is to do it directly in Weblate. To do it manually, copy `translation-en.json` to `translation-.json` and have your translator translate all the value strings (ie right hand side). - -#### Main interface {#main-interface} - The translation is best done [with Weblate](https://hosted.weblate.org/engage/openrefine/?utm_source=widget). Files are periodically merged by the developer team. - -Run the latest (hopefully cloned from github) version and check whether translated words fit to the layout. Not all items can be translated word by word, especially into non-Ìndo-European languages. - -If you see any text which remains in English even when you have checked all items, please create bug report in the issue tracker so that the developers can fix it. - -#### Extensions {#extensions} - -Extensions can be translated via Weblate just like the core software. - -The new extension for Wikidata contains lots of domain-specific concepts, with which you may not be familiar. The Wikidata may not have reconciliation service for your language. I recommend checking the glossary(https://www.wikidata.org/wiki/Wikidata:Glossary) to be consistent. - -By default, the system tries to load the language file corresponding to the currently in-use browser language. To override this setting a new menu item ("Language Settings") has been added at the index page. -To support a new language file, the developer should add a corresponding entry to the dropdown menu in this file: `/OpenRefine/main/webapp/modules/core/scripts/index/lang-settings-ui.html`. The entry should look like: -```javascript - -``` - -## Server-side localisation {#server--backend-coding} - -Currently no back end functions are translated, so things like error messages, undo history, etc may appear in English form. Rather than sending raw error text to the front end, it's better to send an error code which is translated into text on the front end. This allows for multiple languages to be supported. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/version-release-process.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/version-release-process.md deleted file mode 100644 index 0cf0b88df..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/version-release-process.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -id: version-release-process -title: How to do an OpenRefine version release -sidebar_label: How to do an OpenRefine version release ---- - -When releasing a new version of Refine, the following steps should be followed: - -1. Make sure the `master` branch is stable and nothing has broken since the previous version. We need developers to stabilize the trunk and some volunteers to try out `master` for a few days. -2. Change the version number in [RefineServlet.java](http://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/RefineServlet.java#L62) and in the POM files using `mvn versions:set -DnewVersion=2.6-beta -DgenerateBackupPoms=false`. Commit the changes. -3. Compose the list of changes in the code and on the wiki. If the issues have been updated with the appropriate milestone, the Github issue tracker should be able to provide a good starting point for this. -4. Set up build machine. This needs to be Mac OS X or Linux. -5. Download Windows and Mac JREs to bundle them in the Windows and Mac packages from [AdoptOpenJDK](https://adoptopenjdk.net/). You only need the JREs, not the JDKs. Use the lowest version of Java supported (Java 8 currently). Configure the location of these JREs in the `settings.xml` file at the root of the repository. It is important to download recent versions of the JREs as this impacts which HTTPS certificates are accepted by the tool. -6. Insert the production Google credentials in https://github.com/OpenRefine/OpenRefine/blob/bc540a880eceb88e54f85ca43eb54769de3bfa4f/extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java#L36-L39 without committing the changes. -7. [Build the release candidate kits using the shell script (not just Maven)](https://github.com/OpenRefine/OpenRefine/wiki/Building-OpenRefine-From-Source). This must be done on Mac OS X or Linux to be able to build all 3 kits. On Linux you will need to install the `genisoimage` program first. -```shell -./refine dist 2.6-beta.2 -``` -To build the Windows version with embedded JRE, use `mvn package -s settings.xml -P embedded-jre -DskipTests=true`. - -8. On a Mac machine, compress the Mac `.dmg` (`genisoimage` does not compress it by default) with the following command on a mac machine: `hdiutil convert openrefine-uncompressed.dmg -format UDZO -imagekey zlib-level=9 -o openrefine-3.1-mac.dmg`. If running OS X in a VM, it's probably quicker and more reliable to transfer the kits to the host machine first and then to Github. Finder -> Go -> Connect -> smb://10.0.2.2/. You can then sign the generated DMG file with `codesign -s "Apple Distribution: Code for Science and Society, Inc." openrefine-3.1-mac.dmg`. This requires that you have installed the appropriate certificate on your Mac, see below. - -9. Tag the release candidate in git and push the tag to Github. For example: -```shell -git tag -a -m "Second beta" 2.6-beta.2 - git push origin --tags -``` -10. Upload the kits to Github releases [https://github.com/OpenRefine/OpenRefine/releases/](https://github.com/OpenRefine/OpenRefine/releases/) Mention the SHA sums of all uploaded artifacts. -11. Announce the beta/release candidate for testing -12. Repeat build/release candidate/testing cycle, if necessary. -13. Tag the release in git. Build the distributions and upload them. -14. [Update the OpenRefine Homebrew cask](https://github.com/OpenRefine/OpenRefine/wiki/Maintaining-OpenRefine's-Homebrew-Cask) or coordinate an update via the [developer list](https://groups.google.com/forum/#!forum/openrefine-dev) -15. Verify that the correct versions are shown in the widget at [http://openrefine.org/download](http://openrefine.org/download) -16. Announce on the [OpenRefine mailing list](https://groups.google.com/forum/#!forum/openrefine). -17. Update the version in master to the next version number with `-SNAPSHOT` (such as `4.3-SNAPSHOT`) -```shell -mvn versions:set -DnewVersion=4.3-SNAPSHOT -``` -18. If releasing a new major or minor version, create a snapshot of the docs, following [Docusaurus' versioning procedure](https://docusaurus.io/docs/versioning). - -Apple code signing -================== - -We have code signing certificates for our iOS distributions. To use them, follow these steps: -* Request advisory.committee@openrefine.org to be added to the Apple team: you need to provide the email address that corresponds to your AppleID account; -* Create a certificate signing request from your Mac: https://help.apple.com/developer-account/#/devbfa00fef7 -* Go to https://developer.apple.com/account/resources/certificates/add and select "Apple Distribution" as certificate type -* Upload the certificate signing request in the form -* Download the generated certificate -* Import this certificate in the "Keychain Access" app on your mac -* You can now sign code on behalf of the team using the `codesign` utility, such as `codesign -s "Apple Distribution: Code for Science and Society, Inc." openrefine-3.1-mac.dmg`. diff --git a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/writing-extensions.md b/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/writing-extensions.md deleted file mode 100644 index 3511cc31f..000000000 --- a/OpenRefine/docs/versioned_docs/version-3.5/technical-reference/writing-extensions.md +++ /dev/null @@ -1,326 +0,0 @@ ---- -id: writing-extensions -title: Writing Extensions -sidebar_label: Writing Extensions ---- - -## Introduction {#introduction} - -This is a very brief overview of the structure of OpenRefine extensions. For more detailed documentation and step-by-step guides please see the following external documentation/tutorials: - -* Giuliano Tortoreto has [written documentation detailling how to build extension for OpenRefine](https://github.com/giTorto/OpenRefineExtensionDoc/raw/master/main.pdf) -* Owen Stephens has written [a guide to developing an extension which adds new GREL functions to OpenRefine](http://www.meanboyfriend.com/overdue_ideas/2017/05/writing-an-extension-to-add-new-grel-functions-to-openrefine/). - -OpenRefine makes use of a modified version of the [Butterfly framework](https://github.com/OpenRefine/simile-butterfly/tree/openrefine) to provide an extension architecture. OpenRefine extensions are Butterfly modules. You don't really need to know about Butterfly itself, but you might encounter "butterfly" here and there in the code base. - -Extensions that come with the code base are located under [the extensions subdirectory](https://github.com/OpenRefine/OpenRefine/tree/master/extensions), but when you develop your own extension, you can put its code anywhere as long as you point Butterfly to it. That is done by any one of the following methods - -* refer to your extension's directory in [the butterfly.properties file](https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/WEB-INF/butterfly.properties) through a `butterfly.modules.path` setting. -* specify the butterfly.modules.path property on the command line when you run OpenRefine. This overrides the values in the property file, so you need to include the default values first e.g. `-Dbutterfly.modules.path=modules,../../extensions,/path/to/your/extension` - -Please note that you should bundle any dependencies yourself, so you are insulated from OpenRefine packaging changes over time. - -### Directory Layout {#directory-layout} - -A OpenRefine extension sits in a file directory that contains the following files and sub-directories: - -``` -pom.xml - src/ - com/foo/bar/... *.java source files - module/ - *.html, *.vt files - scripts/... *.js files - styles/... *.css and *.less files - images/... image files - MOD-INF/ - lib/*.jar files - classes/... java class files - module.properties - controller.js -``` - -The file named module.properties (see [example](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/sample/module/MOD-INF/module.properties)) contains the extension's metadata. Of importance is the name field, which gives the extension a name that's used in many other places to refer to it. This can be different from the extension's directory name. - -``` -name = my-extension-name -``` - -Your extension's client-side resources (.html, .js, .css files) stored in the module/ subdirectory will be accessible from http://127.0.0.1:3333/extension/my-extension-name/ when OpenRefine is running. - -Also of importance is the dependency - -``` -requires = core -``` - -which makes sure that the core module of OpenRefine is loaded before the extension attempts to hook into it. - -The file named controller.js is responsible for registering the extension's hooks into OpenRefine. Look at the sample-extension extension's [controller.js](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/sample/module/MOD-INF/controller.js) file for an example. It should have a function called init() that does the hook registrations. - -The `pom.xml` file is an [Apache Maven](http://maven.apache.org/) build file. You can make a copy of the sample extension's `pom.xml` file to get started. The important point here is that the Java classes should be built into the `module/MOD-INF/classes` sub-directory. - -Note that your extension's Java code would need to reference some libraries used in OpenRefine and OpenRefine's Java classes themselves. These dependencies are reflected in the Maven configuration for the extension. - -## Sample extension {#sample-extension} - -The sample extension is included in the code base so that you can copy it and get started on writing your own extension. After you copy it, make sure you change its name inside its `module/MOD-INF/controller.js` file. - -### Basic Structure {#basic-structure} - -The sample extension's code is in `refine/extensions/sample/`. In that directory, Java source code is contained under the `src` sub-directory, and webapp code is under the `module` sub-directory. Here is the full directory layout: - -``` -refine/extensions/sample/ - build.xml (ant build script) - src/ - com/google/refine/sampleExtension/ - ... Java source code ... - module/ - MOD-INF/ - module.properties (module settings) - controller.js (module init and routing logic in Javascript) - classes/ - ... compiled Java classes ... - lib/ - ... Java jars ... - ... velocity templates (.vt) ... - ... LESS css files ... - ... client-side files (.html, .css, .js, image files) ... -``` - -The sub-directory `MOD-INF` contains the Butterfly module's metadata and is what Butterfly looks for when it scans directories for modules. `MOD-INF` serves similar functions as `WEB-INF` in other web frameworks. - -Java code is built into the sub-directory `classes` inside `MOD-INF`, and supporting external Java jars are in the `lib` sub-directory. Those will be automatically loaded by Butterfly. (The build.xml script is wired to compile into the `classes` sub-directory.) - -Client-side code is in the inner `module` sub-directory. They can be plain old .html, .css, .js, and image files, or they can be [LESS](http://lesscss.org/) files that get processed into CSS. There are also Velocity .vt files, but they need to be routed inside `MOD-INF/controller.js`. - -`MOD-INF/controller.js` lets you configure the extension's initialization and URL routing in Javascript rather than in Java. For example, when the requested URL path is either `/` or an empty string, we process and return `MOD-INF/index.vt` ( [see http://127.0.0.1:3333/extension/sample/](http://127.0.0.1:3333/extension/sample/) if OpenRefine is running). - -The `init()` function in `controller.js` allows the extension to register various client-side handlers for augmenting pages served by Refine's core. These handlers are feature-specific. For example, [this is where the jython extension adds its parser](https://github.com/OpenRefine/OpenRefine/blob/master/extensions/jython/module/MOD-INF/controller.js#L46). As for the sample extension, it adds its script `project-injection.js` and style `project-injection.less` into the `/project` page. If you [view the source of the /project page](http://127.0.0.1:3333/project), you will see references to those two files. - -### Wiring Up the Extension {#wiring-up-the-extension} - -The Extensions are loaded by the Butterfly framework. Butterfly refers to these as 'modules'. [The location of modules is set in the `main/webapp/butterfly.properties` file](https://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/WEB-INF/butterfly.properties#L27). Butterfly simply descends into each of those paths and looks for any `MOD-INF` directories. - -For more information, see [Extension Points](https://github.com/OpenRefine/OpenRefine/wiki/Extension-Points). - -## Extension points {#extension-points} - -### Client-side: Javascript and CSS {#client-side-javascript-and-css} - -The UI in OpenRefine for working with a project is coded in [the /main/webapp/modules/core/project.vt file](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/project.vt). The file is quite small, and that's because almost all of its content is to be expanded dynamically through the Velocity variables $scriptInjection and $styleInjection. So that your own Javascript and CSS files get loaded, you need to register them with the ClientSideResourceManager, which is done in the /module/MOD-INF/controller.js file. See [the controller.js file in this sample extension code](http://github.com/OpenRefine/OpenRefine/blob/master/extensions/sample/module/MOD-INF/controller.js) for an example. - -In the registration call, the variable `module` is already available to your code by default, and it refers to your own extension. - -``` -ClientSideResourceManager.addPaths( - "project/scripts", - module, - [ - "scripts/foo.js", - "scripts/subdir/bar.js" - ] - ); -``` - -You can specify one or more files for registration, and their paths are relative to the `module` subdirectory of your extension. They are included in the order listed. - -Javascript Bundling: Note that `project.vt` belongs to the core module and is thus under the control of the core module's [controller.js file](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/MOD-INF/controller.js). The Javascript files to be included in `project.vt` are by default bundled together for performance. When debugging, you can prevent this bundling behavior by setting `bundle` to `false` near the top of that `controller.js` file. (If you have commit access to this code base, be sure not to check that change in.) - -### Client-side: Images {#client-side-images} - -We recommend that you always refer to images through your CSS files rather than in your Javascript code. URLs to images will thus be relative to your CSS files, e.g., - -``` -.foo { - background: url(../images/x.png); - } -``` - -If you really really absolutely need to refer to your images in your Javascript code, then look up your extension's URL path in the global Javascript variable `ModuleWirings`: - -``` -ModuleWirings["my-extension"] + "images/x.png" -``` - -### Client-side: HTML Templates {#client-side-html-templates} - -Beside Javascript, CSS, and images, your extension might also include HTML templates that get loaded on the fly by your Javascript code and injected into the page's DOM. For example, here is [the Cluster edit dialog template](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/dialogs/clustering-dialog.html), which gets loaded by code in [the equivalent javascript file 'clustering-dialog.js'](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/dialogs/clustering-dialog.js): - -``` -var dialog = $(DOM.loadHTML("core", "scripts/dialogs/clustering-dialog.html")); -``` - -`DOM.loadHTML` returns the content of the file as a string, and `$(...)` turns it into a DOM fragment. Where `"core"` is, you would want your extension's name. The path of the HTML file is relative to your extension's `module` subdirectory. - -### Client-side: Project UI Extension Points {#client-side-project-ui-extension-points} - -Getting your extension's Javascript code included in `project.vt` doesn't accomplish much by itself unless your code also registers hooks into the UI. For example, you can surely implement an exporter in Javascript, but unless you add a corresponding menu command in the UI, your user can't use your exporter. - -#### Main Menu {#main-menu} - -The main menu can be extended by calling any one of the methods `MenuBar.appendTo`, `MenuBar.insertBefore`, and `MenuBar.insertAfter`. Each method takes 2 arguments: an array of strings that identify a particular existing menu item or submenu, and one new single menu item or submenu or an array of menu items and submenus. For example, to insert 2 menu items and a menu separator before the menu item Project > Export Filtered Rows > Templating..., write this Javascript code wherever that would execute when your Javascript files get loaded: - -``` -MenuBar.insertBefore( - ["core/project", "core/export", "core/export-templating"], - [ - { - "label":"Menu item 1", - "click": function() { ... } - }, - { - "label":"Menu item 2", - "click": function() { ... } - }, - {} // separator - ] - ); -``` - -The array `["core/project", "core/export", "core/export-templating"]` pinpoints the reference menu item. - -See the beginning of [/main/webapp/modules/core/scripts/project/menu-bar.js](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/project/menu-bar.js) for IDs of menu items and submenus. - -#### Column Header Menu {#column-header-menu} - -The drop-down menu of each column can also be extended, but the mechanism is slightly different compared to the main menu. Because the drop-down menu for a particular column is constructed on the fly when the user actually clicks the drop-down menu button, extending the column header menu can't really be done once at start-up time, but must be done every time a column header menu gets created. So, registration in this case involves providing a function that gets called each such time: - -``` -DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { ... do stuff to menu ... }); -``` - -That function takes in the column object (which contains the column's name), the column header UI object (generally not so useful), and the menu to extend. In the previous code line where it says "do stuff to menu", you can write something like this: - -``` -MenuSystem.appendTo(menu, ["core/facet"], [ - { - id: "core/text-facet", - label: "My Facet on " + column.name, - click: function() { - ... use column.name and do something ... - } - }, - ]); -``` - -In addition to `MenuSystem.appendTo`, you can also call `MenuSystem.insertBefore` and `MenuSystem.insertAfter` which the same 3 arguments. To see what IDs you can use, see the function `DataTableColumnHeaderUI.prototype._createMenuForColumnHeader` in [/main/webapp/modules/core/scripts/views/data-table/column-header-ui.js](http://github.com/OpenRefine/OpenRefine/blob/master/main/webapp/modules/core/scripts/views/data-table/column-header-ui.js). - -### Server-side: Ajax Commands {#server-side-ajax-commands} - -The client-side of OpenRefine gets things done by calling AJAX commands on the server-side. These commands must be registered with the OpenRefine servlet, so that the servlet knows how to route AJAX calls from the client-side. This can be done inside the `init` function in your extension's `controller.js` file, e.g., - -``` -function init() { - var RefineServlet = Packages.com.google.refine.RefineServlet; - RefineServlet.registerCommand(module, "my-command", new Packages.com.foo.bar.MyCommand()); - } -``` - -Your command will then be accessible at [http://127.0.0.1:3333/command/my-extension/my-command](http://127.0.0.1:3333/command/my-extension/my-command). - -### Server-side: Operations {#server-side-operations} - -Most commands change the project's data. Most of them do so by creating abstract operations. See the Changes, History, Processes, and Operations section of the [Server Side Architecture](https://github.com/OpenRefine/OpenRefine/wiki/Server-Side-Architecture) document. - -You can register an operation **class** in the `init` function as follows: - -``` -Packages.com.google.refine.operations.OperationRegistry.registerOperation( - module, - "operation-name", - Packages.com.foo.bar.MyOperation - ); -``` - -Do not call `new` to construct an operation instance. You must register the class itself. The class should have a static function for reconstructing an operation instance from a JSON blob: - -``` -static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { - ... - } -``` - -### Server-side: GREL {#server-side-grel} - -GREL can be extended with new functions. This is also done in the `init` function in `controller.js`, e.g., - -``` -Packages.com.google.refine.grel.ControlFunctionRegistry.registerFunction( - "functionName", new Packages.com.foo.bar.TheFunctionClass()); -``` - -You might also want to provide new variables (beyond just `value`, `cells`, `row`, etc.) available to expressions. This is done by registering a binder that implements the interface `com.google.refine.expr.Binder`: - -``` -Packages.com.google.refine.expr.ExpressionUtils.registerBinder( - new Packages.com.foo.bar.MyBinder()); -``` - -### Server-side: Importers {#server-side-importers} - -You can register an importer as follows: - -``` -Packages.com.google.refine.importers.ImporterRegistry.registerImporter( - "importer-name", new Packages.com.foo.bar.MyImporter()); -``` - -The string `"importer-name"` isn't important at all. It's not really related to file extension or mime-type. Just use something unique. Your importer will be explicitly called to test if it can import something. - -### Server-side: Exporters {#server-side-exporters} - -You can register an exporter as follows: - -``` -Packages.com.google.refine.exporters.ExporterRegistry.registerExporter( - "exporter-name", new Packages.com.foo.bar.MyExporter()); -``` - -The string `"exporter-name"` isn't important at all. It's only used by the client-side to tell the server-side which exporter to use. Just use something unique and, of course, relevant. - -### Server-side: Overlay Models {#server-side-overlay-models} - -Overlay models are objects attached onto a core Project object to store and manage additional data for that project. For example, the schema alignment skeleton is managed by the Protograph overlay model. An overlay model implements the interface `com.google.refine.model.OverlayModel` and can be registered like so: - -``` -Packages.com.google.refine.model.Project.registerOverlayModel( - "model-name", - Packages.com.foo.bar.MyOverlayModel); -``` - -Note that you register the **class** , not an instance. The class should implement the following static method for reconstructing an overlay model instance from a JSON blob: - -``` -static public OverlayModel reconstruct(JSONObject o) throws JSONException { - ... - } -``` - -When the project gets saved, the overlay model instance's `write` method will be called: - -``` -public void write(JSONWriter writer, Properties options) throws JSONException { - ... - } -``` - -### Server-side: Scripting Languages {#server-side-scripting-languages} - -A scripting language (such as Jython) can be registered as follows: - -``` -Packages.com.google.refine.expr.MetaParser.registerLanguageParser( - "jython", - "Jython", - Packages.com.google.refine.jython.JythonEvaluable.createParser(), - "return value" - ); -``` - -The first string is the prefix that gets prepended to each expression so that we know which language the expression is in. This should be short, unique, and identifying. The second string is a user-friendly name of the language. The third is an object that implements the interface `com.google.refine.expr.LanguageSpecificParser`. The final string is the default expression in that language that would return the cell's value. - -In 2018 we are making important changes to OpenRefine to modernize it, for the benefit of users and contributors. This page describes the changes that impact developers of extensions or forks and is intended to minimize the effort required on their end to follow the transition. The instructions are written specifically with extension maintainers in mind, but fork maintainers should also find it useful. - -This document describes the migrations in the order they are committed to the master branch. This means that it should be possible to perform each migration in turn, with the ability to run the software between each stage by checking out the appropriate git commit. diff --git a/OpenRefine/docs/versioned_sidebars/version-3.4-sidebars.json b/OpenRefine/docs/versioned_sidebars/version-3.4-sidebars.json deleted file mode 100644 index 89e8788b2..000000000 --- a/OpenRefine/docs/versioned_sidebars/version-3.4-sidebars.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "version-3.4/docs": [ - { - "collapsed": true, - "type": "category", - "label": "User Manual", - "items": [ - { - "type": "doc", - "id": "version-3.4/index" - }, - { - "type": "doc", - "id": "version-3.4/manual/installing" - }, - { - "type": "doc", - "id": "version-3.4/manual/running" - }, - { - "type": "doc", - "id": "version-3.4/manual/starting" - }, - { - "collapsed": true, - "type": "category", - "label": "Exploring data", - "items": [ - { - "type": "doc", - "id": "version-3.4/manual/exploring" - }, - { - "type": "doc", - "id": "version-3.4/manual/facets" - }, - { - "type": "doc", - "id": "version-3.4/manual/sortview" - } - ] - }, - { - "collapsed": true, - "type": "category", - "label": "Transforming data", - "items": [ - { - "type": "doc", - "id": "version-3.4/manual/transforming" - }, - { - "type": "doc", - "id": "version-3.4/manual/cellediting" - }, - { - "type": "doc", - "id": "version-3.4/manual/columnediting" - }, - { - "type": "doc", - "id": "version-3.4/manual/transposing" - } - ] - }, - { - "type": "doc", - "id": "version-3.4/manual/reconciling" - }, - { - "type": "doc", - "id": "version-3.4/manual/wikidata" - }, - { - "collapsed": true, - "type": "category", - "label": "Expressions", - "items": [ - { - "type": "doc", - "id": "version-3.4/manual/expressions" - }, - { - "type": "doc", - "id": "version-3.4/manual/grel" - }, - { - "type": "doc", - "id": "version-3.4/manual/grelfunctions" - }, - { - "type": "doc", - "id": "version-3.4/manual/jythonclojure" - } - ] - }, - { - "type": "doc", - "id": "version-3.4/manual/exporting" - }, - { - "type": "doc", - "id": "version-3.4/manual/troubleshooting" - } - ] - }, - { - "collapsed": true, - "type": "category", - "label": "GREL Reference", - "items": [ - { - "type": "doc", - "id": "version-3.4/manual/grelfunctions" - } - ] - }, - { - "collapsed": true, - "type": "category", - "label": "Technical Reference", - "items": [ - { - "type": "doc", - "id": "version-3.4/technical-reference/technical-reference-index" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/architecture" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/openrefine-api" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/reconciliation-api" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/suggest-api" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/data-extension-api" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/contributing" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/build-test-run" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/development-roadmap" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/version-release-process" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/homebrew-cask-process" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/writing-extensions" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/migrating-older-extensions" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/translating" - }, - { - "type": "doc", - "id": "version-3.4/technical-reference/functional-tests" - } - ] - } - ] -} diff --git a/OpenRefine/docs/versioned_sidebars/version-3.5-sidebars.json b/OpenRefine/docs/versioned_sidebars/version-3.5-sidebars.json deleted file mode 100644 index d925ba1dc..000000000 --- a/OpenRefine/docs/versioned_sidebars/version-3.5-sidebars.json +++ /dev/null @@ -1,217 +0,0 @@ -{ - "version-3.5/docs": [ - { - "collapsed": true, - "type": "category", - "label": "User Manual", - "items": [ - { - "type": "doc", - "id": "version-3.5/index" - }, - { - "type": "doc", - "id": "version-3.5/manual/installing" - }, - { - "type": "doc", - "id": "version-3.5/manual/running" - }, - { - "type": "doc", - "id": "version-3.5/manual/starting" - }, - { - "collapsed": true, - "type": "category", - "label": "Exploring data", - "items": [ - { - "type": "doc", - "id": "version-3.5/manual/exploring" - }, - { - "type": "doc", - "id": "version-3.5/manual/facets" - }, - { - "type": "doc", - "id": "version-3.5/manual/sortview" - } - ] - }, - { - "collapsed": true, - "type": "category", - "label": "Transforming data", - "items": [ - { - "type": "doc", - "id": "version-3.5/manual/transforming" - }, - { - "type": "doc", - "id": "version-3.5/manual/cellediting" - }, - { - "type": "doc", - "id": "version-3.5/manual/columnediting" - }, - { - "type": "doc", - "id": "version-3.5/manual/transposing" - } - ] - }, - { - "type": "doc", - "id": "version-3.5/manual/reconciling" - }, - { - "collapsed": true, - "type": "category", - "label": "Wikibase", - "items": [ - { - "type": "doc", - "id": "version-3.5/manual/wikibase/overview" - }, - { - "type": "doc", - "id": "version-3.5/manual/wikibase/configuration" - }, - { - "type": "doc", - "id": "version-3.5/manual/wikibase/reconciling" - }, - { - "type": "doc", - "id": "version-3.5/manual/wikibase/schema-alignment" - }, - { - "type": "doc", - "id": "version-3.5/manual/wikibase/new-entities" - }, - { - "type": "doc", - "id": "version-3.5/manual/wikibase/quality-assurance" - }, - { - "type": "doc", - "id": "version-3.5/manual/wikibase/uploading" - } - ] - }, - { - "collapsed": true, - "type": "category", - "label": "Expressions", - "items": [ - { - "type": "doc", - "id": "version-3.5/manual/expressions" - }, - { - "type": "doc", - "id": "version-3.5/manual/grel" - }, - { - "type": "doc", - "id": "version-3.5/manual/grelfunctions" - }, - { - "type": "doc", - "id": "version-3.5/manual/jythonclojure" - } - ] - }, - { - "type": "doc", - "id": "version-3.5/manual/exporting" - }, - { - "type": "doc", - "id": "version-3.5/manual/troubleshooting" - } - ] - }, - { - "collapsed": true, - "type": "category", - "label": "GREL Reference", - "items": [ - { - "type": "doc", - "id": "version-3.5/manual/grelfunctions" - } - ] - }, - { - "collapsed": true, - "type": "category", - "label": "Technical Reference", - "items": [ - { - "type": "doc", - "id": "version-3.5/technical-reference/technical-reference-index" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/architecture" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/openrefine-api" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/reconciliation-api" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/contributing" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/build-test-run" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/development-roadmap" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/version-release-process" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/homebrew-cask-process" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/writing-extensions" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/migrating-older-extensions" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/translating-ui" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/translating-docs" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/functional-tests" - }, - { - "type": "doc", - "id": "version-3.5/technical-reference/maintainer-guidelines" - } - ] - } - ] -} diff --git a/OpenRefine/docs/versions.json b/OpenRefine/docs/versions.json deleted file mode 100644 index 61b9fd005..000000000 --- a/OpenRefine/docs/versions.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - "3.5", - "3.4" -] diff --git a/OpenRefine/docs/yarn.lock b/OpenRefine/docs/yarn.lock deleted file mode 100644 index 17141b3f5..000000000 --- a/OpenRefine/docs/yarn.lock +++ /dev/null @@ -1,9509 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -"@algolia/autocomplete-core@1.5.0": - version "1.5.0" - resolved "https://registry.yarnpkg.com/@algolia/autocomplete-core/-/autocomplete-core-1.5.0.tgz#6c91c9de7748e9c103846828a58dfe92bd4d6689" - integrity sha512-E7+VJwcvwMM8vPeaVn7fNUgix8WHV8A1WUeHDi2KHemCaaGc8lvUnP3QnvhMxiDhTe7OpMEv4o2TBUMyDgThaw== - dependencies: - "@algolia/autocomplete-shared" "1.5.0" - -"@algolia/autocomplete-preset-algolia@1.5.0": - version "1.5.0" - resolved "https://registry.yarnpkg.com/@algolia/autocomplete-preset-algolia/-/autocomplete-preset-algolia-1.5.0.tgz#61671f09c0c77133d9baf1356719f8378c48437a" - integrity sha512-iiFxKERGHkvkiupmrFJbvESpP/zv5jSgH714XRiP5LDvUHaYOo4GLAwZCFf2ef/L5tdtPBARvekn6k1Xf33gjA== - dependencies: - "@algolia/autocomplete-shared" "1.5.0" - -"@algolia/autocomplete-shared@1.5.0": - version "1.5.0" - resolved "https://registry.yarnpkg.com/@algolia/autocomplete-shared/-/autocomplete-shared-1.5.0.tgz#09580bc89408a2ab5f29e312120dad68f58019bd" - integrity sha512-bRSkqHHHSwZYbFY3w9hgMyQRm86Wz27bRaGCbNldLfbk0zUjApmE4ajx+ZCVSLqxvcUEjMqZFJzDsder12eKsg== - -"@algolia/cache-browser-local-storage@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.11.0.tgz#1c168add00b398a860db6c86039e33b2843a9425" - integrity sha512-4sr9vHIG1fVA9dONagdzhsI/6M5mjs/qOe2xUP0yBmwsTsuwiZq3+Xu6D3dsxsuFetcJgC6ydQoCW8b7fDJHYQ== - dependencies: - "@algolia/cache-common" "4.11.0" - -"@algolia/cache-browser-local-storage@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.9.1.tgz#784e91580dcca00a8280b0905197f5abbbdf4b48" - integrity sha512-bAUU9vKCy45uTTlzJw0LYu1IjoZsmzL6lgjaVFaW1crhX/4P+JD5ReQv3n/wpiXSFaHq1WEO3WyH2g3ymzeipQ== - dependencies: - "@algolia/cache-common" "4.9.1" - -"@algolia/cache-common@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/cache-common/-/cache-common-4.11.0.tgz#066fe6d58b18e4b028dbef9bb8de07c5e22a3594" - integrity sha512-lODcJRuPXqf+6mp0h6bOxPMlbNoyn3VfjBVcQh70EDP0/xExZbkpecgHyyZK4kWg+evu+mmgvTK3GVHnet/xKw== - -"@algolia/cache-common@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/cache-common/-/cache-common-4.9.1.tgz#2d5f37ba7aab7db76627c4a4fce51a7fd137fa65" - integrity sha512-tcvw4mOfFy44V4ZxDEy9wNGr6vFROZKRpXKTEBgdw/WBn6mX51H1ar4RWtceDEcDU4H5fIv5tsY3ip2hU+fTPg== - -"@algolia/cache-in-memory@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/cache-in-memory/-/cache-in-memory-4.11.0.tgz#763c8cb655e6fd2261588e04214fca0959ac07c1" - integrity sha512-aBz+stMSTBOBaBEQ43zJXz2DnwS7fL6dR0e2myehAgtfAWlWwLDHruc/98VOy1ZAcBk1blE2LCU02bT5HekGxQ== - dependencies: - "@algolia/cache-common" "4.11.0" - -"@algolia/cache-in-memory@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/cache-in-memory/-/cache-in-memory-4.9.1.tgz#3fd1d67aec804b6cc8439015b8b9c712a45c7ae0" - integrity sha512-IEJrHonvdymW2CnRfJtsTVWyfAH05xPEFkGXGCw00+6JNCj8Dln3TeaRLiaaY1srlyGedkemekQm1/Xb46CGOQ== - dependencies: - "@algolia/cache-common" "4.9.1" - -"@algolia/client-account@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/client-account/-/client-account-4.11.0.tgz#67fadd3b0802b013ebaaa4b47bb7babae892374e" - integrity sha512-jwmFBoUSzoMwMqgD3PmzFJV/d19p1RJXB6C1ADz4ju4mU7rkaQLtqyZroQpheLoU5s5Tilmn/T8/0U2XLoJCRQ== - dependencies: - "@algolia/client-common" "4.11.0" - "@algolia/client-search" "4.11.0" - "@algolia/transporter" "4.11.0" - -"@algolia/client-account@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/client-account/-/client-account-4.9.1.tgz#f2c1b3e49de2ee1fca44b8b5e64e1ce0dbdff0db" - integrity sha512-Shpjeuwb7i2LR5QuWREb6UbEQLGB+Pl/J5+wPgILJDP/uWp7jpl0ase9mYNQGKj7TjztpSpQCPZ3dSHPnzZPfw== - dependencies: - "@algolia/client-common" "4.9.1" - "@algolia/client-search" "4.9.1" - "@algolia/transporter" "4.9.1" - -"@algolia/client-analytics@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/client-analytics/-/client-analytics-4.11.0.tgz#cbdc8128205e2da749cafc79e54708d14c413974" - integrity sha512-v5U9585aeEdYml7JqggHAj3E5CQ+jPwGVztPVhakBk8H/cmLyPS2g8wvmIbaEZCHmWn4TqFj3EBHVYxAl36fSA== - dependencies: - "@algolia/client-common" "4.11.0" - "@algolia/client-search" "4.11.0" - "@algolia/requester-common" "4.11.0" - "@algolia/transporter" "4.11.0" - -"@algolia/client-analytics@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/client-analytics/-/client-analytics-4.9.1.tgz#56972496526910c53c5ce7844f4571efba63eb5f" - integrity sha512-/g6OkOSIA+A0t/tjvbL6iG/zV4El4LPFgv/tcAYHTH27BmlNtnEXw+iFpGjeUlQoPily9WVB3QNLMJkaNwL3HA== - dependencies: - "@algolia/client-common" "4.9.1" - "@algolia/client-search" "4.9.1" - "@algolia/requester-common" "4.9.1" - "@algolia/transporter" "4.9.1" - -"@algolia/client-common@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/client-common/-/client-common-4.11.0.tgz#9a2d1f6f8eaad25ba5d6d4ce307ba5bd84e6f999" - integrity sha512-Qy+F+TZq12kc7tgfC+FM3RvYH/Ati7sUiUv/LkvlxFwNwNPwWGoZO81AzVSareXT/ksDDrabD4mHbdTbBPTRmQ== - dependencies: - "@algolia/requester-common" "4.11.0" - "@algolia/transporter" "4.11.0" - -"@algolia/client-common@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/client-common/-/client-common-4.9.1.tgz#ae313b65d3249efcb4fafd2e92ed1fa2fd075482" - integrity sha512-UziRTZ8km3qwoVPIyEre8TV6V+MX7UtbfVqPmSafZ0xu41UUZ+sL56YoKjOXkbKuybeIC9prXMGy/ID5bXkTqg== - dependencies: - "@algolia/requester-common" "4.9.1" - "@algolia/transporter" "4.9.1" - -"@algolia/client-personalization@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/client-personalization/-/client-personalization-4.11.0.tgz#d3bf0e760f85df876b4baf5b81996f0aa3a59940" - integrity sha512-mI+X5IKiijHAzf9fy8VSl/GTT67dzFDnJ0QAM8D9cMPevnfX4U72HRln3Mjd0xEaYUOGve8TK/fMg7d3Z5yG6g== - dependencies: - "@algolia/client-common" "4.11.0" - "@algolia/requester-common" "4.11.0" - "@algolia/transporter" "4.11.0" - -"@algolia/client-recommendation@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/client-recommendation/-/client-recommendation-4.9.1.tgz#217af2a38d37ab12cf23a419cc9a576af9d15b13" - integrity sha512-Drtvvm1PNIOpYf4HFlkPFstFQ3IsN+TRmxur2F7y6Faplb5ybISa8ithu1tmlTdyTf3A78hQUQjgJet6qD2XZw== - dependencies: - "@algolia/client-common" "4.9.1" - "@algolia/requester-common" "4.9.1" - "@algolia/transporter" "4.9.1" - -"@algolia/client-search@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/client-search/-/client-search-4.11.0.tgz#c1105d715a2a04ba27231eca86f5d6620f68f4ae" - integrity sha512-iovPLc5YgiXBdw2qMhU65sINgo9umWbHFzInxoNErWnYoTQWfXsW6P54/NlKx5uscoLVjSf+5RUWwFu5BX+lpw== - dependencies: - "@algolia/client-common" "4.11.0" - "@algolia/requester-common" "4.11.0" - "@algolia/transporter" "4.11.0" - -"@algolia/client-search@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/client-search/-/client-search-4.9.1.tgz#a2fbc47a1b343dade9a8b06310231d51ff675b1b" - integrity sha512-r9Cw2r8kJr45iYncFDht6EshARghU265wuY8Q8oHrpFHjAziEYdsUOdNmQKbsSH5J3gLjDPx1EI5DzVd6ivn3w== - dependencies: - "@algolia/client-common" "4.9.1" - "@algolia/requester-common" "4.9.1" - "@algolia/transporter" "4.9.1" - -"@algolia/logger-common@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/logger-common/-/logger-common-4.11.0.tgz#bac1c2d59d29dee378b57412c8edd435b97de663" - integrity sha512-pRMJFeOY8hoWKIxWuGHIrqnEKN/kqKh7UilDffG/+PeEGxBuku+Wq5CfdTFG0C9ewUvn8mAJn5BhYA5k8y0Jqg== - -"@algolia/logger-common@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/logger-common/-/logger-common-4.9.1.tgz#3323834095f2916338d2535d2df91c4723ac19f2" - integrity sha512-9mPrbFlFyPT7or/7PXTiJjyOewWB9QRkZKVXkt5zHAUiUzGxmmdpJIGpPv3YQnDur8lXrXaRI0MHXUuIDMY1ng== - -"@algolia/logger-console@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/logger-console/-/logger-console-4.11.0.tgz#ced19e3abb22eb782ed5268d51efb5aa9ef109ef" - integrity sha512-wXztMk0a3VbNmYP8Kpc+F7ekuvaqZmozM2eTLok0XIshpAeZ/NJDHDffXK2Pw+NF0wmHqurptLYwKoikjBYvhQ== - dependencies: - "@algolia/logger-common" "4.11.0" - -"@algolia/logger-console@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/logger-console/-/logger-console-4.9.1.tgz#c324ef26843dbed06b44586309331dbb949744ad" - integrity sha512-74VUwjtFjFpjZpi3QoHIPv0kcr3vWUSHX/Vs8PJW3lPsD4CgyhFenQbG9v+ZnyH0JrJwiYTtzfmrVh7IMWZGrQ== - dependencies: - "@algolia/logger-common" "4.9.1" - -"@algolia/requester-browser-xhr@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.11.0.tgz#f9e1ad56f185432aa8dde8cad53ae271fd5d6181" - integrity sha512-Fp3SfDihAAFR8bllg8P5ouWi3+qpEVN5e7hrtVIYldKBOuI/qFv80Zv/3/AMKNJQRYglS4zWyPuqrXm58nz6KA== - dependencies: - "@algolia/requester-common" "4.11.0" - -"@algolia/requester-browser-xhr@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.9.1.tgz#0812f3c7c4105a4646c0fba8429b172b2d0e01c5" - integrity sha512-zc46tk5o0ikOAz3uYiRAMxC2iVKAMFKT7nNZnLB5IzT0uqAh7pz/+D/UvIxP4bKmsllpBSnPcpfQF+OI4Ag/BA== - dependencies: - "@algolia/requester-common" "4.9.1" - -"@algolia/requester-common@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/requester-common/-/requester-common-4.11.0.tgz#d16de98d3ff72434bac39e4d915eab08035946a9" - integrity sha512-+cZGe/9fuYgGuxjaBC+xTGBkK7OIYdfapxhfvEf03dviLMPmhmVYFJtJlzAjQ2YmGDJpHrGgAYj3i/fbs8yhiA== - -"@algolia/requester-common@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/requester-common/-/requester-common-4.9.1.tgz#50fcf4c7c1ed7ae13159167ac1da2844d036a630" - integrity sha512-9hPgXnlCSbqJqF69M5x5WN3h51Dc+mk/iWNeJSVxExHGvCDfBBZd0v6S15i8q2a9cD1I2RnhMpbnX5BmGtabVA== - -"@algolia/requester-node-http@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/requester-node-http/-/requester-node-http-4.11.0.tgz#beb2b6b68d5f4ce15aec80ede623f0ac96991368" - integrity sha512-qJIk9SHRFkKDi6dMT9hba8X1J1z92T5AZIgl+tsApjTGIRQXJLTIm+0q4yOefokfu4CoxYwRZ9QAq+ouGwfeOg== - dependencies: - "@algolia/requester-common" "4.11.0" - -"@algolia/requester-node-http@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/requester-node-http/-/requester-node-http-4.9.1.tgz#70054a0aa5643072404fcb68042eec97c7abd1c8" - integrity sha512-vYNVbSCuyrCSCjHBQJk+tLZtWCjvvDf5tSbRJjyJYMqpnXuIuP7gZm24iHil4NPYBhbBj5NU2ZDAhc/gTn75Ag== - dependencies: - "@algolia/requester-common" "4.9.1" - -"@algolia/transporter@4.11.0": - version "4.11.0" - resolved "https://registry.yarnpkg.com/@algolia/transporter/-/transporter-4.11.0.tgz#a8de3c173093ceceb02b26b577395ce3b3d4b96f" - integrity sha512-k4dyxiaEfYpw4UqybK9q7lrFzehygo6KV3OCYJMMdX0IMWV0m4DXdU27c1zYRYtthaFYaBzGF4Kjcl8p8vxCKw== - dependencies: - "@algolia/cache-common" "4.11.0" - "@algolia/logger-common" "4.11.0" - "@algolia/requester-common" "4.11.0" - -"@algolia/transporter@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@algolia/transporter/-/transporter-4.9.1.tgz#63ef3d9ae3b6556fa1ff1e6265bbab482bd084b7" - integrity sha512-AbjFfGzX+cAuj7Qyc536OxIQzjFOA5FU2ANGStx8LBH+AKXScwfkx67C05riuaRR5adSCLMSEbVvUscH0nF+6A== - dependencies: - "@algolia/cache-common" "4.9.1" - "@algolia/logger-common" "4.9.1" - "@algolia/requester-common" "4.9.1" - -"@babel/code-frame@7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.4.tgz#168da1a36e90da68ae8d49c0f1b48c7c6249213a" - integrity sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg== - dependencies: - "@babel/highlight" "^7.10.4" - -"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.13": - version "7.12.13" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.13.tgz#dcfc826beef65e75c50e21d3837d7d95798dd658" - integrity sha512-HV1Cm0Q3ZrpCR93tkWOYiuYIgLxZXZFVG2VgK+MBWjUqZTundupbfx2aXarXuw5Ko5aMcjtJgbSs4vUGBS5v6g== - dependencies: - "@babel/highlight" "^7.12.13" - -"@babel/code-frame@^7.16.0", "@babel/code-frame@^7.5.5": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.16.0.tgz#0dfc80309beec8411e65e706461c408b0bb9b431" - integrity sha512-IF4EOMEV+bfYwOmNxGzSnjR2EmQod7f1UXOpZM3l4i4o4QNwzjtJAu/HxdjHq0aYBvdqMuQEY1eg0nqW9ZPORA== - dependencies: - "@babel/highlight" "^7.16.0" - -"@babel/compat-data@^7.13.11", "@babel/compat-data@^7.13.15": - version "7.13.15" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.13.15.tgz#7e8eea42d0b64fda2b375b22d06c605222e848f4" - integrity sha512-ltnibHKR1VnrU4ymHyQ/CXtNXI6yZC0oJThyW78Hft8XndANwi+9H+UIklBDraIjFEJzw8wmcM427oDd9KS5wA== - -"@babel/compat-data@^7.16.0", "@babel/compat-data@^7.16.4": - version "7.16.4" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.16.4.tgz#081d6bbc336ec5c2435c6346b2ae1fb98b5ac68e" - integrity sha512-1o/jo7D+kC9ZjHX5v+EHrdjl3PhxMrLSOTGsOdHJ+KL8HCaEK6ehrVL2RS6oHDZp+L7xLirLrPmQtEng769J/Q== - -"@babel/core@7.12.9": - version "7.12.9" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.12.9.tgz#fd450c4ec10cdbb980e2928b7aa7a28484593fc8" - integrity sha512-gTXYh3M5wb7FRXQy+FErKFAv90BnlOuNn1QkCK2lREoPAjrQCO49+HVSrFoe5uakFAF5eenS75KbO2vQiLrTMQ== - dependencies: - "@babel/code-frame" "^7.10.4" - "@babel/generator" "^7.12.5" - "@babel/helper-module-transforms" "^7.12.1" - "@babel/helpers" "^7.12.5" - "@babel/parser" "^7.12.7" - "@babel/template" "^7.12.7" - "@babel/traverse" "^7.12.9" - "@babel/types" "^7.12.7" - convert-source-map "^1.7.0" - debug "^4.1.0" - gensync "^1.0.0-beta.1" - json5 "^2.1.2" - lodash "^4.17.19" - resolve "^1.3.2" - semver "^5.4.1" - source-map "^0.5.0" - -"@babel/core@^7.12.16", "@babel/core@^7.12.3": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.16.0.tgz#c4ff44046f5fe310525cc9eb4ef5147f0c5374d4" - integrity sha512-mYZEvshBRHGsIAiyH5PzCFTCfbWfoYbO/jcSdXQSUQu1/pW0xDZAUP7KEc32heqWTAfAHhV9j1vH8Sav7l+JNQ== - dependencies: - "@babel/code-frame" "^7.16.0" - "@babel/generator" "^7.16.0" - "@babel/helper-compilation-targets" "^7.16.0" - "@babel/helper-module-transforms" "^7.16.0" - "@babel/helpers" "^7.16.0" - "@babel/parser" "^7.16.0" - "@babel/template" "^7.16.0" - "@babel/traverse" "^7.16.0" - "@babel/types" "^7.16.0" - convert-source-map "^1.7.0" - debug "^4.1.0" - gensync "^1.0.0-beta.2" - json5 "^2.1.2" - semver "^6.3.0" - source-map "^0.5.0" - -"@babel/generator@^7.12.15", "@babel/generator@^7.12.5", "@babel/generator@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.16.0.tgz#d40f3d1d5075e62d3500bccb67f3daa8a95265b2" - integrity sha512-RR8hUCfRQn9j9RPKEVXo9LiwoxLPYn6hNZlvUOR8tSnaxlD0p0+la00ZP9/SnRt6HchKr+X0fO2r8vrETiJGew== - dependencies: - "@babel/types" "^7.16.0" - jsesc "^2.5.1" - source-map "^0.5.0" - -"@babel/helper-annotate-as-pure@^7.12.13": - version "7.12.13" - resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.12.13.tgz#0f58e86dfc4bb3b1fcd7db806570e177d439b6ab" - integrity sha512-7YXfX5wQ5aYM/BOlbSccHDbuXXFPxeoUmfWtz8le2yTkTZc+BxsiEnENFoi2SlmA8ewDkG2LgIMIVzzn2h8kfw== - dependencies: - "@babel/types" "^7.12.13" - -"@babel/helper-annotate-as-pure@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.16.0.tgz#9a1f0ebcda53d9a2d00108c4ceace6a5d5f1f08d" - integrity sha512-ItmYF9vR4zA8cByDocY05o0LGUkp1zhbTQOH1NFyl5xXEqlTJQCEJjieriw+aFpxo16swMxUnUiKS7a/r4vtHg== - dependencies: - "@babel/types" "^7.16.0" - -"@babel/helper-builder-binary-assignment-operator-visitor@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.16.0.tgz#f1a686b92da794020c26582eb852e9accd0d7882" - integrity sha512-9KuleLT0e77wFUku6TUkqZzCEymBdtuQQ27MhEKzf9UOOJu3cYj98kyaDAzxpC7lV6DGiZFuC8XqDsq8/Kl6aQ== - dependencies: - "@babel/helper-explode-assignable-expression" "^7.16.0" - "@babel/types" "^7.16.0" - -"@babel/helper-compilation-targets@^7.13.0": - version "7.13.16" - resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.13.16.tgz#6e91dccf15e3f43e5556dffe32d860109887563c" - integrity sha512-3gmkYIrpqsLlieFwjkGgLaSHmhnvlAYzZLlYVjlW+QwI+1zE17kGxuJGmIqDQdYp56XdmGeD+Bswx0UTyG18xA== - dependencies: - "@babel/compat-data" "^7.13.15" - "@babel/helper-validator-option" "^7.12.17" - browserslist "^4.14.5" - semver "^6.3.0" - -"@babel/helper-compilation-targets@^7.16.0", "@babel/helper-compilation-targets@^7.16.3": - version "7.16.3" - resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.16.3.tgz#5b480cd13f68363df6ec4dc8ac8e2da11363cbf0" - integrity sha512-vKsoSQAyBmxS35JUOOt+07cLc6Nk/2ljLIHwmq2/NM6hdioUaqEXq/S+nXvbvXbZkNDlWOymPanJGOc4CBjSJA== - dependencies: - "@babel/compat-data" "^7.16.0" - "@babel/helper-validator-option" "^7.14.5" - browserslist "^4.17.5" - semver "^6.3.0" - -"@babel/helper-create-class-features-plugin@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.16.0.tgz#090d4d166b342a03a9fec37ef4fd5aeb9c7c6a4b" - integrity sha512-XLwWvqEaq19zFlF5PTgOod4bUA+XbkR4WLQBct1bkzmxJGB0ZEJaoKF4c8cgH9oBtCDuYJ8BP5NB9uFiEgO5QA== - dependencies: - "@babel/helper-annotate-as-pure" "^7.16.0" - "@babel/helper-function-name" "^7.16.0" - "@babel/helper-member-expression-to-functions" "^7.16.0" - "@babel/helper-optimise-call-expression" "^7.16.0" - "@babel/helper-replace-supers" "^7.16.0" - "@babel/helper-split-export-declaration" "^7.16.0" - -"@babel/helper-create-regexp-features-plugin@^7.12.13": - version "7.12.17" - resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.12.17.tgz#a2ac87e9e319269ac655b8d4415e94d38d663cb7" - integrity sha512-p2VGmBu9oefLZ2nQpgnEnG0ZlRPvL8gAGvPUMQwUdaE8k49rOMuZpOwdQoy5qJf6K8jL3bcAMhVUlHAjIgJHUg== - dependencies: - "@babel/helper-annotate-as-pure" "^7.12.13" - regexpu-core "^4.7.1" - -"@babel/helper-create-regexp-features-plugin@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.16.0.tgz#06b2348ce37fccc4f5e18dcd8d75053f2a7c44ff" - integrity sha512-3DyG0zAFAZKcOp7aVr33ddwkxJ0Z0Jr5V99y3I690eYLpukJsJvAbzTy1ewoCqsML8SbIrjH14Jc/nSQ4TvNPA== - dependencies: - "@babel/helper-annotate-as-pure" "^7.16.0" - regexpu-core "^4.7.1" - -"@babel/helper-define-polyfill-provider@^0.3.0": - version "0.3.0" - resolved "https://registry.yarnpkg.com/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.3.0.tgz#c5b10cf4b324ff840140bb07e05b8564af2ae971" - integrity sha512-7hfT8lUljl/tM3h+izTX/pO3W3frz2ok6Pk+gzys8iJqDfZrZy2pXjRTZAvG2YmfHun1X4q8/UZRLatMfqc5Tg== - dependencies: - "@babel/helper-compilation-targets" "^7.13.0" - "@babel/helper-module-imports" "^7.12.13" - "@babel/helper-plugin-utils" "^7.13.0" - "@babel/traverse" "^7.13.0" - debug "^4.1.1" - lodash.debounce "^4.0.8" - resolve "^1.14.2" - semver "^6.1.2" - -"@babel/helper-explode-assignable-expression@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.16.0.tgz#753017337a15f46f9c09f674cff10cee9b9d7778" - integrity sha512-Hk2SLxC9ZbcOhLpg/yMznzJ11W++lg5GMbxt1ev6TXUiJB0N42KPC+7w8a+eWGuqDnUYuwStJoZHM7RgmIOaGQ== - dependencies: - "@babel/types" "^7.16.0" - -"@babel/helper-function-name@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.16.0.tgz#b7dd0797d00bbfee4f07e9c4ea5b0e30c8bb1481" - integrity sha512-BZh4mEk1xi2h4HFjWUXRQX5AEx4rvaZxHgax9gcjdLWdkjsY7MKt5p0otjsg5noXw+pB+clMCjw+aEVYADMjog== - dependencies: - "@babel/helper-get-function-arity" "^7.16.0" - "@babel/template" "^7.16.0" - "@babel/types" "^7.16.0" - -"@babel/helper-get-function-arity@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.16.0.tgz#0088c7486b29a9cb5d948b1a1de46db66e089cfa" - integrity sha512-ASCquNcywC1NkYh/z7Cgp3w31YW8aojjYIlNg4VeJiHkqyP4AzIvr4qx7pYDb4/s8YcsZWqqOSxgkvjUz1kpDQ== - dependencies: - "@babel/types" "^7.16.0" - -"@babel/helper-hoist-variables@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.16.0.tgz#4c9023c2f1def7e28ff46fc1dbcd36a39beaa81a" - integrity sha512-1AZlpazjUR0EQZQv3sgRNfM9mEVWPK3M6vlalczA+EECcPz3XPh6VplbErL5UoMpChhSck5wAJHthlj1bYpcmg== - dependencies: - "@babel/types" "^7.16.0" - -"@babel/helper-member-expression-to-functions@^7.13.12": - version "7.13.12" - resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.13.12.tgz#dfe368f26d426a07299d8d6513821768216e6d72" - integrity sha512-48ql1CLL59aKbU94Y88Xgb2VFy7a95ykGRbJJaaVv+LX5U8wFpLfiGXJJGUozsmA1oEh/o5Bp60Voq7ACyA/Sw== - dependencies: - "@babel/types" "^7.13.12" - -"@babel/helper-member-expression-to-functions@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.16.0.tgz#29287040efd197c77636ef75188e81da8bccd5a4" - integrity sha512-bsjlBFPuWT6IWhl28EdrQ+gTvSvj5tqVP5Xeftp07SEuz5pLnsXZuDkDD3Rfcxy0IsHmbZ+7B2/9SHzxO0T+sQ== - dependencies: - "@babel/types" "^7.16.0" - -"@babel/helper-module-imports@^7.12.13", "@babel/helper-module-imports@^7.13.12": - version "7.13.12" - resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.13.12.tgz#c6a369a6f3621cb25da014078684da9196b61977" - integrity sha512-4cVvR2/1B693IuOvSI20xqqa/+bl7lqAMR59R4iu39R9aOX8/JoYY1sFaNvUMyMBGnHdwvJgUrzNLoUZxXypxA== - dependencies: - "@babel/types" "^7.13.12" - -"@babel/helper-module-imports@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.16.0.tgz#90538e60b672ecf1b448f5f4f5433d37e79a3ec3" - integrity sha512-kkH7sWzKPq0xt3H1n+ghb4xEMP8k0U7XV3kkB+ZGy69kDk2ySFW1qPi06sjKzFY3t1j6XbJSqr4mF9L7CYVyhg== - dependencies: - "@babel/types" "^7.16.0" - -"@babel/helper-module-transforms@^7.12.1": - version "7.13.14" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.13.14.tgz#e600652ba48ccb1641775413cb32cfa4e8b495ef" - integrity sha512-QuU/OJ0iAOSIatyVZmfqB0lbkVP0kDRiKj34xy+QNsnVZi/PA6BoSoreeqnxxa9EHFAIL0R9XOaAR/G9WlIy5g== - dependencies: - "@babel/helper-module-imports" "^7.13.12" - "@babel/helper-replace-supers" "^7.13.12" - "@babel/helper-simple-access" "^7.13.12" - "@babel/helper-split-export-declaration" "^7.12.13" - "@babel/helper-validator-identifier" "^7.12.11" - "@babel/template" "^7.12.13" - "@babel/traverse" "^7.13.13" - "@babel/types" "^7.13.14" - -"@babel/helper-module-transforms@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.16.0.tgz#1c82a8dd4cb34577502ebd2909699b194c3e9bb5" - integrity sha512-My4cr9ATcaBbmaEa8M0dZNA74cfI6gitvUAskgDtAFmAqyFKDSHQo5YstxPbN+lzHl2D9l/YOEFqb2mtUh4gfA== - dependencies: - "@babel/helper-module-imports" "^7.16.0" - "@babel/helper-replace-supers" "^7.16.0" - "@babel/helper-simple-access" "^7.16.0" - "@babel/helper-split-export-declaration" "^7.16.0" - "@babel/helper-validator-identifier" "^7.15.7" - "@babel/template" "^7.16.0" - "@babel/traverse" "^7.16.0" - "@babel/types" "^7.16.0" - -"@babel/helper-optimise-call-expression@^7.12.13": - version "7.12.13" - resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.12.13.tgz#5c02d171b4c8615b1e7163f888c1c81c30a2aaea" - integrity sha512-BdWQhoVJkp6nVjB7nkFWcn43dkprYauqtk++Py2eaf/GRDFm5BxRqEIZCiHlZUGAVmtwKcsVL1dC68WmzeFmiA== - dependencies: - "@babel/types" "^7.12.13" - -"@babel/helper-optimise-call-expression@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.16.0.tgz#cecdb145d70c54096b1564f8e9f10cd7d193b338" - integrity sha512-SuI467Gi2V8fkofm2JPnZzB/SUuXoJA5zXe/xzyPP2M04686RzFKFHPK6HDVN6JvWBIEW8tt9hPR7fXdn2Lgpw== - dependencies: - "@babel/types" "^7.16.0" - -"@babel/helper-plugin-utils@7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz#2f75a831269d4f677de49986dff59927533cf375" - integrity sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg== - -"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.13.0", "@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3": - version "7.13.0" - resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.13.0.tgz#806526ce125aed03373bc416a828321e3a6a33af" - integrity sha512-ZPafIPSwzUlAoWT8DKs1W2VyF2gOWthGd5NGFMsBcMMol+ZhK+EQY/e6V96poa6PA/Bh+C9plWN0hXO1uB8AfQ== - -"@babel/helper-plugin-utils@^7.14.5": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.14.5.tgz#5ac822ce97eec46741ab70a517971e443a70c5a9" - integrity sha512-/37qQCE3K0vvZKwoK4XU/irIJQdIfCJuhU5eKnNxpFDsOkgFaUAwbv+RYw6eYgsC0E4hS7r5KqGULUogqui0fQ== - -"@babel/helper-remap-async-to-generator@^7.16.0", "@babel/helper-remap-async-to-generator@^7.16.4": - version "7.16.4" - resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.16.4.tgz#5d7902f61349ff6b963e07f06a389ce139fbfe6e" - integrity sha512-vGERmmhR+s7eH5Y/cp8PCVzj4XEjerq8jooMfxFdA5xVtAk9Sh4AQsrWgiErUEBjtGrBtOFKDUcWQFW4/dFwMA== - dependencies: - "@babel/helper-annotate-as-pure" "^7.16.0" - "@babel/helper-wrap-function" "^7.16.0" - "@babel/types" "^7.16.0" - -"@babel/helper-replace-supers@^7.13.12": - version "7.13.12" - resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.13.12.tgz#6442f4c1ad912502481a564a7386de0c77ff3804" - integrity sha512-Gz1eiX+4yDO8mT+heB94aLVNCL+rbuT2xy4YfyNqu8F+OI6vMvJK891qGBTqL9Uc8wxEvRW92Id6G7sDen3fFw== - dependencies: - "@babel/helper-member-expression-to-functions" "^7.13.12" - "@babel/helper-optimise-call-expression" "^7.12.13" - "@babel/traverse" "^7.13.0" - "@babel/types" "^7.13.12" - -"@babel/helper-replace-supers@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.16.0.tgz#73055e8d3cf9bcba8ddb55cad93fedc860f68f17" - integrity sha512-TQxuQfSCdoha7cpRNJvfaYxxxzmbxXw/+6cS7V02eeDYyhxderSoMVALvwupA54/pZcOTtVeJ0xccp1nGWladA== - dependencies: - "@babel/helper-member-expression-to-functions" "^7.16.0" - "@babel/helper-optimise-call-expression" "^7.16.0" - "@babel/traverse" "^7.16.0" - "@babel/types" "^7.16.0" - -"@babel/helper-simple-access@^7.13.12": - version "7.13.12" - resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.13.12.tgz#dd6c538afb61819d205a012c31792a39c7a5eaf6" - integrity sha512-7FEjbrx5SL9cWvXioDbnlYTppcZGuCY6ow3/D5vMggb2Ywgu4dMrpTJX0JdQAIcRRUElOIxF3yEooa9gUb9ZbA== - dependencies: - "@babel/types" "^7.13.12" - -"@babel/helper-simple-access@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.16.0.tgz#21d6a27620e383e37534cf6c10bba019a6f90517" - integrity sha512-o1rjBT/gppAqKsYfUdfHq5Rk03lMQrkPHG1OWzHWpLgVXRH4HnMM9Et9CVdIqwkCQlobnGHEJMsgWP/jE1zUiw== - dependencies: - "@babel/types" "^7.16.0" - -"@babel/helper-skip-transparent-expression-wrappers@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.16.0.tgz#0ee3388070147c3ae051e487eca3ebb0e2e8bb09" - integrity sha512-+il1gTy0oHwUsBQZyJvukbB4vPMdcYBrFHa0Uc4AizLxbq6BOYC51Rv4tWocX9BLBDLZ4kc6qUFpQ6HRgL+3zw== - dependencies: - "@babel/types" "^7.16.0" - -"@babel/helper-split-export-declaration@^7.12.13": - version "7.12.13" - resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.12.13.tgz#e9430be00baf3e88b0e13e6f9d4eaf2136372b05" - integrity sha512-tCJDltF83htUtXx5NLcaDqRmknv652ZWCHyoTETf1CXYJdPC7nohZohjUgieXhv0hTJdRf2FjDueFehdNucpzg== - dependencies: - "@babel/types" "^7.12.13" - -"@babel/helper-split-export-declaration@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.16.0.tgz#29672f43663e936df370aaeb22beddb3baec7438" - integrity sha512-0YMMRpuDFNGTHNRiiqJX19GjNXA4H0E8jZ2ibccfSxaCogbm3am5WN/2nQNj0YnQwGWM1J06GOcQ2qnh3+0paw== - dependencies: - "@babel/types" "^7.16.0" - -"@babel/helper-validator-identifier@^7.12.11": - version "7.12.11" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz#c9a1f021917dcb5ccf0d4e453e399022981fc9ed" - integrity sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw== - -"@babel/helper-validator-identifier@^7.15.7": - version "7.15.7" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.15.7.tgz#220df993bfe904a4a6b02ab4f3385a5ebf6e2389" - integrity sha512-K4JvCtQqad9OY2+yTU8w+E82ywk/fe+ELNlt1G8z3bVGlZfn/hOcQQsUhGhW/N+tb3fxK800wLtKOE/aM0m72w== - -"@babel/helper-validator-option@^7.12.17": - version "7.12.17" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.12.17.tgz#d1fbf012e1a79b7eebbfdc6d270baaf8d9eb9831" - integrity sha512-TopkMDmLzq8ngChwRlyjR6raKD6gMSae4JdYDB8bByKreQgG0RBTuKe9LRxW3wFtUnjxOPRKBDwEH6Mg5KeDfw== - -"@babel/helper-validator-option@^7.14.5": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.14.5.tgz#6e72a1fff18d5dfcb878e1e62f1a021c4b72d5a3" - integrity sha512-OX8D5eeX4XwcroVW45NMvoYaIuFI+GQpA2a8Gi+X/U/cDUIRsV37qQfF905F0htTRCREQIB4KqPeaveRJUl3Ow== - -"@babel/helper-wrap-function@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/helper-wrap-function/-/helper-wrap-function-7.16.0.tgz#b3cf318afce774dfe75b86767cd6d68f3482e57c" - integrity sha512-VVMGzYY3vkWgCJML+qVLvGIam902mJW0FvT7Avj1zEe0Gn7D93aWdLblYARTxEw+6DhZmtzhBM2zv0ekE5zg1g== - dependencies: - "@babel/helper-function-name" "^7.16.0" - "@babel/template" "^7.16.0" - "@babel/traverse" "^7.16.0" - "@babel/types" "^7.16.0" - -"@babel/helpers@^7.12.5": - version "7.13.17" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.13.17.tgz#b497c7a00e9719d5b613b8982bda6ed3ee94caf6" - integrity sha512-Eal4Gce4kGijo1/TGJdqp3WuhllaMLSrW6XcL0ulyUAQOuxHcCafZE8KHg9857gcTehsm/v7RcOx2+jp0Ryjsg== - dependencies: - "@babel/template" "^7.12.13" - "@babel/traverse" "^7.13.17" - "@babel/types" "^7.13.17" - -"@babel/helpers@^7.16.0": - version "7.16.3" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.16.3.tgz#27fc64f40b996e7074dc73128c3e5c3e7f55c43c" - integrity sha512-Xn8IhDlBPhvYTvgewPKawhADichOsbkZuzN7qz2BusOM0brChsyXMDJvldWaYMMUNiCQdQzNEioXTp3sC8Nt8w== - dependencies: - "@babel/template" "^7.16.0" - "@babel/traverse" "^7.16.3" - "@babel/types" "^7.16.0" - -"@babel/highlight@^7.10.4", "@babel/highlight@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.16.0.tgz#6ceb32b2ca4b8f5f361fb7fd821e3fddf4a1725a" - integrity sha512-t8MH41kUQylBtu2+4IQA3atqevA2lRgqA2wyVB/YiWmsDSuylZZuXOUy9ric30hfzauEFfdsuk/eXTRrGrfd0g== - dependencies: - "@babel/helper-validator-identifier" "^7.15.7" - chalk "^2.0.0" - js-tokens "^4.0.0" - -"@babel/highlight@^7.12.13": - version "7.13.10" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.13.10.tgz#a8b2a66148f5b27d666b15d81774347a731d52d1" - integrity sha512-5aPpe5XQPzflQrFwL1/QoeHkP2MsA4JCntcXHRhEsdsfPVkvPi2w7Qix4iV7t5S/oC9OodGrggd8aco1g3SZFg== - dependencies: - "@babel/helper-validator-identifier" "^7.12.11" - chalk "^2.0.0" - js-tokens "^4.0.0" - -"@babel/parser@^7.12.13", "@babel/parser@^7.12.16", "@babel/parser@^7.12.7": - version "7.13.16" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.13.16.tgz#0f18179b0448e6939b1f3f5c4c355a3a9bcdfd37" - integrity sha512-6bAg36mCwuqLO0hbR+z7PHuqWiCeP7Dzg73OpQwsAB1Eb8HnGEz5xYBzCfbu+YjoaJsJs+qheDxVAuqbt3ILEw== - -"@babel/parser@^7.16.0", "@babel/parser@^7.16.3": - version "7.16.4" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.16.4.tgz#d5f92f57cf2c74ffe9b37981c0e72fee7311372e" - integrity sha512-6V0qdPUaiVHH3RtZeLIsc+6pDhbYzHR8ogA8w+f+Wc77DuXto19g2QUwveINoS34Uw+W8/hQDGJCx+i4n7xcng== - -"@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.16.2": - version "7.16.2" - resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.16.2.tgz#2977fca9b212db153c195674e57cfab807733183" - integrity sha512-h37CvpLSf8gb2lIJ2CgC3t+EjFbi0t8qS7LCS1xcJIlEXE4czlofwaW7W1HA8zpgOCzI9C1nmoqNR1zWkk0pQg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.16.0.tgz#358972eaab006f5eb0826183b0c93cbcaf13e1e2" - integrity sha512-4tcFwwicpWTrpl9qjf7UsoosaArgImF85AxqCRZlgc3IQDvkUHjJpruXAL58Wmj+T6fypWTC/BakfEkwIL/pwA== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-skip-transparent-expression-wrappers" "^7.16.0" - "@babel/plugin-proposal-optional-chaining" "^7.16.0" - -"@babel/plugin-proposal-async-generator-functions@^7.16.4": - version "7.16.4" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.16.4.tgz#e606eb6015fec6fa5978c940f315eae4e300b081" - integrity sha512-/CUekqaAaZCQHleSK/9HajvcD/zdnJiKRiuUFq8ITE+0HsPzquf53cpFiqAwl/UfmJbR6n5uGPQSPdrmKOvHHg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-remap-async-to-generator" "^7.16.4" - "@babel/plugin-syntax-async-generators" "^7.8.4" - -"@babel/plugin-proposal-class-properties@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.16.0.tgz#c029618267ddebc7280fa286e0f8ca2a278a2d1a" - integrity sha512-mCF3HcuZSY9Fcx56Lbn+CGdT44ioBMMvjNVldpKtj8tpniETdLjnxdHI1+sDWXIM1nNt+EanJOZ3IG9lzVjs7A== - dependencies: - "@babel/helper-create-class-features-plugin" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-proposal-class-static-block@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.16.0.tgz#5296942c564d8144c83eea347d0aa8a0b89170e7" - integrity sha512-mAy3sdcY9sKAkf3lQbDiv3olOfiLqI51c9DR9b19uMoR2Z6r5pmGl7dfNFqEvqOyqbf1ta4lknK4gc5PJn3mfA== - dependencies: - "@babel/helper-create-class-features-plugin" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-class-static-block" "^7.14.5" - -"@babel/plugin-proposal-dynamic-import@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.16.0.tgz#783eca61d50526202f9b296095453977e88659f1" - integrity sha512-QGSA6ExWk95jFQgwz5GQ2Dr95cf7eI7TKutIXXTb7B1gCLTCz5hTjFTQGfLFBBiC5WSNi7udNwWsqbbMh1c4yQ== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-dynamic-import" "^7.8.3" - -"@babel/plugin-proposal-export-namespace-from@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.16.0.tgz#9c01dee40b9d6b847b656aaf4a3976a71740f222" - integrity sha512-CjI4nxM/D+5wCnhD11MHB1AwRSAYeDT+h8gCdcVJZ/OK7+wRzFsf7PFPWVpVpNRkHMmMkQWAHpTq+15IXQ1diA== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-export-namespace-from" "^7.8.3" - -"@babel/plugin-proposal-json-strings@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.16.0.tgz#cae35a95ed1d2a7fa29c4dc41540b84a72e9ab25" - integrity sha512-kouIPuiv8mSi5JkEhzApg5Gn6hFyKPnlkO0a9YSzqRurH8wYzSlf6RJdzluAsbqecdW5pBvDJDfyDIUR/vLxvg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-json-strings" "^7.8.3" - -"@babel/plugin-proposal-logical-assignment-operators@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.16.0.tgz#a711b8ceb3ffddd3ef88d3a49e86dbd3cc7db3fd" - integrity sha512-pbW0fE30sVTYXXm9lpVQQ/Vc+iTeQKiXlaNRZPPN2A2VdlWyAtsUrsQ3xydSlDW00TFMK7a8m3cDTkBF5WnV3Q== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4" - -"@babel/plugin-proposal-nullish-coalescing-operator@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.16.0.tgz#44e1cce08fe2427482cf446a91bb451528ed0596" - integrity sha512-3bnHA8CAFm7cG93v8loghDYyQ8r97Qydf63BeYiGgYbjKKB/XP53W15wfRC7dvKfoiJ34f6Rbyyx2btExc8XsQ== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3" - -"@babel/plugin-proposal-numeric-separator@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.16.0.tgz#5d418e4fbbf8b9b7d03125d3a52730433a373734" - integrity sha512-FAhE2I6mjispy+vwwd6xWPyEx3NYFS13pikDBWUAFGZvq6POGs5eNchw8+1CYoEgBl9n11I3NkzD7ghn25PQ9Q== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-numeric-separator" "^7.10.4" - -"@babel/plugin-proposal-object-rest-spread@7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.12.1.tgz#def9bd03cea0f9b72283dac0ec22d289c7691069" - integrity sha512-s6SowJIjzlhx8o7lsFx5zmY4At6CTtDvgNQDdPzkBQucle58A6b/TTeEBYtyDgmcXjUTM+vE8YOGHZzzbc/ioA== - dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - "@babel/plugin-syntax-object-rest-spread" "^7.8.0" - "@babel/plugin-transform-parameters" "^7.12.1" - -"@babel/plugin-proposal-object-rest-spread@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.16.0.tgz#5fb32f6d924d6e6712810362a60e12a2609872e6" - integrity sha512-LU/+jp89efe5HuWJLmMmFG0+xbz+I2rSI7iLc1AlaeSMDMOGzWlc5yJrMN1d04osXN4sSfpo4O+azkBNBes0jg== - dependencies: - "@babel/compat-data" "^7.16.0" - "@babel/helper-compilation-targets" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-object-rest-spread" "^7.8.3" - "@babel/plugin-transform-parameters" "^7.16.0" - -"@babel/plugin-proposal-optional-catch-binding@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.16.0.tgz#5910085811ab4c28b00d6ebffa4ab0274d1e5f16" - integrity sha512-kicDo0A/5J0nrsCPbn89mTG3Bm4XgYi0CZtvex9Oyw7gGZE3HXGD0zpQNH+mo+tEfbo8wbmMvJftOwpmPy7aVw== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-optional-catch-binding" "^7.8.3" - -"@babel/plugin-proposal-optional-chaining@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.16.0.tgz#56dbc3970825683608e9efb55ea82c2a2d6c8dc0" - integrity sha512-Y4rFpkZODfHrVo70Uaj6cC1JJOt3Pp0MdWSwIKtb8z1/lsjl9AmnB7ErRFV+QNGIfcY1Eruc2UMx5KaRnXjMyg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-skip-transparent-expression-wrappers" "^7.16.0" - "@babel/plugin-syntax-optional-chaining" "^7.8.3" - -"@babel/plugin-proposal-private-methods@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.16.0.tgz#b4dafb9c717e4301c5776b30d080d6383c89aff6" - integrity sha512-IvHmcTHDFztQGnn6aWq4t12QaBXTKr1whF/dgp9kz84X6GUcwq9utj7z2wFCUfeOup/QKnOlt2k0zxkGFx9ubg== - dependencies: - "@babel/helper-create-class-features-plugin" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-proposal-private-property-in-object@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.16.0.tgz#69e935b2c5c79d2488112d886f0c4e2790fee76f" - integrity sha512-3jQUr/HBbMVZmi72LpjQwlZ55i1queL8KcDTQEkAHihttJnAPrcvG9ZNXIfsd2ugpizZo595egYV6xy+pv4Ofw== - dependencies: - "@babel/helper-annotate-as-pure" "^7.16.0" - "@babel/helper-create-class-features-plugin" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-private-property-in-object" "^7.14.5" - -"@babel/plugin-proposal-unicode-property-regex@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.16.0.tgz#890482dfc5ea378e42e19a71e709728cabf18612" - integrity sha512-ti7IdM54NXv29cA4+bNNKEMS4jLMCbJgl+Drv+FgYy0erJLAxNAIXcNjNjrRZEcWq0xJHsNVwQezskMFpF8N9g== - dependencies: - "@babel/helper-create-regexp-features-plugin" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-proposal-unicode-property-regex@^7.4.4": - version "7.12.13" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.12.13.tgz#bebde51339be829c17aaaaced18641deb62b39ba" - integrity sha512-XyJmZidNfofEkqFV5VC/bLabGmO5QzenPO/YOfGuEbgU+2sSwMmio3YLb4WtBgcmmdwZHyVyv8on77IUjQ5Gvg== - dependencies: - "@babel/helper-create-regexp-features-plugin" "^7.12.13" - "@babel/helper-plugin-utils" "^7.12.13" - -"@babel/plugin-syntax-async-generators@^7.8.4": - version "7.8.4" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz#a983fb1aeb2ec3f6ed042a210f640e90e786fe0d" - integrity sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw== - dependencies: - "@babel/helper-plugin-utils" "^7.8.0" - -"@babel/plugin-syntax-class-properties@^7.12.13": - version "7.12.13" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz#b5c987274c4a3a82b89714796931a6b53544ae10" - integrity sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA== - dependencies: - "@babel/helper-plugin-utils" "^7.12.13" - -"@babel/plugin-syntax-class-static-block@^7.14.5": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz#195df89b146b4b78b3bf897fd7a257c84659d406" - integrity sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-syntax-dynamic-import@^7.8.3": - version "7.8.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz#62bf98b2da3cd21d626154fc96ee5b3cb68eacb3" - integrity sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ== - dependencies: - "@babel/helper-plugin-utils" "^7.8.0" - -"@babel/plugin-syntax-export-namespace-from@^7.8.3": - version "7.8.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz#028964a9ba80dbc094c915c487ad7c4e7a66465a" - integrity sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q== - dependencies: - "@babel/helper-plugin-utils" "^7.8.3" - -"@babel/plugin-syntax-json-strings@^7.8.3": - version "7.8.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz#01ca21b668cd8218c9e640cb6dd88c5412b2c96a" - integrity sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA== - dependencies: - "@babel/helper-plugin-utils" "^7.8.0" - -"@babel/plugin-syntax-jsx@7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.12.1.tgz#9d9d357cc818aa7ae7935917c1257f67677a0926" - integrity sha512-1yRi7yAtB0ETgxdY9ti/p2TivUxJkTdhu/ZbF9MshVGqOx1TdB3b7xCXs49Fupgg50N45KcAsRP/ZqWjs9SRjg== - dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - -"@babel/plugin-syntax-jsx@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.16.0.tgz#f9624394317365a9a88c82358d3f8471154698f1" - integrity sha512-8zv2+xiPHwly31RK4RmnEYY5zziuF3O7W2kIDW+07ewWDh6Oi0dRq8kwvulRkFgt6DB97RlKs5c1y068iPlCUg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-syntax-logical-assignment-operators@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz#ca91ef46303530448b906652bac2e9fe9941f699" - integrity sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig== - dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - -"@babel/plugin-syntax-nullish-coalescing-operator@^7.8.3": - version "7.8.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz#167ed70368886081f74b5c36c65a88c03b66d1a9" - integrity sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ== - dependencies: - "@babel/helper-plugin-utils" "^7.8.0" - -"@babel/plugin-syntax-numeric-separator@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz#b9b070b3e33570cd9fd07ba7fa91c0dd37b9af97" - integrity sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug== - dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - -"@babel/plugin-syntax-object-rest-spread@7.8.3", "@babel/plugin-syntax-object-rest-spread@^7.8.0", "@babel/plugin-syntax-object-rest-spread@^7.8.3": - version "7.8.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz#60e225edcbd98a640332a2e72dd3e66f1af55871" - integrity sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA== - dependencies: - "@babel/helper-plugin-utils" "^7.8.0" - -"@babel/plugin-syntax-optional-catch-binding@^7.8.3": - version "7.8.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz#6111a265bcfb020eb9efd0fdfd7d26402b9ed6c1" - integrity sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q== - dependencies: - "@babel/helper-plugin-utils" "^7.8.0" - -"@babel/plugin-syntax-optional-chaining@^7.8.3": - version "7.8.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz#4f69c2ab95167e0180cd5336613f8c5788f7d48a" - integrity sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg== - dependencies: - "@babel/helper-plugin-utils" "^7.8.0" - -"@babel/plugin-syntax-private-property-in-object@^7.14.5": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz#0dc6671ec0ea22b6e94a1114f857970cd39de1ad" - integrity sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-syntax-top-level-await@^7.14.5": - version "7.14.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz#c1cfdadc35a646240001f06138247b741c34d94c" - integrity sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-syntax-typescript@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.16.0.tgz#2feeb13d9334cc582ea9111d3506f773174179bb" - integrity sha512-Xv6mEXqVdaqCBfJFyeab0fH2DnUoMsDmhamxsSi4j8nLd4Vtw213WMJr55xxqipC/YVWyPY3K0blJncPYji+dQ== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-arrow-functions@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.16.0.tgz#951706f8b449c834ed07bd474c0924c944b95a8e" - integrity sha512-vIFb5250Rbh7roWARvCLvIJ/PtAU5Lhv7BtZ1u24COwpI9Ypjsh+bZcKk6rlIyalK+r0jOc1XQ8I4ovNxNrWrA== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-async-to-generator@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.16.0.tgz#df12637f9630ddfa0ef9d7a11bc414d629d38604" - integrity sha512-PbIr7G9kR8tdH6g8Wouir5uVjklETk91GMVSUq+VaOgiinbCkBP6Q7NN/suM/QutZkMJMvcyAriogcYAdhg8Gw== - dependencies: - "@babel/helper-module-imports" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-remap-async-to-generator" "^7.16.0" - -"@babel/plugin-transform-block-scoped-functions@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.16.0.tgz#c618763233ad02847805abcac4c345ce9de7145d" - integrity sha512-V14As3haUOP4ZWrLJ3VVx5rCnrYhMSHN/jX7z6FAt5hjRkLsb0snPCmJwSOML5oxkKO4FNoNv7V5hw/y2bjuvg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-block-scoping@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.16.0.tgz#bcf433fb482fe8c3d3b4e8a66b1c4a8e77d37c16" - integrity sha512-27n3l67/R3UrXfizlvHGuTwsRIFyce3D/6a37GRxn28iyTPvNXaW4XvznexRh1zUNLPjbLL22Id0XQElV94ruw== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-classes@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-classes/-/plugin-transform-classes-7.16.0.tgz#54cf5ff0b2242c6573d753cd4bfc7077a8b282f5" - integrity sha512-HUxMvy6GtAdd+GKBNYDWCIA776byUQH8zjnfjxwT1P1ARv/wFu8eBDpmXQcLS/IwRtrxIReGiplOwMeyO7nsDQ== - dependencies: - "@babel/helper-annotate-as-pure" "^7.16.0" - "@babel/helper-function-name" "^7.16.0" - "@babel/helper-optimise-call-expression" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-replace-supers" "^7.16.0" - "@babel/helper-split-export-declaration" "^7.16.0" - globals "^11.1.0" - -"@babel/plugin-transform-computed-properties@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.16.0.tgz#e0c385507d21e1b0b076d66bed6d5231b85110b7" - integrity sha512-63l1dRXday6S8V3WFY5mXJwcRAnPYxvFfTlt67bwV1rTyVTM5zrp0DBBb13Kl7+ehkCVwIZPumPpFP/4u70+Tw== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-destructuring@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.16.0.tgz#ad3d7e74584ad5ea4eadb1e6642146c590dee33c" - integrity sha512-Q7tBUwjxLTsHEoqktemHBMtb3NYwyJPTJdM+wDwb0g8PZ3kQUIzNvwD5lPaqW/p54TXBc/MXZu9Jr7tbUEUM8Q== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-dotall-regex@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.16.0.tgz#50bab00c1084b6162d0a58a818031cf57798e06f" - integrity sha512-FXlDZfQeLILfJlC6I1qyEwcHK5UpRCFkaoVyA1nk9A1L1Yu583YO4un2KsLBsu3IJb4CUbctZks8tD9xPQubLw== - dependencies: - "@babel/helper-create-regexp-features-plugin" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-dotall-regex@^7.4.4": - version "7.12.13" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.12.13.tgz#3f1601cc29905bfcb67f53910f197aeafebb25ad" - integrity sha512-foDrozE65ZFdUC2OfgeOCrEPTxdB3yjqxpXh8CH+ipd9CHd4s/iq81kcUpyH8ACGNEPdFqbtzfgzbT/ZGlbDeQ== - dependencies: - "@babel/helper-create-regexp-features-plugin" "^7.12.13" - "@babel/helper-plugin-utils" "^7.12.13" - -"@babel/plugin-transform-duplicate-keys@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.16.0.tgz#8bc2e21813e3e89e5e5bf3b60aa5fc458575a176" - integrity sha512-LIe2kcHKAZOJDNxujvmp6z3mfN6V9lJxubU4fJIGoQCkKe3Ec2OcbdlYP+vW++4MpxwG0d1wSDOJtQW5kLnkZQ== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-exponentiation-operator@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.16.0.tgz#a180cd2881e3533cef9d3901e48dad0fbeff4be4" - integrity sha512-OwYEvzFI38hXklsrbNivzpO3fh87skzx8Pnqi4LoSYeav0xHlueSoCJrSgTPfnbyzopo5b3YVAJkFIcUpK2wsw== - dependencies: - "@babel/helper-builder-binary-assignment-operator-visitor" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-for-of@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.16.0.tgz#f7abaced155260e2461359bbc7c7248aca5e6bd2" - integrity sha512-5QKUw2kO+GVmKr2wMYSATCTTnHyscl6sxFRAY+rvN7h7WB0lcG0o4NoV6ZQU32OZGVsYUsfLGgPQpDFdkfjlJQ== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-function-name@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.16.0.tgz#02e3699c284c6262236599f751065c5d5f1f400e" - integrity sha512-lBzMle9jcOXtSOXUpc7tvvTpENu/NuekNJVova5lCCWCV9/U1ho2HH2y0p6mBg8fPm/syEAbfaaemYGOHCY3mg== - dependencies: - "@babel/helper-function-name" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-literals@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-literals/-/plugin-transform-literals-7.16.0.tgz#79711e670ffceb31bd298229d50f3621f7980cac" - integrity sha512-gQDlsSF1iv9RU04clgXqRjrPyyoJMTclFt3K1cjLmTKikc0s/6vE3hlDeEVC71wLTRu72Fq7650kABrdTc2wMQ== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-member-expression-literals@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.16.0.tgz#5251b4cce01eaf8314403d21aedb269d79f5e64b" - integrity sha512-WRpw5HL4Jhnxw8QARzRvwojp9MIE7Tdk3ez6vRyUk1MwgjJN0aNpRoXainLR5SgxmoXx/vsXGZ6OthP6t/RbUg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-modules-amd@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.16.0.tgz#09abd41e18dcf4fd479c598c1cef7bd39eb1337e" - integrity sha512-rWFhWbCJ9Wdmzln1NmSCqn7P0RAD+ogXG/bd9Kg5c7PKWkJtkiXmYsMBeXjDlzHpVTJ4I/hnjs45zX4dEv81xw== - dependencies: - "@babel/helper-module-transforms" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - babel-plugin-dynamic-import-node "^2.3.3" - -"@babel/plugin-transform-modules-commonjs@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.16.0.tgz#add58e638c8ddc4875bd9a9ecb5c594613f6c922" - integrity sha512-Dzi+NWqyEotgzk/sb7kgQPJQf7AJkQBWsVp1N6JWc1lBVo0vkElUnGdr1PzUBmfsCCN5OOFya3RtpeHk15oLKQ== - dependencies: - "@babel/helper-module-transforms" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-simple-access" "^7.16.0" - babel-plugin-dynamic-import-node "^2.3.3" - -"@babel/plugin-transform-modules-systemjs@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.16.0.tgz#a92cf240afeb605f4ca16670453024425e421ea4" - integrity sha512-yuGBaHS3lF1m/5R+6fjIke64ii5luRUg97N2wr+z1sF0V+sNSXPxXDdEEL/iYLszsN5VKxVB1IPfEqhzVpiqvg== - dependencies: - "@babel/helper-hoist-variables" "^7.16.0" - "@babel/helper-module-transforms" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-validator-identifier" "^7.15.7" - babel-plugin-dynamic-import-node "^2.3.3" - -"@babel/plugin-transform-modules-umd@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.16.0.tgz#195f26c2ad6d6a391b70880effce18ce625e06a7" - integrity sha512-nx4f6no57himWiHhxDM5pjwhae5vLpTK2zCnDH8+wNLJy0TVER/LJRHl2bkt6w9Aad2sPD5iNNoUpY3X9sTGDg== - dependencies: - "@babel/helper-module-transforms" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-named-capturing-groups-regex@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.16.0.tgz#d3db61cc5d5b97986559967cd5ea83e5c32096ca" - integrity sha512-LogN88uO+7EhxWc8WZuQ8vxdSyVGxhkh8WTC3tzlT8LccMuQdA81e9SGV6zY7kY2LjDhhDOFdQVxdGwPyBCnvg== - dependencies: - "@babel/helper-create-regexp-features-plugin" "^7.16.0" - -"@babel/plugin-transform-new-target@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.16.0.tgz#af823ab576f752215a49937779a41ca65825ab35" - integrity sha512-fhjrDEYv2DBsGN/P6rlqakwRwIp7rBGLPbrKxwh7oVt5NNkIhZVOY2GRV+ULLsQri1bDqwDWnU3vhlmx5B2aCw== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-object-super@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.16.0.tgz#fb20d5806dc6491a06296ac14ea8e8d6fedda72b" - integrity sha512-fds+puedQHn4cPLshoHcR1DTMN0q1V9ou0mUjm8whx9pGcNvDrVVrgw+KJzzCaiTdaYhldtrUps8DWVMgrSEyg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-replace-supers" "^7.16.0" - -"@babel/plugin-transform-parameters@^7.12.1": - version "7.13.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.13.0.tgz#8fa7603e3097f9c0b7ca1a4821bc2fb52e9e5007" - integrity sha512-Jt8k/h/mIwE2JFEOb3lURoY5C85ETcYPnbuAJ96zRBzh1XHtQZfs62ChZ6EP22QlC8c7Xqr9q+e1SU5qttwwjw== - dependencies: - "@babel/helper-plugin-utils" "^7.13.0" - -"@babel/plugin-transform-parameters@^7.16.0", "@babel/plugin-transform-parameters@^7.16.3": - version "7.16.3" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.16.3.tgz#fa9e4c874ee5223f891ee6fa8d737f4766d31d15" - integrity sha512-3MaDpJrOXT1MZ/WCmkOFo7EtmVVC8H4EUZVrHvFOsmwkk4lOjQj8rzv8JKUZV4YoQKeoIgk07GO+acPU9IMu/w== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-property-literals@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.16.0.tgz#a95c552189a96a00059f6776dc4e00e3690c78d1" - integrity sha512-XLldD4V8+pOqX2hwfWhgwXzGdnDOThxaNTgqagOcpBgIxbUvpgU2FMvo5E1RyHbk756WYgdbS0T8y0Cj9FKkWQ== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-react-constant-elements@^7.12.1": - version "7.13.13" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-constant-elements/-/plugin-transform-react-constant-elements-7.13.13.tgz#0208b1d942bf939cd4f7aa5b255d42602aa4a920" - integrity sha512-SNJU53VM/SjQL0bZhyU+f4kJQz7bQQajnrZRSaU21hruG/NWY41AEM9AWXeXX90pYr/C2yAmTgI6yW3LlLrAUQ== - dependencies: - "@babel/helper-plugin-utils" "^7.13.0" - -"@babel/plugin-transform-react-display-name@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.16.0.tgz#9a0ad8aa8e8790883a7bd2736f66229a58125676" - integrity sha512-FJFdJAqaCpndL+pIf0aeD/qlQwT7QXOvR6Cc8JPvNhKJBi2zc/DPc4g05Y3fbD/0iWAMQFGij4+Xw+4L/BMpTg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-react-jsx-development@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-development/-/plugin-transform-react-jsx-development-7.16.0.tgz#1cb52874678d23ab11d0d16488d54730807303ef" - integrity sha512-qq65iSqBRq0Hr3wq57YG2AmW0H6wgTnIzpffTphrUWUgLCOK+zf1f7G0vuOiXrp7dU1qq+fQBoqZ3wCDAkhFzw== - dependencies: - "@babel/plugin-transform-react-jsx" "^7.16.0" - -"@babel/plugin-transform-react-jsx@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.16.0.tgz#55b797d4960c3de04e07ad1c0476e2bc6a4889f1" - integrity sha512-rqDgIbukZ44pqq7NIRPGPGNklshPkvlmvqjdx3OZcGPk4zGIenYkxDTvl3LsSL8gqcc3ZzGmXPE6hR/u/voNOw== - dependencies: - "@babel/helper-annotate-as-pure" "^7.16.0" - "@babel/helper-module-imports" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-jsx" "^7.16.0" - "@babel/types" "^7.16.0" - -"@babel/plugin-transform-react-pure-annotations@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-pure-annotations/-/plugin-transform-react-pure-annotations-7.16.0.tgz#23db6ddf558d8abde41b8ad9d59f48ad5532ccab" - integrity sha512-NC/Bj2MG+t8Ef5Pdpo34Ay74X4Rt804h5y81PwOpfPtmAK3i6CizmQqwyBQzIepz1Yt8wNr2Z2L7Lu3qBMfZMA== - dependencies: - "@babel/helper-annotate-as-pure" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-regenerator@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.16.0.tgz#eaee422c84b0232d03aea7db99c97deeaf6125a4" - integrity sha512-JAvGxgKuwS2PihiSFaDrp94XOzzTUeDeOQlcKzVAyaPap7BnZXK/lvMDiubkPTdotPKOIZq9xWXWnggUMYiExg== - dependencies: - regenerator-transform "^0.14.2" - -"@babel/plugin-transform-reserved-words@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.16.0.tgz#fff4b9dcb19e12619394bda172d14f2d04c0379c" - integrity sha512-Dgs8NNCehHSvXdhEhln8u/TtJxfVwGYCgP2OOr5Z3Ar+B+zXicEOKNTyc+eca2cuEOMtjW6m9P9ijOt8QdqWkg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-runtime@^7.12.15": - version "7.16.4" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.16.4.tgz#f9ba3c7034d429c581e1bd41b4952f3db3c2c7e8" - integrity sha512-pru6+yHANMTukMtEZGC4fs7XPwg35v8sj5CIEmE+gEkFljFiVJxEWxx/7ZDkTK+iZRYo1bFXBtfIN95+K3cJ5A== - dependencies: - "@babel/helper-module-imports" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - babel-plugin-polyfill-corejs2 "^0.3.0" - babel-plugin-polyfill-corejs3 "^0.4.0" - babel-plugin-polyfill-regenerator "^0.3.0" - semver "^6.3.0" - -"@babel/plugin-transform-shorthand-properties@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.16.0.tgz#090372e3141f7cc324ed70b3daf5379df2fa384d" - integrity sha512-iVb1mTcD8fuhSv3k99+5tlXu5N0v8/DPm2mO3WACLG6al1CGZH7v09HJyUb1TtYl/Z+KrM6pHSIJdZxP5A+xow== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-spread@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-spread/-/plugin-transform-spread-7.16.0.tgz#d21ca099bbd53ab307a8621e019a7bd0f40cdcfb" - integrity sha512-Ao4MSYRaLAQczZVp9/7E7QHsCuK92yHRrmVNRe/SlEJjhzivq0BSn8mEraimL8wizHZ3fuaHxKH0iwzI13GyGg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-skip-transparent-expression-wrappers" "^7.16.0" - -"@babel/plugin-transform-sticky-regex@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.16.0.tgz#c35ea31a02d86be485f6aa510184b677a91738fd" - integrity sha512-/ntT2NljR9foobKk4E/YyOSwcGUXtYWv5tinMK/3RkypyNBNdhHUaq6Orw5DWq9ZcNlS03BIlEALFeQgeVAo4Q== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-template-literals@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.16.0.tgz#a8eced3a8e7b8e2d40ec4ec4548a45912630d302" - integrity sha512-Rd4Ic89hA/f7xUSJQk5PnC+4so50vBoBfxjdQAdvngwidM8jYIBVxBZ/sARxD4e0yMXRbJVDrYf7dyRtIIKT6Q== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-typeof-symbol@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.16.0.tgz#8b19a244c6f8c9d668dca6a6f754ad6ead1128f2" - integrity sha512-++V2L8Bdf4vcaHi2raILnptTBjGEFxn5315YU+e8+EqXIucA+q349qWngCLpUYqqv233suJ6NOienIVUpS9cqg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-typescript@^7.16.0": - version "7.16.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.16.1.tgz#cc0670b2822b0338355bc1b3d2246a42b8166409" - integrity sha512-NO4XoryBng06jjw/qWEU2LhcLJr1tWkhpMam/H4eas/CDKMX/b2/Ylb6EI256Y7+FVPCawwSM1rrJNOpDiz+Lg== - dependencies: - "@babel/helper-create-class-features-plugin" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/plugin-syntax-typescript" "^7.16.0" - -"@babel/plugin-transform-unicode-escapes@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.16.0.tgz#1a354064b4c45663a32334f46fa0cf6100b5b1f3" - integrity sha512-VFi4dhgJM7Bpk8lRc5CMaRGlKZ29W9C3geZjt9beuzSUrlJxsNwX7ReLwaL6WEvsOf2EQkyIJEPtF8EXjB/g2A== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/plugin-transform-unicode-regex@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.16.0.tgz#293b80950177c8c85aede87cef280259fb995402" - integrity sha512-jHLK4LxhHjvCeZDWyA9c+P9XH1sOxRd1RO9xMtDVRAOND/PczPqizEtVdx4TQF/wyPaewqpT+tgQFYMnN/P94A== - dependencies: - "@babel/helper-create-regexp-features-plugin" "^7.16.0" - "@babel/helper-plugin-utils" "^7.14.5" - -"@babel/preset-env@^7.12.1", "@babel/preset-env@^7.12.16": - version "7.16.4" - resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.16.4.tgz#4f6ec33b2a3fe72d6bfdcdf3859500232563a2e3" - integrity sha512-v0QtNd81v/xKj4gNKeuAerQ/azeNn/G1B1qMLeXOcV8+4TWlD2j3NV1u8q29SDFBXx/NBq5kyEAO+0mpRgacjA== - dependencies: - "@babel/compat-data" "^7.16.4" - "@babel/helper-compilation-targets" "^7.16.3" - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-validator-option" "^7.14.5" - "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression" "^7.16.2" - "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining" "^7.16.0" - "@babel/plugin-proposal-async-generator-functions" "^7.16.4" - "@babel/plugin-proposal-class-properties" "^7.16.0" - "@babel/plugin-proposal-class-static-block" "^7.16.0" - "@babel/plugin-proposal-dynamic-import" "^7.16.0" - "@babel/plugin-proposal-export-namespace-from" "^7.16.0" - "@babel/plugin-proposal-json-strings" "^7.16.0" - "@babel/plugin-proposal-logical-assignment-operators" "^7.16.0" - "@babel/plugin-proposal-nullish-coalescing-operator" "^7.16.0" - "@babel/plugin-proposal-numeric-separator" "^7.16.0" - "@babel/plugin-proposal-object-rest-spread" "^7.16.0" - "@babel/plugin-proposal-optional-catch-binding" "^7.16.0" - "@babel/plugin-proposal-optional-chaining" "^7.16.0" - "@babel/plugin-proposal-private-methods" "^7.16.0" - "@babel/plugin-proposal-private-property-in-object" "^7.16.0" - "@babel/plugin-proposal-unicode-property-regex" "^7.16.0" - "@babel/plugin-syntax-async-generators" "^7.8.4" - "@babel/plugin-syntax-class-properties" "^7.12.13" - "@babel/plugin-syntax-class-static-block" "^7.14.5" - "@babel/plugin-syntax-dynamic-import" "^7.8.3" - "@babel/plugin-syntax-export-namespace-from" "^7.8.3" - "@babel/plugin-syntax-json-strings" "^7.8.3" - "@babel/plugin-syntax-logical-assignment-operators" "^7.10.4" - "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3" - "@babel/plugin-syntax-numeric-separator" "^7.10.4" - "@babel/plugin-syntax-object-rest-spread" "^7.8.3" - "@babel/plugin-syntax-optional-catch-binding" "^7.8.3" - "@babel/plugin-syntax-optional-chaining" "^7.8.3" - "@babel/plugin-syntax-private-property-in-object" "^7.14.5" - "@babel/plugin-syntax-top-level-await" "^7.14.5" - "@babel/plugin-transform-arrow-functions" "^7.16.0" - "@babel/plugin-transform-async-to-generator" "^7.16.0" - "@babel/plugin-transform-block-scoped-functions" "^7.16.0" - "@babel/plugin-transform-block-scoping" "^7.16.0" - "@babel/plugin-transform-classes" "^7.16.0" - "@babel/plugin-transform-computed-properties" "^7.16.0" - "@babel/plugin-transform-destructuring" "^7.16.0" - "@babel/plugin-transform-dotall-regex" "^7.16.0" - "@babel/plugin-transform-duplicate-keys" "^7.16.0" - "@babel/plugin-transform-exponentiation-operator" "^7.16.0" - "@babel/plugin-transform-for-of" "^7.16.0" - "@babel/plugin-transform-function-name" "^7.16.0" - "@babel/plugin-transform-literals" "^7.16.0" - "@babel/plugin-transform-member-expression-literals" "^7.16.0" - "@babel/plugin-transform-modules-amd" "^7.16.0" - "@babel/plugin-transform-modules-commonjs" "^7.16.0" - "@babel/plugin-transform-modules-systemjs" "^7.16.0" - "@babel/plugin-transform-modules-umd" "^7.16.0" - "@babel/plugin-transform-named-capturing-groups-regex" "^7.16.0" - "@babel/plugin-transform-new-target" "^7.16.0" - "@babel/plugin-transform-object-super" "^7.16.0" - "@babel/plugin-transform-parameters" "^7.16.3" - "@babel/plugin-transform-property-literals" "^7.16.0" - "@babel/plugin-transform-regenerator" "^7.16.0" - "@babel/plugin-transform-reserved-words" "^7.16.0" - "@babel/plugin-transform-shorthand-properties" "^7.16.0" - "@babel/plugin-transform-spread" "^7.16.0" - "@babel/plugin-transform-sticky-regex" "^7.16.0" - "@babel/plugin-transform-template-literals" "^7.16.0" - "@babel/plugin-transform-typeof-symbol" "^7.16.0" - "@babel/plugin-transform-unicode-escapes" "^7.16.0" - "@babel/plugin-transform-unicode-regex" "^7.16.0" - "@babel/preset-modules" "^0.1.5" - "@babel/types" "^7.16.0" - babel-plugin-polyfill-corejs2 "^0.3.0" - babel-plugin-polyfill-corejs3 "^0.4.0" - babel-plugin-polyfill-regenerator "^0.3.0" - core-js-compat "^3.19.1" - semver "^6.3.0" - -"@babel/preset-modules@^0.1.5": - version "0.1.5" - resolved "https://registry.yarnpkg.com/@babel/preset-modules/-/preset-modules-0.1.5.tgz#ef939d6e7f268827e1841638dc6ff95515e115d9" - integrity sha512-A57th6YRG7oR3cq/yt/Y84MvGgE0eJG2F1JLhKuyG+jFxEgrd/HAMJatiFtmOiZurz+0DkrvbheCLaV5f2JfjA== - dependencies: - "@babel/helper-plugin-utils" "^7.0.0" - "@babel/plugin-proposal-unicode-property-regex" "^7.4.4" - "@babel/plugin-transform-dotall-regex" "^7.4.4" - "@babel/types" "^7.4.4" - esutils "^2.0.2" - -"@babel/preset-react@^7.12.13", "@babel/preset-react@^7.12.5": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/preset-react/-/preset-react-7.16.0.tgz#f71d3e8dff5218478011df037fad52660ee6d82a" - integrity sha512-d31IFW2bLRB28uL1WoElyro8RH5l6531XfxMtCeCmp6RVAF1uTfxxUA0LH1tXl+psZdwfmIbwoG4U5VwgbhtLw== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-validator-option" "^7.14.5" - "@babel/plugin-transform-react-display-name" "^7.16.0" - "@babel/plugin-transform-react-jsx" "^7.16.0" - "@babel/plugin-transform-react-jsx-development" "^7.16.0" - "@babel/plugin-transform-react-pure-annotations" "^7.16.0" - -"@babel/preset-typescript@^7.12.16": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/preset-typescript/-/preset-typescript-7.16.0.tgz#b0b4f105b855fb3d631ec036cdc9d1ffd1fa5eac" - integrity sha512-txegdrZYgO9DlPbv+9QOVpMnKbOtezsLHWsnsRF4AjbSIsVaujrq1qg8HK0mxQpWv0jnejt0yEoW1uWpvbrDTg== - dependencies: - "@babel/helper-plugin-utils" "^7.14.5" - "@babel/helper-validator-option" "^7.14.5" - "@babel/plugin-transform-typescript" "^7.16.0" - -"@babel/runtime-corejs3@^7.12.13": - version "7.16.3" - resolved "https://registry.yarnpkg.com/@babel/runtime-corejs3/-/runtime-corejs3-7.16.3.tgz#1e25de4fa994c57c18e5fdda6cc810dac70f5590" - integrity sha512-IAdDC7T0+wEB4y2gbIL0uOXEYpiZEeuFUTVbdGq+UwCcF35T/tS8KrmMomEwEc5wBbyfH3PJVpTSUqrhPDXFcQ== - dependencies: - core-js-pure "^3.19.0" - regenerator-runtime "^0.13.4" - -"@babel/runtime@^7.1.2", "@babel/runtime@^7.10.2", "@babel/runtime@^7.10.3", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.8.4": - version "7.16.3" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.16.3.tgz#b86f0db02a04187a3c17caa77de69840165d42d5" - integrity sha512-WBwekcqacdY2e9AF/Q7WLFUWmdJGJTkbjqTjoMDgXkVZ3ZRUvOPsLb5KdwISoQVsbP+DQzVZW4Zhci0DvpbNTQ== - dependencies: - regenerator-runtime "^0.13.4" - -"@babel/template@^7.12.13", "@babel/template@^7.12.7": - version "7.12.13" - resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.12.13.tgz#530265be8a2589dbb37523844c5bcb55947fb327" - integrity sha512-/7xxiGA57xMo/P2GVvdEumr8ONhFOhfgq2ihK3h1e6THqzTAkHbkXgB0xI9yeTfIUoH3+oAeHhqm/I43OTbbjA== - dependencies: - "@babel/code-frame" "^7.12.13" - "@babel/parser" "^7.12.13" - "@babel/types" "^7.12.13" - -"@babel/template@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.16.0.tgz#d16a35ebf4cd74e202083356fab21dd89363ddd6" - integrity sha512-MnZdpFD/ZdYhXwiunMqqgyZyucaYsbL0IrjoGjaVhGilz+x8YB++kRfygSOIj1yOtWKPlx7NBp+9I1RQSgsd5A== - dependencies: - "@babel/code-frame" "^7.16.0" - "@babel/parser" "^7.16.0" - "@babel/types" "^7.16.0" - -"@babel/traverse@^7.12.13", "@babel/traverse@^7.12.9", "@babel/traverse@^7.13.0", "@babel/traverse@^7.13.13", "@babel/traverse@^7.13.17", "@babel/traverse@^7.16.0", "@babel/traverse@^7.16.3": - version "7.16.3" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.16.3.tgz#f63e8a938cc1b780f66d9ed3c54f532ca2d14787" - integrity sha512-eolumr1vVMjqevCpwVO99yN/LoGL0EyHiLO5I043aYQvwOJ9eR5UsZSClHVCzfhBduMAsSzgA/6AyqPjNayJag== - dependencies: - "@babel/code-frame" "^7.16.0" - "@babel/generator" "^7.16.0" - "@babel/helper-function-name" "^7.16.0" - "@babel/helper-hoist-variables" "^7.16.0" - "@babel/helper-split-export-declaration" "^7.16.0" - "@babel/parser" "^7.16.3" - "@babel/types" "^7.16.0" - debug "^4.1.0" - globals "^11.1.0" - -"@babel/types@^7.12.13", "@babel/types@^7.12.6", "@babel/types@^7.12.7", "@babel/types@^7.13.12", "@babel/types@^7.13.14", "@babel/types@^7.13.17", "@babel/types@^7.4.4": - version "7.13.17" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.13.17.tgz#48010a115c9fba7588b4437dd68c9469012b38b4" - integrity sha512-RawydLgxbOPDlTLJNtoIypwdmAy//uQIzlKt2+iBiJaRlVuI6QLUxVAyWGNfOzp8Yu4L4lLIacoCyTNtpb4wiA== - dependencies: - "@babel/helper-validator-identifier" "^7.12.11" - to-fast-properties "^2.0.0" - -"@babel/types@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.16.0.tgz#db3b313804f96aadd0b776c4823e127ad67289ba" - integrity sha512-PJgg/k3SdLsGb3hhisFvtLOw5ts113klrpLuIPtCJIU+BB24fqq6lf8RWqKJEjzqXR9AEH1rIb5XTqwBHB+kQg== - dependencies: - "@babel/helper-validator-identifier" "^7.15.7" - to-fast-properties "^2.0.0" - -"@crowdin/cli@3": - version "3.7.7" - resolved "https://registry.yarnpkg.com/@crowdin/cli/-/cli-3.7.7.tgz#0d2baf888b37aac1cd8d8cb484b0826fa7b82a26" - integrity sha512-wN0NXLDymC9PgT3I6jMx/YLjcEtegAnFg7VaavzLfc1vFegD4lKlrvRmc0qV2IIiRMZLShEsS0QtYtAyhi/qew== - dependencies: - shelljs "^0.8.4" - -"@docsearch/css@3.0.0-alpha.42": - version "3.0.0-alpha.42" - resolved "https://registry.yarnpkg.com/@docsearch/css/-/css-3.0.0-alpha.42.tgz#deb6049e999d6ca9451eba4793cb5b6da28c8773" - integrity sha512-AGwI2AXUacYhVOHmYnsXoYDJKO6Ued2W+QO80GERbMLhC7GH5tfvtW5REs/s7jSdcU3vzFoxT8iPDBCh/PkrlQ== - -"@docsearch/react@^3.0.0-alpha.36": - version "3.0.0-alpha.42" - resolved "https://registry.yarnpkg.com/@docsearch/react/-/react-3.0.0-alpha.42.tgz#1d22a2b05779f24d090ff8d7ff2699e4d50dff5c" - integrity sha512-1aOslZJDxwUUcm2QRNmlEePUgL8P5fOAeFdOLDMctHQkV2iTja9/rKVbkP8FZbIUnZxuuCCn8ErLrjD/oXWOag== - dependencies: - "@algolia/autocomplete-core" "1.5.0" - "@algolia/autocomplete-preset-algolia" "1.5.0" - "@docsearch/css" "3.0.0-alpha.42" - algoliasearch "^4.0.0" - -"@docusaurus/core@2.0.0-beta.759298296", "@docusaurus/core@^2.0.0-beta.10": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/core/-/core-2.0.0-beta.759298296.tgz#2390c401bf2ab46bf28eb93cd81759d682811388" - integrity sha512-qCvKt0KKjjMChAnlv1V/WocGNJ9up2uY9RKUzsBYIu0lKBGmrmmbPgXVJeQ/QYFqHtHR+qqYLfGsheRjXKXLbA== - dependencies: - "@babel/core" "^7.12.16" - "@babel/generator" "^7.12.15" - "@babel/plugin-syntax-dynamic-import" "^7.8.3" - "@babel/plugin-transform-runtime" "^7.12.15" - "@babel/preset-env" "^7.12.16" - "@babel/preset-react" "^7.12.13" - "@babel/preset-typescript" "^7.12.16" - "@babel/runtime" "^7.12.5" - "@babel/runtime-corejs3" "^7.12.13" - "@babel/traverse" "^7.12.13" - "@docusaurus/cssnano-preset" "2.0.0-beta.759298296" - "@docusaurus/react-loadable" "5.5.0" - "@docusaurus/types" "2.0.0-beta.759298296" - "@docusaurus/utils" "2.0.0-beta.759298296" - "@docusaurus/utils-common" "2.0.0-beta.759298296" - "@docusaurus/utils-validation" "2.0.0-beta.759298296" - "@slorber/static-site-generator-webpack-plugin" "^4.0.0" - "@svgr/webpack" "^5.5.0" - autoprefixer "^10.2.5" - babel-loader "^8.2.2" - babel-plugin-dynamic-import-node "2.3.0" - boxen "^5.0.1" - chalk "^4.1.1" - chokidar "^3.5.1" - clean-css "^5.1.2" - commander "^5.1.0" - copy-webpack-plugin "^9.0.0" - core-js "^3.9.1" - css-loader "^5.1.1" - css-minimizer-webpack-plugin "^3.0.1" - cssnano "^5.0.4" - del "^6.0.0" - detect-port "^1.3.0" - escape-html "^1.0.3" - eta "^1.12.1" - express "^4.17.1" - file-loader "^6.2.0" - fs-extra "^10.0.0" - github-slugger "^1.3.0" - globby "^11.0.2" - html-minifier-terser "^5.1.1" - html-tags "^3.1.0" - html-webpack-plugin "^5.3.2" - import-fresh "^3.3.0" - is-root "^2.1.0" - leven "^3.1.0" - lodash "^4.17.20" - mini-css-extract-plugin "^1.6.0" - module-alias "^2.2.2" - nprogress "^0.2.0" - postcss "^8.2.15" - postcss-loader "^5.3.0" - prompts "^2.4.1" - react-dev-utils "^11.0.1" - react-error-overlay "^6.0.9" - react-helmet "^6.1.0" - react-loadable "^5.5.0" - react-loadable-ssr-addon-v5-slorber "^1.0.1" - react-router "^5.2.0" - react-router-config "^5.1.1" - react-router-dom "^5.2.0" - resolve-pathname "^3.0.0" - rtl-detect "^1.0.3" - semver "^7.3.4" - serve-handler "^6.1.3" - shelljs "^0.8.4" - std-env "^2.2.1" - strip-ansi "^6.0.0" - terser-webpack-plugin "^5.1.3" - tslib "^2.2.0" - update-notifier "^5.1.0" - url-loader "^4.1.1" - wait-on "^5.3.0" - webpack "^5.40.0" - webpack-bundle-analyzer "^4.4.2" - webpack-dev-server "^3.11.2" - webpack-merge "^5.8.0" - webpackbar "^5.0.0-3" - -"@docusaurus/cssnano-preset@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/cssnano-preset/-/cssnano-preset-2.0.0-beta.759298296.tgz#f5c4b5aae787fddebdd362e29da508cb97ccb932" - integrity sha512-JorG6umrBUxooQ3xbYKh9htmCQTdpzpVjEMdWgBjIn8DZWHH3e+u9IfcemAWz8GUm7wUz94PxDYKiXIICaZ0xA== - dependencies: - cssnano-preset-advanced "^5.1.1" - postcss "^8.2.15" - postcss-sort-media-queries "^3.10.11" - -"@docusaurus/mdx-loader@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/mdx-loader/-/mdx-loader-2.0.0-beta.759298296.tgz#9c56162e58dc24cb384c5c6ed4e414700b8bdffd" - integrity sha512-sEiWj5Aehxs8JeqUsa/lfKbiEmJZpQU84drSXlejaW2NvEATuIqnNlWkGHEkK9t2qVx3/gwUFhaEt+dumttDVA== - dependencies: - "@babel/parser" "^7.12.16" - "@babel/traverse" "^7.12.13" - "@docusaurus/core" "2.0.0-beta.759298296" - "@docusaurus/utils" "2.0.0-beta.759298296" - "@mdx-js/mdx" "^1.6.21" - "@mdx-js/react" "^1.6.21" - escape-html "^1.0.3" - file-loader "^6.2.0" - fs-extra "^10.0.0" - github-slugger "^1.3.0" - gray-matter "^4.0.3" - mdast-util-to-string "^2.0.0" - remark-emoji "^2.1.0" - stringify-object "^3.3.0" - unist-util-visit "^2.0.2" - url-loader "^4.1.1" - webpack "^5.40.0" - -"@docusaurus/plugin-content-blog@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-blog/-/plugin-content-blog-2.0.0-beta.759298296.tgz#48fc2350ecfeb18c4dbdf0fe514d73dd278708cd" - integrity sha512-EzfiGmfsY60Q6r53oGAxa2/643Rrs/QTLdFi3TbsFMfPZyg8sutNDdFbIXWMaDeQeO9qlreqT4wESxV3GltZkw== - dependencies: - "@docusaurus/core" "2.0.0-beta.759298296" - "@docusaurus/mdx-loader" "2.0.0-beta.759298296" - "@docusaurus/types" "2.0.0-beta.759298296" - "@docusaurus/utils" "2.0.0-beta.759298296" - "@docusaurus/utils-validation" "2.0.0-beta.759298296" - chalk "^4.1.1" - escape-string-regexp "^4.0.0" - feed "^4.2.2" - fs-extra "^10.0.0" - globby "^11.0.2" - loader-utils "^2.0.0" - lodash "^4.17.20" - reading-time "^1.3.0" - remark-admonitions "^1.2.1" - tslib "^2.2.0" - webpack "^5.40.0" - -"@docusaurus/plugin-content-docs@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-docs/-/plugin-content-docs-2.0.0-beta.759298296.tgz#171d37add12c08ca5728c1e99f8cde9b2e4cd76c" - integrity sha512-I7cw05gIwaTvJ4vPXj6Kul3Cmo0DeoiSv3zLraL6WZyWOPmvvj0Lj2RMDvXO4ovEuzWGqTykpWRrJbvC/Q5VHg== - dependencies: - "@docusaurus/core" "2.0.0-beta.759298296" - "@docusaurus/mdx-loader" "2.0.0-beta.759298296" - "@docusaurus/types" "2.0.0-beta.759298296" - "@docusaurus/utils" "2.0.0-beta.759298296" - "@docusaurus/utils-validation" "2.0.0-beta.759298296" - chalk "^4.1.1" - combine-promises "^1.1.0" - escape-string-regexp "^4.0.0" - execa "^5.0.0" - fs-extra "^10.0.0" - globby "^11.0.2" - import-fresh "^3.2.2" - js-yaml "^4.0.0" - loader-utils "^1.2.3" - lodash "^4.17.20" - remark-admonitions "^1.2.1" - shelljs "^0.8.4" - tslib "^2.2.0" - utility-types "^3.10.0" - webpack "^5.40.0" - -"@docusaurus/plugin-content-pages@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-pages/-/plugin-content-pages-2.0.0-beta.759298296.tgz#8b6e709b6b3a0935effffaec99a3068d245bbb7e" - integrity sha512-l/FNV1op2zrwTmSWrLVOghtfMpEoaXROvF19c7tU382BuRLi8fz0TixN/ZDr7f8avoN7smZ8j5HzaLb9lh6kZg== - dependencies: - "@docusaurus/core" "2.0.0-beta.759298296" - "@docusaurus/mdx-loader" "2.0.0-beta.759298296" - "@docusaurus/types" "2.0.0-beta.759298296" - "@docusaurus/utils" "2.0.0-beta.759298296" - "@docusaurus/utils-validation" "2.0.0-beta.759298296" - globby "^11.0.2" - lodash "^4.17.20" - minimatch "^3.0.4" - remark-admonitions "^1.2.1" - slash "^3.0.0" - tslib "^2.1.0" - webpack "^5.40.0" - -"@docusaurus/plugin-debug@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-debug/-/plugin-debug-2.0.0-beta.759298296.tgz#dcca80e4bd9a4f7ff3f49445a3b8abdd8af41f15" - integrity sha512-P1TVKBnWEWp/Q3gV8RcFH/kFY1fpgHABO4tjPqC6MOXrPURMGaBmqkicWzgg3ntoIIrqZthkjVLvyXAnPPpAIg== - dependencies: - "@docusaurus/core" "2.0.0-beta.759298296" - "@docusaurus/types" "2.0.0-beta.759298296" - "@docusaurus/utils" "2.0.0-beta.759298296" - react-json-view "^1.21.3" - tslib "^2.1.0" - -"@docusaurus/plugin-google-analytics@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-analytics/-/plugin-google-analytics-2.0.0-beta.759298296.tgz#a66e1ea1e6635766d22b0ccf12fd72de0899367a" - integrity sha512-/LiLgSD8QI1DhXriZWSSoSmhtCYmvEDl6LyeeFuZV2gxMmBoqaAnb90d/fXWfaMMF2z7kcqUQOBhXb1aXTl8Gw== - dependencies: - "@docusaurus/core" "2.0.0-beta.759298296" - -"@docusaurus/plugin-google-gtag@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-google-gtag/-/plugin-google-gtag-2.0.0-beta.759298296.tgz#fa19004ee3355ff2e8ba65485cf4a82518ab4562" - integrity sha512-2bonG4VES4Ab3SZVv5VpgiCyqctpO7AtpHRF+r7+F3+NtlNV398Ov8mOuXkdcWLqM+xh3syHnqCeoAUzFjyBVw== - dependencies: - "@docusaurus/core" "2.0.0-beta.759298296" - -"@docusaurus/plugin-sitemap@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/plugin-sitemap/-/plugin-sitemap-2.0.0-beta.759298296.tgz#6e0a82fe85057f5972f260b636cbf7599abadb18" - integrity sha512-+Pv6CoBMufPmh1eqmOf+cMTdUOtpl5xh5Ggdt7FRpZABNTrAXCEdU3wv/s/vYx2+aXYB1LFhBUdMidZ41HMlMg== - dependencies: - "@docusaurus/core" "2.0.0-beta.759298296" - "@docusaurus/types" "2.0.0-beta.759298296" - "@docusaurus/utils" "2.0.0-beta.759298296" - "@docusaurus/utils-common" "2.0.0-beta.759298296" - "@docusaurus/utils-validation" "2.0.0-beta.759298296" - fs-extra "^10.0.0" - sitemap "^7.0.0" - tslib "^2.2.0" - -"@docusaurus/preset-classic@^2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/preset-classic/-/preset-classic-2.0.0-beta.759298296.tgz#6e1b3c8aed580f6e45fbf16151978530ca180410" - integrity sha512-NNkjwbN8G/GiQM8FukgIhQjmjjE23aKBny8q9wNVFdn/FGfLBXz21dEUQzqeaAfrr9xWLKOBeNFIJU1DTv98rg== - dependencies: - "@docusaurus/core" "2.0.0-beta.759298296" - "@docusaurus/plugin-content-blog" "2.0.0-beta.759298296" - "@docusaurus/plugin-content-docs" "2.0.0-beta.759298296" - "@docusaurus/plugin-content-pages" "2.0.0-beta.759298296" - "@docusaurus/plugin-debug" "2.0.0-beta.759298296" - "@docusaurus/plugin-google-analytics" "2.0.0-beta.759298296" - "@docusaurus/plugin-google-gtag" "2.0.0-beta.759298296" - "@docusaurus/plugin-sitemap" "2.0.0-beta.759298296" - "@docusaurus/theme-classic" "2.0.0-beta.759298296" - "@docusaurus/theme-search-algolia" "2.0.0-beta.759298296" - -"@docusaurus/react-loadable@5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.0.tgz#6d6f0c8fd9a434b62a1ab1f8645ee7bde5a9ec21" - integrity sha512-Ld/kwUE6yATIOTLq3JCsWiTa/drisajwKqBQ2Rw6IcT+sFsKfYek8F2jSH8f68AT73xX97UehduZeCSlnuCBIg== - dependencies: - prop-types "^15.6.2" - -"@docusaurus/theme-classic@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/theme-classic/-/theme-classic-2.0.0-beta.759298296.tgz#5b5a67be17527e870eda62648dab15ce9bf117bb" - integrity sha512-EKvthWxalq1wh0LQCC0mOzUjwBOytg2gg6Z6yFeDQiqR940VEZUy/7nKOA0QrK6dPjB2yhICXamJ3xZ40GsfPg== - dependencies: - "@docusaurus/core" "2.0.0-beta.759298296" - "@docusaurus/plugin-content-blog" "2.0.0-beta.759298296" - "@docusaurus/plugin-content-docs" "2.0.0-beta.759298296" - "@docusaurus/plugin-content-pages" "2.0.0-beta.759298296" - "@docusaurus/theme-common" "2.0.0-beta.759298296" - "@docusaurus/types" "2.0.0-beta.759298296" - "@docusaurus/utils" "2.0.0-beta.759298296" - "@docusaurus/utils-common" "2.0.0-beta.759298296" - "@docusaurus/utils-validation" "2.0.0-beta.759298296" - "@mdx-js/mdx" "^1.6.21" - "@mdx-js/react" "^1.6.21" - chalk "^4.1.1" - clsx "^1.1.1" - copy-text-to-clipboard "^3.0.1" - fs-extra "^10.0.0" - globby "^11.0.2" - infima "0.2.0-alpha.26" - lodash "^4.17.20" - parse-numeric-range "^1.2.0" - postcss "^8.2.15" - prism-react-renderer "^1.2.1" - prismjs "^1.23.0" - prop-types "^15.7.2" - react-router-dom "^5.2.0" - rtlcss "^3.1.2" - -"@docusaurus/theme-common@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/theme-common/-/theme-common-2.0.0-beta.759298296.tgz#917156961a0c8b162dfb72280b62d34b92f83183" - integrity sha512-OlFHH9ukTYXkh2Q+7Cd29JYA2JMwhLA/vkZMTpLjjmR1w2MiaCfB3wF1qq+C84EZ7hrmWrI2fHoJmkXVDIVTiQ== - dependencies: - "@docusaurus/core" "2.0.0-beta.759298296" - "@docusaurus/plugin-content-blog" "2.0.0-beta.759298296" - "@docusaurus/plugin-content-docs" "2.0.0-beta.759298296" - "@docusaurus/plugin-content-pages" "2.0.0-beta.759298296" - "@docusaurus/types" "2.0.0-beta.759298296" - tslib "^2.1.0" - -"@docusaurus/theme-search-algolia@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/theme-search-algolia/-/theme-search-algolia-2.0.0-beta.759298296.tgz#c9e9b1b43267af757c4ceac72010328aa3f6a3fd" - integrity sha512-+JHLK2PL88NoJh6JFYeUgjPjxWq3n2SepNnFj+7rX7lymXmaVdSy7nQKpM8FpkN0Q23zCdYpdnm+y+fgBBPojg== - dependencies: - "@docsearch/react" "^3.0.0-alpha.36" - "@docusaurus/core" "2.0.0-beta.759298296" - "@docusaurus/theme-common" "2.0.0-beta.759298296" - "@docusaurus/utils" "2.0.0-beta.759298296" - "@docusaurus/utils-validation" "2.0.0-beta.759298296" - algoliasearch "^4.8.4" - algoliasearch-helper "^3.3.4" - clsx "^1.1.1" - eta "^1.12.1" - lodash "^4.17.20" - -"@docusaurus/types@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/types/-/types-2.0.0-beta.759298296.tgz#27b48ce8e55aa8548c405e087b245e5b7699131e" - integrity sha512-1qk9qRHZGHiZbwUr6YY41DFOA451QMlHf3zFhNPr7hGxyPPbbzvo+awEpEwhniTgoGjlPNRjw/KJEu0kRUl5OQ== - dependencies: - commander "^5.1.0" - joi "^17.4.0" - querystring "0.2.0" - webpack "^5.40.0" - webpack-merge "^5.8.0" - -"@docusaurus/utils-common@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/utils-common/-/utils-common-2.0.0-beta.759298296.tgz#228aa5eae19666c5ccff3cd0b172cbca01a2fdf5" - integrity sha512-it0LfFUPEEOXYB5xi/moeH1dJ43bBAE41kvfYefBwN4YRprziFNKcv3vk+xcBPscpUksoEATSbaIBFWvvgjVvQ== - dependencies: - "@docusaurus/types" "2.0.0-beta.759298296" - tslib "^2.2.0" - -"@docusaurus/utils-validation@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/utils-validation/-/utils-validation-2.0.0-beta.759298296.tgz#ed264ed2c524ab34127356ae099232f25a564f2b" - integrity sha512-Zn7eAo+q/DdDgRscj5bNw8eTdLOFT8U9BNpdYRCC9sMMQR+shgkuKL3gKlC/wDOWEx4nQd1R83bLhmPTXppE2A== - dependencies: - "@docusaurus/utils" "2.0.0-beta.759298296" - chalk "^4.1.1" - joi "^17.4.0" - tslib "^2.1.0" - -"@docusaurus/utils@2.0.0-beta.759298296": - version "2.0.0-beta.759298296" - resolved "https://registry.yarnpkg.com/@docusaurus/utils/-/utils-2.0.0-beta.759298296.tgz#d7d404b0982583c5baa38fa844c863c5ec6907ac" - integrity sha512-QQCgNVu2Vuuw9m2k0iX4CFSR9emSTHs20A7tdoNl+WRcep8DjtG0Oy2TyQTH3C7l2l46jSzbCG1wHtf3hKLvjg== - dependencies: - "@docusaurus/types" "2.0.0-beta.759298296" - "@types/github-slugger" "^1.3.0" - chalk "^4.1.1" - escape-string-regexp "^4.0.0" - fs-extra "^10.0.0" - gray-matter "^4.0.3" - lodash "^4.17.20" - resolve-pathname "^3.0.0" - tslib "^2.2.0" - -"@hapi/hoek@^9.0.0": - version "9.2.0" - resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.2.0.tgz#f3933a44e365864f4dad5db94158106d511e8131" - integrity sha512-sqKVVVOe5ivCaXDWivIJYVSaEgdQK9ul7a4Kity5Iw7u9+wBAPbX1RMSnLLmp7O4Vzj0WOWwMAJsTL00xwaNug== - -"@hapi/topo@^5.0.0": - version "5.0.0" - resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-5.0.0.tgz#c19af8577fa393a06e9c77b60995af959be721e7" - integrity sha512-tFJlT47db0kMqVm3H4nQYgn6Pwg10GTZHb1pwmSiv1K4ks6drQOtfEF5ZnPjkvC+y4/bUPHK+bc87QvLcL+WMw== - dependencies: - "@hapi/hoek" "^9.0.0" - -"@mdx-js/mdx@^1.6.21": - version "1.6.22" - resolved "https://registry.yarnpkg.com/@mdx-js/mdx/-/mdx-1.6.22.tgz#8a723157bf90e78f17dc0f27995398e6c731f1ba" - integrity sha512-AMxuLxPz2j5/6TpF/XSdKpQP1NlG0z11dFOlq+2IP/lSgl11GY8ji6S/rgsViN/L0BDvHvUMruRb7ub+24LUYA== - dependencies: - "@babel/core" "7.12.9" - "@babel/plugin-syntax-jsx" "7.12.1" - "@babel/plugin-syntax-object-rest-spread" "7.8.3" - "@mdx-js/util" "1.6.22" - babel-plugin-apply-mdx-type-prop "1.6.22" - babel-plugin-extract-import-names "1.6.22" - camelcase-css "2.0.1" - detab "2.0.4" - hast-util-raw "6.0.1" - lodash.uniq "4.5.0" - mdast-util-to-hast "10.0.1" - remark-footnotes "2.0.0" - remark-mdx "1.6.22" - remark-parse "8.0.3" - remark-squeeze-paragraphs "4.0.0" - style-to-object "0.3.0" - unified "9.2.0" - unist-builder "2.0.3" - unist-util-visit "2.0.3" - -"@mdx-js/react@^1.6.21": - version "1.6.22" - resolved "https://registry.yarnpkg.com/@mdx-js/react/-/react-1.6.22.tgz#ae09b4744fddc74714ee9f9d6f17a66e77c43573" - integrity sha512-TDoPum4SHdfPiGSAaRBw7ECyI8VaHpK8GJugbJIJuqyh6kzw9ZLJZW3HGL3NNrJGxcAixUvqROm+YuQOo5eXtg== - -"@mdx-js/util@1.6.22": - version "1.6.22" - resolved "https://registry.yarnpkg.com/@mdx-js/util/-/util-1.6.22.tgz#219dfd89ae5b97a8801f015323ffa4b62f45718b" - integrity sha512-H1rQc1ZOHANWBvPcW+JpGwr+juXSxM8Q8YCkm3GhZd8REu1fHR3z99CErO1p9pkcfcxZnMdIZdIsXkOHY0NilA== - -"@nodelib/fs.scandir@2.1.4": - version "2.1.4" - resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.4.tgz#d4b3549a5db5de2683e0c1071ab4f140904bbf69" - integrity sha512-33g3pMJk3bg5nXbL/+CY6I2eJDzZAni49PfJnL5fghPTggPvBd/pFNSgJsdAgWptuFu7qq/ERvOYFlhvsLTCKA== - dependencies: - "@nodelib/fs.stat" "2.0.4" - run-parallel "^1.1.9" - -"@nodelib/fs.stat@2.0.4", "@nodelib/fs.stat@^2.0.2": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.4.tgz#a3f2dd61bab43b8db8fa108a121cfffe4c676655" - integrity sha512-IYlHJA0clt2+Vg7bccq+TzRdJvv19c2INqBSsoOLp1je7xjtr7J26+WXR72MCdvU9q1qTzIWDfhMf+DRvQJK4Q== - -"@nodelib/fs.walk@^1.2.3": - version "1.2.6" - resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.6.tgz#cce9396b30aa5afe9e3756608f5831adcb53d063" - integrity sha512-8Broas6vTtW4GIXTAHDoE32hnN2M5ykgCpWGbuXHQ15vEMqr23pB76e/GZcYsZCHALv50ktd24qhEyKr6wBtow== - dependencies: - "@nodelib/fs.scandir" "2.1.4" - fastq "^1.6.0" - -"@polka/url@^1.0.0-next.9": - version "1.0.0-next.12" - resolved "https://registry.yarnpkg.com/@polka/url/-/url-1.0.0-next.12.tgz#431ec342a7195622f86688bbda82e3166ce8cb28" - integrity sha512-6RglhutqrGFMO1MNUXp95RBuYIuc8wTnMAV5MUhLmjTOy78ncwOw7RgeQ/HeymkKXRhZd0s2DNrM1rL7unk3MQ== - -"@sideway/address@^4.1.0": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.1.tgz#9e321e74310963fdf8eebfbee09c7bd69972de4d" - integrity sha512-+I5aaQr3m0OAmMr7RQ3fR9zx55sejEYR2BFJaxL+zT3VM2611X0SHvPWIbAUBZVTn/YzYKbV8gJ2oT/QELknfQ== - dependencies: - "@hapi/hoek" "^9.0.0" - -"@sideway/address@^4.1.3": - version "4.1.3" - resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.3.tgz#d93cce5d45c5daec92ad76db492cc2ee3c64ab27" - integrity sha512-8ncEUtmnTsMmL7z1YPB47kPUq7LpKWJNFPsRzHiIajGC5uXlWGn+AmkYPcHNl8S4tcEGx+cnORnNYaw2wvL+LQ== - dependencies: - "@hapi/hoek" "^9.0.0" - -"@sideway/formula@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.0.tgz#fe158aee32e6bd5de85044be615bc08478a0a13c" - integrity sha512-vHe7wZ4NOXVfkoRb8T5otiENVlT7a3IAiw7H5M2+GO+9CDgcVUUsX1zalAztCmwyOr2RUTGJdgB+ZvSVqmdHmg== - -"@sideway/pinpoint@^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df" - integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ== - -"@sindresorhus/is@^0.14.0": - version "0.14.0" - resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" - integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ== - -"@slorber/static-site-generator-webpack-plugin@^4.0.0": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@slorber/static-site-generator-webpack-plugin/-/static-site-generator-webpack-plugin-4.0.1.tgz#0c8852146441aaa683693deaa5aee2f991d94841" - integrity sha512-PSv4RIVO1Y3kvHxjvqeVisk3E9XFoO04uwYBDWe217MFqKspplYswTuKLiJu0aLORQWzuQjfVsSlLPojwfYsLw== - dependencies: - bluebird "^3.7.1" - cheerio "^0.22.0" - eval "^0.1.4" - url "^0.11.0" - webpack-sources "^1.4.3" - -"@svgr/babel-plugin-add-jsx-attribute@^5.4.0": - version "5.4.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-add-jsx-attribute/-/babel-plugin-add-jsx-attribute-5.4.0.tgz#81ef61947bb268eb9d50523446f9c638fb355906" - integrity sha512-ZFf2gs/8/6B8PnSofI0inYXr2SDNTDScPXhN7k5EqD4aZ3gi6u+rbmZHVB8IM3wDyx8ntKACZbtXSm7oZGRqVg== - -"@svgr/babel-plugin-remove-jsx-attribute@^5.4.0": - version "5.4.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-remove-jsx-attribute/-/babel-plugin-remove-jsx-attribute-5.4.0.tgz#6b2c770c95c874654fd5e1d5ef475b78a0a962ef" - integrity sha512-yaS4o2PgUtwLFGTKbsiAy6D0o3ugcUhWK0Z45umJ66EPWunAz9fuFw2gJuje6wqQvQWOTJvIahUwndOXb7QCPg== - -"@svgr/babel-plugin-remove-jsx-empty-expression@^5.0.1": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-remove-jsx-empty-expression/-/babel-plugin-remove-jsx-empty-expression-5.0.1.tgz#25621a8915ed7ad70da6cea3d0a6dbc2ea933efd" - integrity sha512-LA72+88A11ND/yFIMzyuLRSMJ+tRKeYKeQ+mR3DcAZ5I4h5CPWN9AHyUzJbWSYp/u2u0xhmgOe0+E41+GjEueA== - -"@svgr/babel-plugin-replace-jsx-attribute-value@^5.0.1": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-replace-jsx-attribute-value/-/babel-plugin-replace-jsx-attribute-value-5.0.1.tgz#0b221fc57f9fcd10e91fe219e2cd0dd03145a897" - integrity sha512-PoiE6ZD2Eiy5mK+fjHqwGOS+IXX0wq/YDtNyIgOrc6ejFnxN4b13pRpiIPbtPwHEc+NT2KCjteAcq33/F1Y9KQ== - -"@svgr/babel-plugin-svg-dynamic-title@^5.4.0": - version "5.4.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-svg-dynamic-title/-/babel-plugin-svg-dynamic-title-5.4.0.tgz#139b546dd0c3186b6e5db4fefc26cb0baea729d7" - integrity sha512-zSOZH8PdZOpuG1ZVx/cLVePB2ibo3WPpqo7gFIjLV9a0QsuQAzJiwwqmuEdTaW2pegyBE17Uu15mOgOcgabQZg== - -"@svgr/babel-plugin-svg-em-dimensions@^5.4.0": - version "5.4.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-svg-em-dimensions/-/babel-plugin-svg-em-dimensions-5.4.0.tgz#6543f69526632a133ce5cabab965deeaea2234a0" - integrity sha512-cPzDbDA5oT/sPXDCUYoVXEmm3VIoAWAPT6mSPTJNbQaBNUuEKVKyGH93oDY4e42PYHRW67N5alJx/eEol20abw== - -"@svgr/babel-plugin-transform-react-native-svg@^5.4.0": - version "5.4.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-transform-react-native-svg/-/babel-plugin-transform-react-native-svg-5.4.0.tgz#00bf9a7a73f1cad3948cdab1f8dfb774750f8c80" - integrity sha512-3eYP/SaopZ41GHwXma7Rmxcv9uRslRDTY1estspeB1w1ueZWd/tPlMfEOoccYpEMZU3jD4OU7YitnXcF5hLW2Q== - -"@svgr/babel-plugin-transform-svg-component@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-transform-svg-component/-/babel-plugin-transform-svg-component-5.5.0.tgz#583a5e2a193e214da2f3afeb0b9e8d3250126b4a" - integrity sha512-q4jSH1UUvbrsOtlo/tKcgSeiCHRSBdXoIoqX1pgcKK/aU3JD27wmMKwGtpB8qRYUYoyXvfGxUVKchLuR5pB3rQ== - -"@svgr/babel-preset@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/babel-preset/-/babel-preset-5.5.0.tgz#8af54f3e0a8add7b1e2b0fcd5a882c55393df327" - integrity sha512-4FiXBjvQ+z2j7yASeGPEi8VD/5rrGQk4Xrq3EdJmoZgz/tpqChpo5hgXDvmEauwtvOc52q8ghhZK4Oy7qph4ig== - dependencies: - "@svgr/babel-plugin-add-jsx-attribute" "^5.4.0" - "@svgr/babel-plugin-remove-jsx-attribute" "^5.4.0" - "@svgr/babel-plugin-remove-jsx-empty-expression" "^5.0.1" - "@svgr/babel-plugin-replace-jsx-attribute-value" "^5.0.1" - "@svgr/babel-plugin-svg-dynamic-title" "^5.4.0" - "@svgr/babel-plugin-svg-em-dimensions" "^5.4.0" - "@svgr/babel-plugin-transform-react-native-svg" "^5.4.0" - "@svgr/babel-plugin-transform-svg-component" "^5.5.0" - -"@svgr/core@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/core/-/core-5.5.0.tgz#82e826b8715d71083120fe8f2492ec7d7874a579" - integrity sha512-q52VOcsJPvV3jO1wkPtzTuKlvX7Y3xIcWRpCMtBF3MrteZJtBfQw/+u0B1BHy5ColpQc1/YVTrPEtSYIMNZlrQ== - dependencies: - "@svgr/plugin-jsx" "^5.5.0" - camelcase "^6.2.0" - cosmiconfig "^7.0.0" - -"@svgr/hast-util-to-babel-ast@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/hast-util-to-babel-ast/-/hast-util-to-babel-ast-5.5.0.tgz#5ee52a9c2533f73e63f8f22b779f93cd432a5461" - integrity sha512-cAaR/CAiZRB8GP32N+1jocovUtvlj0+e65TB50/6Lcime+EA49m/8l+P2ko+XPJ4dw3xaPS3jOL4F2X4KWxoeQ== - dependencies: - "@babel/types" "^7.12.6" - -"@svgr/plugin-jsx@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/plugin-jsx/-/plugin-jsx-5.5.0.tgz#1aa8cd798a1db7173ac043466d7b52236b369000" - integrity sha512-V/wVh33j12hGh05IDg8GpIUXbjAPnTdPTKuP4VNLggnwaHMPNQNae2pRnyTAILWCQdz5GyMqtO488g7CKM8CBA== - dependencies: - "@babel/core" "^7.12.3" - "@svgr/babel-preset" "^5.5.0" - "@svgr/hast-util-to-babel-ast" "^5.5.0" - svg-parser "^2.0.2" - -"@svgr/plugin-svgo@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/plugin-svgo/-/plugin-svgo-5.5.0.tgz#02da55d85320549324e201c7b2e53bf431fcc246" - integrity sha512-r5swKk46GuQl4RrVejVwpeeJaydoxkdwkM1mBKOgJLBUJPGaLci6ylg/IjhrRsREKDkr4kbMWdgOtbXEh0fyLQ== - dependencies: - cosmiconfig "^7.0.0" - deepmerge "^4.2.2" - svgo "^1.2.2" - -"@svgr/webpack@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@svgr/webpack/-/webpack-5.5.0.tgz#aae858ee579f5fa8ce6c3166ef56c6a1b381b640" - integrity sha512-DOBOK255wfQxguUta2INKkzPj6AIS6iafZYiYmHn6W3pHlycSRRlvWKCfLDG10fXfLWqE3DJHgRUOyJYmARa7g== - dependencies: - "@babel/core" "^7.12.3" - "@babel/plugin-transform-react-constant-elements" "^7.12.1" - "@babel/preset-env" "^7.12.1" - "@babel/preset-react" "^7.12.5" - "@svgr/core" "^5.5.0" - "@svgr/plugin-jsx" "^5.5.0" - "@svgr/plugin-svgo" "^5.5.0" - loader-utils "^2.0.0" - -"@szmarczak/http-timer@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-1.1.2.tgz#b1665e2c461a2cd92f4c1bbf50d5454de0d4b421" - integrity sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA== - dependencies: - defer-to-connect "^1.0.1" - -"@trysound/sax@0.1.1": - version "0.1.1" - resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.1.1.tgz#3348564048e7a2d7398c935d466c0414ebb6a669" - integrity sha512-Z6DoceYb/1xSg5+e+ZlPZ9v0N16ZvZ+wYMraFue4HYrE4ttONKtsvruIRf6t9TBR0YvSOfi1hUU0fJfBLCDYow== - -"@trysound/sax@0.2.0": - version "0.2.0" - resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad" - integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA== - -"@types/cssnano@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@types/cssnano/-/cssnano-4.0.1.tgz#67fa912753d80973a016e7684a47fedf338aacff" - integrity sha512-hGOroxRTBkYl5gSBRJOffhV4+io+Y2bFX1VP7LgKEVHJt/LPPJaWUIuDAz74Vlp7l7hCDZfaDi7iPxwNwuVA4Q== - dependencies: - postcss "5 - 7" - -"@types/eslint-scope@^3.7.0": - version "3.7.0" - resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.0.tgz#4792816e31119ebd506902a482caec4951fabd86" - integrity sha512-O/ql2+rrCUe2W2rs7wMR+GqPRcgB6UiqN5RhrR5xruFlY7l9YLMn0ZkDzjoHLeiFkR8MCQZVudUuuvQ2BLC9Qw== - dependencies: - "@types/eslint" "*" - "@types/estree" "*" - -"@types/eslint@*": - version "7.2.10" - resolved "https://registry.yarnpkg.com/@types/eslint/-/eslint-7.2.10.tgz#4b7a9368d46c0f8cd5408c23288a59aa2394d917" - integrity sha512-kUEPnMKrqbtpCq/KTaGFFKAcz6Ethm2EjCoKIDaCmfRBWLbFuTcOJfTlorwbnboXBzahqWLgUp1BQeKHiJzPUQ== - dependencies: - "@types/estree" "*" - "@types/json-schema" "*" - -"@types/estree@*": - version "0.0.47" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.47.tgz#d7a51db20f0650efec24cd04994f523d93172ed4" - integrity sha512-c5ciR06jK8u9BstrmJyO97m+klJrrhCf9u3rLu3DEAJBirxRqSCvDQoYKmxuYwQI5SZChAWu+tq9oVlGRuzPAg== - -"@types/estree@^0.0.50": - version "0.0.50" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.50.tgz#1e0caa9364d3fccd2931c3ed96fdbeaa5d4cca83" - integrity sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw== - -"@types/github-slugger@^1.3.0": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@types/github-slugger/-/github-slugger-1.3.0.tgz#16ab393b30d8ae2a111ac748a015ac05a1fc5524" - integrity sha512-J/rMZa7RqiH/rT29TEVZO4nBoDP9XJOjnbbIofg7GQKs4JIduEO3WLpte+6WeUz/TcrXKlY+bM7FYrp8yFB+3g== - -"@types/glob@^7.1.1": - version "7.2.0" - resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb" - integrity sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA== - dependencies: - "@types/minimatch" "*" - "@types/node" "*" - -"@types/hast@^2.0.0": - version "2.3.1" - resolved "https://registry.yarnpkg.com/@types/hast/-/hast-2.3.1.tgz#b16872f2a6144c7025f296fb9636a667ebb79cd9" - integrity sha512-viwwrB+6xGzw+G1eWpF9geV3fnsDgXqHG+cqgiHrvQfDUW5hzhCyV7Sy3UJxhfRFBsgky2SSW33qi/YrIkjX5Q== - dependencies: - "@types/unist" "*" - -"@types/html-minifier-terser@^6.0.0": - version "6.0.0" - resolved "https://registry.yarnpkg.com/@types/html-minifier-terser/-/html-minifier-terser-6.0.0.tgz#563c1c6c132cd204e71512f9c0b394ff90d3fae7" - integrity sha512-NZwaaynfs1oIoLAV1vg18e7QMVDvw+6SQrdJc8w3BwUaoroVSf6EBj/Sk4PBWGxsq0dzhA2drbsuMC1/6C6KgQ== - -"@types/json-schema@*", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.6": - version "7.0.7" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad" - integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA== - -"@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": - version "7.0.9" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.9.tgz#97edc9037ea0c38585320b28964dde3b39e4660d" - integrity sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ== - -"@types/mdast@^3.0.0": - version "3.0.3" - resolved "https://registry.yarnpkg.com/@types/mdast/-/mdast-3.0.3.tgz#2d7d671b1cd1ea3deb306ea75036c2a0407d2deb" - integrity sha512-SXPBMnFVQg1s00dlMCc/jCdvPqdE4mXaMMCeRlxLDmTAEoegHT53xKtkDnzDTOcmMHUfcjyf36/YYZ6SxRdnsw== - dependencies: - "@types/unist" "*" - -"@types/minimatch@*": - version "3.0.5" - resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" - integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ== - -"@types/node@*": - version "15.0.1" - resolved "https://registry.yarnpkg.com/@types/node/-/node-15.0.1.tgz#ef34dea0881028d11398be5bf4e856743e3dc35a" - integrity sha512-TMkXt0Ck1y0KKsGr9gJtWGjttxlZnnvDtphxUOSd0bfaR6Q1jle+sPvrzNR1urqYTWMinoKvjKfXUGsumaO1PA== - -"@types/node@^15.0.1": - version "15.12.3" - resolved "https://registry.yarnpkg.com/@types/node/-/node-15.12.3.tgz#2817bf5f25bc82f56579018c53f7d41b1830b1af" - integrity sha512-SNt65CPCXvGNDZ3bvk1TQ0Qxoe3y1RKH88+wZ2Uf05dduBCqqFQ76ADP9pbT+Cpvj60SkRppMCh2Zo8tDixqjQ== - -"@types/parse-json@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" - integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== - -"@types/parse5@^5.0.0": - version "5.0.3" - resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-5.0.3.tgz#e7b5aebbac150f8b5fdd4a46e7f0bd8e65e19109" - integrity sha512-kUNnecmtkunAoQ3CnjmMkzNU/gtxG8guhi+Fk2U/kOpIKjIMKnXGp4IJCgQJrXSgMsWYimYG4TGjz/UzbGEBTw== - -"@types/q@^1.5.1": - version "1.5.4" - resolved "https://registry.yarnpkg.com/@types/q/-/q-1.5.4.tgz#15925414e0ad2cd765bfef58842f7e26a7accb24" - integrity sha512-1HcDas8SEj4z1Wc696tH56G8OlRaH/sqZOynNNB+HF0WOeXPaxTtbYzJY2oEfiUxjSKjhCKr+MvR7dCHcEelug== - -"@types/sax@^1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@types/sax/-/sax-1.2.1.tgz#e0248be936ece791a82db1a57f3fb5f7c87e8172" - integrity sha512-dqYdvN7Sbw8QT/0Ci5rhjE4/iCMJEM0Y9rHpCu+gGXD9Lwbz28t6HI2yegsB6BoV1sShRMU6lAmAcgRjmFy7LA== - dependencies: - "@types/node" "*" - -"@types/unist@*", "@types/unist@^2.0.0", "@types/unist@^2.0.2", "@types/unist@^2.0.3": - version "2.0.3" - resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.3.tgz#9c088679876f374eb5983f150d4787aa6fb32d7e" - integrity sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ== - -"@webassemblyjs/ast@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.1.tgz#2bfd767eae1a6996f432ff7e8d7fc75679c0b6a7" - integrity sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw== - dependencies: - "@webassemblyjs/helper-numbers" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - -"@webassemblyjs/floating-point-hex-parser@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz#f6c61a705f0fd7a6aecaa4e8198f23d9dc179e4f" - integrity sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ== - -"@webassemblyjs/helper-api-error@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz#1a63192d8788e5c012800ba6a7a46c705288fd16" - integrity sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg== - -"@webassemblyjs/helper-buffer@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz#832a900eb444884cde9a7cad467f81500f5e5ab5" - integrity sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA== - -"@webassemblyjs/helper-numbers@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz#64d81da219fbbba1e3bd1bfc74f6e8c4e10a62ae" - integrity sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ== - dependencies: - "@webassemblyjs/floating-point-hex-parser" "1.11.1" - "@webassemblyjs/helper-api-error" "1.11.1" - "@xtuc/long" "4.2.2" - -"@webassemblyjs/helper-wasm-bytecode@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz#f328241e41e7b199d0b20c18e88429c4433295e1" - integrity sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q== - -"@webassemblyjs/helper-wasm-section@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz#21ee065a7b635f319e738f0dd73bfbda281c097a" - integrity sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-buffer" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/wasm-gen" "1.11.1" - -"@webassemblyjs/ieee754@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz#963929e9bbd05709e7e12243a099180812992614" - integrity sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ== - dependencies: - "@xtuc/ieee754" "^1.2.0" - -"@webassemblyjs/leb128@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/leb128/-/leb128-1.11.1.tgz#ce814b45574e93d76bae1fb2644ab9cdd9527aa5" - integrity sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw== - dependencies: - "@xtuc/long" "4.2.2" - -"@webassemblyjs/utf8@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/utf8/-/utf8-1.11.1.tgz#d1f8b764369e7c6e6bae350e854dec9a59f0a3ff" - integrity sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ== - -"@webassemblyjs/wasm-edit@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz#ad206ebf4bf95a058ce9880a8c092c5dec8193d6" - integrity sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-buffer" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/helper-wasm-section" "1.11.1" - "@webassemblyjs/wasm-gen" "1.11.1" - "@webassemblyjs/wasm-opt" "1.11.1" - "@webassemblyjs/wasm-parser" "1.11.1" - "@webassemblyjs/wast-printer" "1.11.1" - -"@webassemblyjs/wasm-gen@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz#86c5ea304849759b7d88c47a32f4f039ae3c8f76" - integrity sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/ieee754" "1.11.1" - "@webassemblyjs/leb128" "1.11.1" - "@webassemblyjs/utf8" "1.11.1" - -"@webassemblyjs/wasm-opt@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz#657b4c2202f4cf3b345f8a4c6461c8c2418985f2" - integrity sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-buffer" "1.11.1" - "@webassemblyjs/wasm-gen" "1.11.1" - "@webassemblyjs/wasm-parser" "1.11.1" - -"@webassemblyjs/wasm-parser@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz#86ca734534f417e9bd3c67c7a1c75d8be41fb199" - integrity sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/helper-api-error" "1.11.1" - "@webassemblyjs/helper-wasm-bytecode" "1.11.1" - "@webassemblyjs/ieee754" "1.11.1" - "@webassemblyjs/leb128" "1.11.1" - "@webassemblyjs/utf8" "1.11.1" - -"@webassemblyjs/wast-printer@1.11.1": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz#d0c73beda8eec5426f10ae8ef55cee5e7084c2f0" - integrity sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg== - dependencies: - "@webassemblyjs/ast" "1.11.1" - "@xtuc/long" "4.2.2" - -"@xtuc/ieee754@^1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790" - integrity sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA== - -"@xtuc/long@4.2.2": - version "4.2.2" - resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d" - integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ== - -accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7: - version "1.3.7" - resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" - integrity sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA== - dependencies: - mime-types "~2.1.24" - negotiator "0.6.2" - -acorn-import-assertions@^1.7.6: - version "1.8.0" - resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz#ba2b5939ce62c238db6d93d81c9b111b29b855e9" - integrity sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw== - -acorn-walk@^8.0.0: - version "8.1.0" - resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.1.0.tgz#d3c6a9faf00987a5e2b9bdb506c2aa76cd707f83" - integrity sha512-mjmzmv12YIG/G8JQdQuz2MUDShEJ6teYpT5bmWA4q7iwoGen8xtt3twF3OvzIUl+Q06aWIjvnwQUKvQ6TtMRjg== - -acorn@^8.0.4: - version "8.2.1" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.2.1.tgz#0d36af126fb6755095879c1dc6fd7edf7d60a5fb" - integrity sha512-z716cpm5TX4uzOzILx8PavOE6C6DKshHDw1aQN52M/yNSqE9s5O8SMfyhCCfCJ3HmTL0NkVOi+8a/55T7YB3bg== - -acorn@^8.4.1: - version "8.5.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.5.0.tgz#4512ccb99b3698c752591e9bb4472e38ad43cee2" - integrity sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q== - -address@1.1.2, address@^1.0.1: - version "1.1.2" - resolved "https://registry.yarnpkg.com/address/-/address-1.1.2.tgz#bf1116c9c758c51b7a933d296b72c221ed9428b6" - integrity sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA== - -aggregate-error@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" - integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== - dependencies: - clean-stack "^2.0.0" - indent-string "^4.0.0" - -ajv-errors@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/ajv-errors/-/ajv-errors-1.0.1.tgz#f35986aceb91afadec4102fbd85014950cefa64d" - integrity sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ== - -ajv-formats@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520" - integrity sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA== - dependencies: - ajv "^8.0.0" - -ajv-keywords@^3.1.0, ajv-keywords@^3.5.2: - version "3.5.2" - resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d" - integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== - -ajv-keywords@^5.0.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-5.1.0.tgz#69d4d385a4733cdbeab44964a1170a88f87f0e16" - integrity sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw== - dependencies: - fast-deep-equal "^3.1.3" - -ajv@^6.1.0, ajv@^6.12.4, ajv@^6.12.5: - version "6.12.6" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" - integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== - dependencies: - fast-deep-equal "^3.1.1" - fast-json-stable-stringify "^2.0.0" - json-schema-traverse "^0.4.1" - uri-js "^4.2.2" - -ajv@^8.0.0, ajv@^8.8.0: - version "8.8.2" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.8.2.tgz#01b4fef2007a28bf75f0b7fc009f62679de4abbb" - integrity sha512-x9VuX+R/jcFj1DHo/fCp99esgGDWiHENrKxaCENuCxpoMCmAt/COCGVDwA7kleEpEzJjDnvh3yGoOuLu0Dtllw== - dependencies: - fast-deep-equal "^3.1.1" - json-schema-traverse "^1.0.0" - require-from-string "^2.0.2" - uri-js "^4.2.2" - -algoliasearch-helper@^3.3.4: - version "3.6.2" - resolved "https://registry.yarnpkg.com/algoliasearch-helper/-/algoliasearch-helper-3.6.2.tgz#45e19b12589cfa0c611b573287f65266ea2cc14a" - integrity sha512-Xx0NOA6k4ySn+R2l3UMSONAaMkyfmrZ3AP1geEMo32MxDJQJesZABZYsldO9fa6FKQxH91afhi4hO1G0Zc2opg== - dependencies: - events "^1.1.1" - -algoliasearch@^4.0.0: - version "4.9.1" - resolved "https://registry.yarnpkg.com/algoliasearch/-/algoliasearch-4.9.1.tgz#1fa8ece3f9808e465226176b88b953801c2274e0" - integrity sha512-EeJUYXzBEhZSsL6tXc3hseLBCtlNLa1MZ4mlMK6EeX38yRjY5vgnFcNNml6uUhlOjvheKxgkKRpPWkxgL8Cqkg== - dependencies: - "@algolia/cache-browser-local-storage" "4.9.1" - "@algolia/cache-common" "4.9.1" - "@algolia/cache-in-memory" "4.9.1" - "@algolia/client-account" "4.9.1" - "@algolia/client-analytics" "4.9.1" - "@algolia/client-common" "4.9.1" - "@algolia/client-recommendation" "4.9.1" - "@algolia/client-search" "4.9.1" - "@algolia/logger-common" "4.9.1" - "@algolia/logger-console" "4.9.1" - "@algolia/requester-browser-xhr" "4.9.1" - "@algolia/requester-common" "4.9.1" - "@algolia/requester-node-http" "4.9.1" - "@algolia/transporter" "4.9.1" - -algoliasearch@^4.8.4: - version "4.11.0" - resolved "https://registry.yarnpkg.com/algoliasearch/-/algoliasearch-4.11.0.tgz#234befb3ac355c094077f0edf3777240b1ee013c" - integrity sha512-IXRj8kAP2WrMmj+eoPqPc6P7Ncq1yZkFiyDrjTBObV1ADNL8Z/KdZ+dWC5MmYcBLAbcB/mMCpak5N/D1UIZvsA== - dependencies: - "@algolia/cache-browser-local-storage" "4.11.0" - "@algolia/cache-common" "4.11.0" - "@algolia/cache-in-memory" "4.11.0" - "@algolia/client-account" "4.11.0" - "@algolia/client-analytics" "4.11.0" - "@algolia/client-common" "4.11.0" - "@algolia/client-personalization" "4.11.0" - "@algolia/client-search" "4.11.0" - "@algolia/logger-common" "4.11.0" - "@algolia/logger-console" "4.11.0" - "@algolia/requester-browser-xhr" "4.11.0" - "@algolia/requester-common" "4.11.0" - "@algolia/requester-node-http" "4.11.0" - "@algolia/transporter" "4.11.0" - -alphanum-sort@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/alphanum-sort/-/alphanum-sort-1.0.2.tgz#97a1119649b211ad33691d9f9f486a8ec9fbe0a3" - integrity sha1-l6ERlkmyEa0zaR2fn0hqjsn74KM= - -ansi-align@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/ansi-align/-/ansi-align-3.0.0.tgz#b536b371cf687caaef236c18d3e21fe3797467cb" - integrity sha512-ZpClVKqXN3RGBmKibdfWzqCY4lnjEuoNzU5T0oEFpfd/z5qJHVarukridD4juLO2FXMiwUQxr9WqQtaYa8XRYw== - dependencies: - string-width "^3.0.0" - -ansi-colors@^3.0.0: - version "3.2.4" - resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-3.2.4.tgz#e3a3da4bfbae6c86a9c285625de124a234026fbf" - integrity sha512-hHUXGagefjN2iRrID63xckIvotOXOojhQKWIPUZ4mNUZ9nLZW+7FMNoE1lOkEhNWYsx/7ysGIuJYCiMAA9FnrA== - -ansi-escapes@^4.3.1: - version "4.3.2" - resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" - integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== - dependencies: - type-fest "^0.21.3" - -ansi-html-community@0.0.8: - version "0.0.8" - resolved "https://registry.yarnpkg.com/ansi-html-community/-/ansi-html-community-0.0.8.tgz#69fbc4d6ccbe383f9736934ae34c3f8290f1bf41" - integrity sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw== - -ansi-regex@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" - integrity sha1-w7M6te42DYbg5ijwRorn7yfWVN8= - -ansi-regex@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997" - integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== - -ansi-regex@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" - integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== - -ansi-regex@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" - integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== - -ansi-styles@^3.2.0, ansi-styles@^3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" - integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== - dependencies: - color-convert "^1.9.0" - -ansi-styles@^4.0.0, ansi-styles@^4.1.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" - integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== - dependencies: - color-convert "^2.0.1" - -anymatch@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-2.0.0.tgz#bcb24b4f37934d9aa7ac17b4adaf89e7c76ef2eb" - integrity sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw== - dependencies: - micromatch "^3.1.4" - normalize-path "^2.1.1" - -anymatch@~3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" - integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg== - dependencies: - normalize-path "^3.0.0" - picomatch "^2.0.4" - -arg@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.0.tgz#a20e2bb5710e82950a516b3f933fee5ed478be90" - integrity sha512-4P8Zm2H+BRS+c/xX1LrHw0qKpEhdlZjLCgWy+d78T9vqa2Z2SiD2wMrYuWIAFy5IZUD7nnNXroRttz+0RzlrzQ== - -argparse@^1.0.7: - version "1.0.10" - resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" - integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== - dependencies: - sprintf-js "~1.0.2" - -argparse@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" - integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== - -arr-diff@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-4.0.0.tgz#d6461074febfec71e7e15235761a329a5dc7c520" - integrity sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA= - -arr-flatten@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/arr-flatten/-/arr-flatten-1.1.0.tgz#36048bbff4e7b47e136644316c99669ea5ae91f1" - integrity sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg== - -arr-union@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/arr-union/-/arr-union-3.1.0.tgz#e39b09aea9def866a8f206e288af63919bae39c4" - integrity sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ= - -array-flatten@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" - integrity sha1-ml9pkFGx5wczKPKgCJaLZOopVdI= - -array-flatten@^2.1.0: - version "2.1.2" - resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-2.1.2.tgz#24ef80a28c1a893617e2149b0c6d0d788293b099" - integrity sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ== - -array-union@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/array-union/-/array-union-1.0.2.tgz#9a34410e4f4e3da23dea375be5be70f24778ec39" - integrity sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk= - dependencies: - array-uniq "^1.0.1" - -array-union@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" - integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== - -array-uniq@^1.0.1: - version "1.0.3" - resolved "https://registry.yarnpkg.com/array-uniq/-/array-uniq-1.0.3.tgz#af6ac877a25cc7f74e058894753858dfdb24fdb6" - integrity sha1-r2rId6Jcx/dOBYiUdThY39sk/bY= - -array-unique@^0.3.2: - version "0.3.2" - resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" - integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg= - -asap@~2.0.3: - version "2.0.6" - resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" - integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= - -assign-symbols@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367" - integrity sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c= - -async-each@^1.0.1: - version "1.0.3" - resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.3.tgz#b727dbf87d7651602f06f4d4ac387f47d91b0cbf" - integrity sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ== - -async-limiter@~1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd" - integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ== - -async@^2.6.2: - version "2.6.3" - resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff" - integrity sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg== - dependencies: - lodash "^4.17.14" - -atob@^2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" - integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== - -autoprefixer@^10.2.5, autoprefixer@^10.3.7: - version "10.4.0" - resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-10.4.0.tgz#c3577eb32a1079a440ec253e404eaf1eb21388c8" - integrity sha512-7FdJ1ONtwzV1G43GDD0kpVMn/qbiNqyOPMFTX5nRffI+7vgWoFEc6DcXOxHJxrWNDXrZh18eDsZjvZGUljSRGA== - dependencies: - browserslist "^4.17.5" - caniuse-lite "^1.0.30001272" - fraction.js "^4.1.1" - normalize-range "^0.1.2" - picocolors "^1.0.0" - postcss-value-parser "^4.1.0" - -axios@^0.21.1: - version "0.21.4" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" - integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== - dependencies: - follow-redirects "^1.14.0" - -babel-loader@^8.2.2: - version "8.2.2" - resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-8.2.2.tgz#9363ce84c10c9a40e6c753748e1441b60c8a0b81" - integrity sha512-JvTd0/D889PQBtUXJ2PXaKU/pjZDMtHA9V2ecm+eNRmmBCMR09a+fmpGTNwnJtFmFl5Ei7Vy47LjBb+L0wQ99g== - dependencies: - find-cache-dir "^3.3.1" - loader-utils "^1.4.0" - make-dir "^3.1.0" - schema-utils "^2.6.5" - -babel-plugin-apply-mdx-type-prop@1.6.22: - version "1.6.22" - resolved "https://registry.yarnpkg.com/babel-plugin-apply-mdx-type-prop/-/babel-plugin-apply-mdx-type-prop-1.6.22.tgz#d216e8fd0de91de3f1478ef3231e05446bc8705b" - integrity sha512-VefL+8o+F/DfK24lPZMtJctrCVOfgbqLAGZSkxwhazQv4VxPg3Za/i40fu22KR2m8eEda+IfSOlPLUSIiLcnCQ== - dependencies: - "@babel/helper-plugin-utils" "7.10.4" - "@mdx-js/util" "1.6.22" - -babel-plugin-dynamic-import-node@2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.0.tgz#f00f507bdaa3c3e3ff6e7e5e98d90a7acab96f7f" - integrity sha512-o6qFkpeQEBxcqt0XYlWzAVxNCSCZdUgcR8IRlhD/8DylxjjO4foPcvTW0GGKa/cVt3rvxZ7o5ippJ+/0nvLhlQ== - dependencies: - object.assign "^4.1.0" - -babel-plugin-dynamic-import-node@^2.3.3: - version "2.3.3" - resolved "https://registry.yarnpkg.com/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz#84fda19c976ec5c6defef57f9427b3def66e17a3" - integrity sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ== - dependencies: - object.assign "^4.1.0" - -babel-plugin-extract-import-names@1.6.22: - version "1.6.22" - resolved "https://registry.yarnpkg.com/babel-plugin-extract-import-names/-/babel-plugin-extract-import-names-1.6.22.tgz#de5f9a28eb12f3eb2578bf74472204e66d1a13dc" - integrity sha512-yJ9BsJaISua7d8zNT7oRG1ZLBJCIdZ4PZqmH8qa9N5AK01ifk3fnkc98AXhtzE7UkfCsEumvoQWgoYLhOnJ7jQ== - dependencies: - "@babel/helper-plugin-utils" "7.10.4" - -babel-plugin-polyfill-corejs2@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.0.tgz#407082d0d355ba565af24126fb6cb8e9115251fd" - integrity sha512-wMDoBJ6uG4u4PNFh72Ty6t3EgfA91puCuAwKIazbQlci+ENb/UU9A3xG5lutjUIiXCIn1CY5L15r9LimiJyrSA== - dependencies: - "@babel/compat-data" "^7.13.11" - "@babel/helper-define-polyfill-provider" "^0.3.0" - semver "^6.1.1" - -babel-plugin-polyfill-corejs3@^0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.4.0.tgz#0b571f4cf3d67f911512f5c04842a7b8e8263087" - integrity sha512-YxFreYwUfglYKdLUGvIF2nJEsGwj+RhWSX/ije3D2vQPOXuyMLMtg/cCGMDpOA7Nd+MwlNdnGODbd2EwUZPlsw== - dependencies: - "@babel/helper-define-polyfill-provider" "^0.3.0" - core-js-compat "^3.18.0" - -babel-plugin-polyfill-regenerator@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.3.0.tgz#9ebbcd7186e1a33e21c5e20cae4e7983949533be" - integrity sha512-dhAPTDLGoMW5/84wkgwiLRwMnio2i1fUe53EuvtKMv0pn2p3S8OCoV1xAzfJPl0KOX7IB89s2ib85vbYiea3jg== - dependencies: - "@babel/helper-define-polyfill-provider" "^0.3.0" - -bail@^1.0.0: - version "1.0.5" - resolved "https://registry.yarnpkg.com/bail/-/bail-1.0.5.tgz#b6fa133404a392cbc1f8c4bf63f5953351e7a776" - integrity sha512-xFbRxM1tahm08yHBP16MMjVUAvDaBMD38zsM9EMAUN61omwLmKlOpB/Zku5QkjZ8TZ4vn53pj+t518cH0S03RQ== - -balanced-match@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" - integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== - -base16@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/base16/-/base16-1.0.0.tgz#e297f60d7ec1014a7a971a39ebc8a98c0b681e70" - integrity sha1-4pf2DX7BAUp6lxo568ipjAtoHnA= - -base@^0.11.1: - version "0.11.2" - resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f" - integrity sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg== - dependencies: - cache-base "^1.0.1" - class-utils "^0.3.5" - component-emitter "^1.2.1" - define-property "^1.0.0" - isobject "^3.0.1" - mixin-deep "^1.2.0" - pascalcase "^0.1.1" - -batch@0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/batch/-/batch-0.6.1.tgz#dc34314f4e679318093fc760272525f94bf25c16" - integrity sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY= - -big.js@^5.2.2: - version "5.2.2" - resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" - integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ== - -binary-extensions@^1.0.0: - version "1.13.1" - resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.13.1.tgz#598afe54755b2868a5330d2aff9d4ebb53209b65" - integrity sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw== - -binary-extensions@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" - integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== - -bindings@^1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" - integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== - dependencies: - file-uri-to-path "1.0.0" - -bluebird@^3.7.1: - version "3.7.2" - resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" - integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== - -body-parser@1.19.0: - version "1.19.0" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.0.tgz#96b2709e57c9c4e09a6fd66a8fd979844f69f08a" - integrity sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw== - dependencies: - bytes "3.1.0" - content-type "~1.0.4" - debug "2.6.9" - depd "~1.1.2" - http-errors "1.7.2" - iconv-lite "0.4.24" - on-finished "~2.3.0" - qs "6.7.0" - raw-body "2.4.0" - type-is "~1.6.17" - -bonjour@^3.5.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/bonjour/-/bonjour-3.5.0.tgz#8e890a183d8ee9a2393b3844c691a42bcf7bc9f5" - integrity sha1-jokKGD2O6aI5OzhExpGkK897yfU= - dependencies: - array-flatten "^2.1.0" - deep-equal "^1.0.1" - dns-equal "^1.0.0" - dns-txt "^2.0.2" - multicast-dns "^6.0.1" - multicast-dns-service-types "^1.1.0" - -boolbase@^1.0.0, boolbase@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" - integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24= - -boxen@^5.0.0, boxen@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/boxen/-/boxen-5.0.1.tgz#657528bdd3f59a772b8279b831f27ec2c744664b" - integrity sha512-49VBlw+PrWEF51aCmy7QIteYPIFZxSpvqBdP/2itCPPlJ49kj9zg/XPRFrdkne2W+CfwXUls8exMvu1RysZpKA== - dependencies: - ansi-align "^3.0.0" - camelcase "^6.2.0" - chalk "^4.1.0" - cli-boxes "^2.2.1" - string-width "^4.2.0" - type-fest "^0.20.2" - widest-line "^3.1.0" - wrap-ansi "^7.0.0" - -brace-expansion@^1.1.7: - version "1.1.11" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" - integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== - dependencies: - balanced-match "^1.0.0" - concat-map "0.0.1" - -braces@^2.3.1, braces@^2.3.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729" - integrity sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w== - dependencies: - arr-flatten "^1.1.0" - array-unique "^0.3.2" - extend-shallow "^2.0.1" - fill-range "^4.0.0" - isobject "^3.0.1" - repeat-element "^1.1.2" - snapdragon "^0.8.1" - snapdragon-node "^2.0.1" - split-string "^3.0.2" - to-regex "^3.0.1" - -braces@^3.0.1, braces@~3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" - integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== - dependencies: - fill-range "^7.0.1" - -browserslist@4.14.2: - version "4.14.2" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.14.2.tgz#1b3cec458a1ba87588cc5e9be62f19b6d48813ce" - integrity sha512-HI4lPveGKUR0x2StIz+2FXfDk9SfVMrxn6PLh1JeGUwcuoDkdKZebWiyLRJ68iIPDpMI4JLVDf7S7XzslgWOhw== - dependencies: - caniuse-lite "^1.0.30001125" - electron-to-chromium "^1.3.564" - escalade "^3.0.2" - node-releases "^1.1.61" - -browserslist@^4.0.0, browserslist@^4.14.5: - version "4.16.5" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.16.5.tgz#952825440bca8913c62d0021334cbe928ef062ae" - integrity sha512-C2HAjrM1AI/djrpAUU/tr4pml1DqLIzJKSLDBXBrNErl9ZCCTXdhwxdJjYc16953+mBWf7Lw+uUJgpgb8cN71A== - dependencies: - caniuse-lite "^1.0.30001214" - colorette "^1.2.2" - electron-to-chromium "^1.3.719" - escalade "^3.1.1" - node-releases "^1.1.71" - -browserslist@^4.16.0, browserslist@^4.16.6: - version "4.16.6" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.16.6.tgz#d7901277a5a88e554ed305b183ec9b0c08f66fa2" - integrity sha512-Wspk/PqO+4W9qp5iUTJsa1B/QrYn1keNCcEP5OvP7WBwT4KaDly0uONYmC6Xa3Z5IqnUgS0KcgLYu1l74x0ZXQ== - dependencies: - caniuse-lite "^1.0.30001219" - colorette "^1.2.2" - electron-to-chromium "^1.3.723" - escalade "^3.1.1" - node-releases "^1.1.71" - -browserslist@^4.17.5, browserslist@^4.18.1: - version "4.18.1" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.18.1.tgz#60d3920f25b6860eb917c6c7b185576f4d8b017f" - integrity sha512-8ScCzdpPwR2wQh8IT82CA2VgDwjHyqMovPBZSNH54+tm4Jk2pCuv90gmAdH6J84OCRWi0b4gMe6O6XPXuJnjgQ== - dependencies: - caniuse-lite "^1.0.30001280" - electron-to-chromium "^1.3.896" - escalade "^3.1.1" - node-releases "^2.0.1" - picocolors "^1.0.0" - -buffer-from@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" - integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== - -buffer-indexof@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/buffer-indexof/-/buffer-indexof-1.1.1.tgz#52fabcc6a606d1a00302802648ef68f639da268c" - integrity sha512-4/rOEg86jivtPTeOUUT61jJO1Ya1TrR/OkqCSZDyq84WJh3LuuiphBYJN+fm5xufIk4XAFcEwte/8WzC8If/1g== - -bytes@3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048" - integrity sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg= - -bytes@3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" - integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg== - -cache-base@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/cache-base/-/cache-base-1.0.1.tgz#0a7f46416831c8b662ee36fe4e7c59d76f666ab2" - integrity sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ== - dependencies: - collection-visit "^1.0.0" - component-emitter "^1.2.1" - get-value "^2.0.6" - has-value "^1.0.0" - isobject "^3.0.1" - set-value "^2.0.0" - to-object-path "^0.3.0" - union-value "^1.0.0" - unset-value "^1.0.0" - -cacheable-request@^6.0.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-6.1.0.tgz#20ffb8bd162ba4be11e9567d823db651052ca912" - integrity sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg== - dependencies: - clone-response "^1.0.2" - get-stream "^5.1.0" - http-cache-semantics "^4.0.0" - keyv "^3.0.0" - lowercase-keys "^2.0.0" - normalize-url "^4.1.0" - responselike "^1.0.2" - -call-bind@^1.0.0, call-bind@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" - integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== - dependencies: - function-bind "^1.1.1" - get-intrinsic "^1.0.2" - -callsites@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" - integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== - -camel-case@^4.1.1, camel-case@^4.1.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/camel-case/-/camel-case-4.1.2.tgz#9728072a954f805228225a6deea6b38461e1bd5a" - integrity sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw== - dependencies: - pascal-case "^3.1.2" - tslib "^2.0.3" - -camelcase-css@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/camelcase-css/-/camelcase-css-2.0.1.tgz#ee978f6947914cc30c6b44741b6ed1df7f043fd5" - integrity sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA== - -camelcase@^5.0.0: - version "5.3.1" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" - integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== - -camelcase@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.2.0.tgz#924af881c9d525ac9d87f40d964e5cea982a1809" - integrity sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg== - -caniuse-api@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/caniuse-api/-/caniuse-api-3.0.0.tgz#5e4d90e2274961d46291997df599e3ed008ee4c0" - integrity sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw== - dependencies: - browserslist "^4.0.0" - caniuse-lite "^1.0.0" - lodash.memoize "^4.1.2" - lodash.uniq "^4.5.0" - -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001125, caniuse-lite@^1.0.30001214, caniuse-lite@^1.0.30001219, caniuse-lite@^1.0.30001272, caniuse-lite@^1.0.30001280: - version "1.0.30001303" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001303.tgz" - integrity sha512-/Mqc1oESndUNszJP0kx0UaQU9kEv9nNtJ7Kn8AdA0mNnH8eR1cj0kG+NbNuC1Wq/b21eA8prhKRA3bbkjONegQ== - -ccount@^1.0.0, ccount@^1.0.3: - version "1.1.0" - resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.1.0.tgz#246687debb6014735131be8abab2d93898f8d043" - integrity sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg== - -chalk@2.4.2, chalk@^2.0.0, chalk@^2.4.1: - version "2.4.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" - integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== - dependencies: - ansi-styles "^3.2.1" - escape-string-regexp "^1.0.5" - supports-color "^5.3.0" - -chalk@^4.1.0, chalk@^4.1.1: - version "4.1.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" - integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== - dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" - -character-entities-legacy@^1.0.0: - version "1.1.4" - resolved "https://registry.yarnpkg.com/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz#94bc1845dce70a5bb9d2ecc748725661293d8fc1" - integrity sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA== - -character-entities@^1.0.0: - version "1.2.4" - resolved "https://registry.yarnpkg.com/character-entities/-/character-entities-1.2.4.tgz#e12c3939b7eaf4e5b15e7ad4c5e28e1d48c5b16b" - integrity sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw== - -character-reference-invalid@^1.0.0: - version "1.1.4" - resolved "https://registry.yarnpkg.com/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz#083329cda0eae272ab3dbbf37e9a382c13af1560" - integrity sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg== - -cheerio@^0.22.0: - version "0.22.0" - resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-0.22.0.tgz#a9baa860a3f9b595a6b81b1a86873121ed3a269e" - integrity sha1-qbqoYKP5tZWmuBsahocxIe06Jp4= - dependencies: - css-select "~1.2.0" - dom-serializer "~0.1.0" - entities "~1.1.1" - htmlparser2 "^3.9.1" - lodash.assignin "^4.0.9" - lodash.bind "^4.1.4" - lodash.defaults "^4.0.1" - lodash.filter "^4.4.0" - lodash.flatten "^4.2.0" - lodash.foreach "^4.3.0" - lodash.map "^4.4.0" - lodash.merge "^4.4.0" - lodash.pick "^4.2.1" - lodash.reduce "^4.4.0" - lodash.reject "^4.4.0" - lodash.some "^4.4.0" - -chokidar@^2.1.8: - version "2.1.8" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917" - integrity sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg== - dependencies: - anymatch "^2.0.0" - async-each "^1.0.1" - braces "^2.3.2" - glob-parent "^3.1.0" - inherits "^2.0.3" - is-binary-path "^1.0.0" - is-glob "^4.0.0" - normalize-path "^3.0.0" - path-is-absolute "^1.0.0" - readdirp "^2.2.1" - upath "^1.1.1" - optionalDependencies: - fsevents "^1.2.7" - -chokidar@^3.5.1: - version "3.5.2" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.2.tgz#dba3976fcadb016f66fd365021d91600d01c1e75" - integrity sha512-ekGhOnNVPgT77r4K/U3GDhu+FQ2S8TnK/s2KbIGXi0SZWuwkZ2QNyfWdZW+TVfn84DpEP7rLeCt2UI6bJ8GwbQ== - dependencies: - anymatch "~3.1.2" - braces "~3.0.2" - glob-parent "~5.1.2" - is-binary-path "~2.1.0" - is-glob "~4.0.1" - normalize-path "~3.0.0" - readdirp "~3.6.0" - optionalDependencies: - fsevents "~2.3.2" - -chrome-trace-event@^1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz#1015eced4741e15d06664a957dbbf50d041e26ac" - integrity sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg== - -ci-info@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" - integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ== - -ci-info@^3.0.0: - version "3.1.1" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.1.1.tgz#9a32fcefdf7bcdb6f0a7e1c0f8098ec57897b80a" - integrity sha512-kdRWLBIJwdsYJWYJFtAFFYxybguqeF91qpZaggjG5Nf8QKdizFG2hjqvaTXbxFIcYbSaD74KpAXv6BSm17DHEQ== - -class-utils@^0.3.5: - version "0.3.6" - resolved "https://registry.yarnpkg.com/class-utils/-/class-utils-0.3.6.tgz#f93369ae8b9a7ce02fd41faad0ca83033190c463" - integrity sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg== - dependencies: - arr-union "^3.1.0" - define-property "^0.2.5" - isobject "^3.0.0" - static-extend "^0.1.1" - -clean-css@^4.2.3: - version "4.2.4" - resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-4.2.4.tgz#733bf46eba4e607c6891ea57c24a989356831178" - integrity sha512-EJUDT7nDVFDvaQgAo2G/PJvxmp1o/c6iXLbswsBbUFXi1Nr+AjA2cKmfbKDMjMvzEe75g3P6JkaDDAKk96A85A== - dependencies: - source-map "~0.6.0" - -clean-css@^5.1.2: - version "5.2.2" - resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-5.2.2.tgz#d3a7c6ee2511011e051719838bdcf8314dc4548d" - integrity sha512-/eR8ru5zyxKzpBLv9YZvMXgTSSQn7AdkMItMYynsFgGwTveCRVam9IUPFloE85B4vAIj05IuKmmEoV7/AQjT0w== - dependencies: - source-map "~0.6.0" - -clean-css@^5.1.5: - version "5.1.5" - resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-5.1.5.tgz#3b0af240dcfc9a3779a08c2332df3ebd4474f232" - integrity sha512-9dr/cU/LjMpU57PXlSvDkVRh0rPxJBXiBtD0+SgYt8ahTCsXtfKjCkNYgIoTC6mBg8CFr5EKhW3DKCaGMUbUfQ== - dependencies: - source-map "~0.6.0" - -clean-stack@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" - integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== - -cli-boxes@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.1.tgz#ddd5035d25094fce220e9cab40a45840a440318f" - integrity sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw== - -cliui@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/cliui/-/cliui-5.0.0.tgz#deefcfdb2e800784aa34f46fa08e06851c7bbbc5" - integrity sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA== - dependencies: - string-width "^3.1.0" - strip-ansi "^5.2.0" - wrap-ansi "^5.1.0" - -clone-deep@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-4.0.1.tgz#c19fd9bdbbf85942b4fd979c84dcf7d5f07c2387" - integrity sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ== - dependencies: - is-plain-object "^2.0.4" - kind-of "^6.0.2" - shallow-clone "^3.0.0" - -clone-response@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/clone-response/-/clone-response-1.0.2.tgz#d1dc973920314df67fbeb94223b4ee350239e96b" - integrity sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws= - dependencies: - mimic-response "^1.0.0" - -clsx@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.1.1.tgz#98b3134f9abbdf23b2663491ace13c5c03a73188" - integrity sha512-6/bPho624p3S2pMyvP5kKBPXnI3ufHLObBFCfgx+LkeR5lg2XYy2hqZqUf45ypD8COn2bhgGJSUE+l5dhNBieA== - -coa@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/coa/-/coa-2.0.2.tgz#43f6c21151b4ef2bf57187db0d73de229e3e7ec3" - integrity sha512-q5/jG+YQnSy4nRTV4F7lPepBJZ8qBNJJDBuJdoejDyLXgmL7IEo+Le2JDZudFTFt7mrCqIRaSjws4ygRCTCAXA== - dependencies: - "@types/q" "^1.5.1" - chalk "^2.4.1" - q "^1.1.2" - -collapse-white-space@^1.0.2: - version "1.0.6" - resolved "https://registry.yarnpkg.com/collapse-white-space/-/collapse-white-space-1.0.6.tgz#e63629c0016665792060dbbeb79c42239d2c5287" - integrity sha512-jEovNnrhMuqyCcjfEJA56v0Xq8SkIoPKDyaHahwo3POf4qcSXqMYuwNcOTzp74vTsR9Tn08z4MxWqAhcekogkQ== - -collection-visit@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0" - integrity sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA= - dependencies: - map-visit "^1.0.0" - object-visit "^1.0.0" - -color-convert@^1.9.0: - version "1.9.3" - resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" - integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== - dependencies: - color-name "1.1.3" - -color-convert@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" - integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== - dependencies: - color-name "~1.1.4" - -color-name@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" - integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= - -color-name@~1.1.4: - version "1.1.4" - resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" - integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== - -colord@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/colord/-/colord-2.0.1.tgz#1e7fb1f9fa1cf74f42c58cb9c20320bab8435aa0" - integrity sha512-vm5YpaWamD0Ov6TSG0GGmUIwstrWcfKQV/h2CmbR7PbNu41+qdB5PW9lpzhjedrpm08uuYvcXi0Oel1RLZIJuA== - -colord@^2.6: - version "2.8.0" - resolved "https://registry.yarnpkg.com/colord/-/colord-2.8.0.tgz#64fb7aa03de7652b5a39eee50271a104c2783b12" - integrity sha512-kNkVV4KFta3TYQv0bzs4xNwLaeag261pxgzGQSh4cQ1rEhYjcTJfFRP0SDlbhLONg0eSoLzrDd79PosjbltufA== - -colord@^2.9.1: - version "2.9.1" - resolved "https://registry.yarnpkg.com/colord/-/colord-2.9.1.tgz#c961ea0efeb57c9f0f4834458f26cb9cc4a3f90e" - integrity sha512-4LBMSt09vR0uLnPVkOUBnmxgoaeN4ewRbx801wY/bXcltXfpR/G46OdWn96XpYmCWuYvO46aBZP4NgX8HpNAcw== - -colorette@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94" - integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w== - -combine-promises@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/combine-promises/-/combine-promises-1.1.0.tgz#72db90743c0ca7aab7d0d8d2052fd7b0f674de71" - integrity sha512-ZI9jvcLDxqwaXEixOhArm3r7ReIivsXkpbyEWyeOhzz1QS0iSgBPnWvEqvIQtYyamGCYA88gFhmUrs9hrrQ0pg== - -comma-separated-tokens@^1.0.0: - version "1.0.8" - resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz#632b80b6117867a158f1080ad498b2fbe7e3f5ea" - integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw== - -commander@^2.20.0: - version "2.20.3" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" - integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== - -commander@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068" - integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA== - -commander@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae" - integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg== - -commander@^6.2.0: - version "6.2.1" - resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c" - integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA== - -commander@^7.1.0, commander@^7.2.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" - integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== - -commander@^8.1.0: - version "8.2.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-8.2.0.tgz#37fe2bde301d87d47a53adeff8b5915db1381ca8" - integrity sha512-LLKxDvHeL91/8MIyTAD5BFMNtoIwztGPMiM/7Bl8rIPmHCZXRxmSWr91h57dpOpnQ6jIUqEWdXE/uBYMfiVZDA== - -commondir@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" - integrity sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs= - -component-emitter@^1.2.1: - version "1.3.0" - resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0" - integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg== - -compressible@~2.0.16: - version "2.0.18" - resolved "https://registry.yarnpkg.com/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba" - integrity sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg== - dependencies: - mime-db ">= 1.43.0 < 2" - -compression@^1.7.4: - version "1.7.4" - resolved "https://registry.yarnpkg.com/compression/-/compression-1.7.4.tgz#95523eff170ca57c29a0ca41e6fe131f41e5bb8f" - integrity sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ== - dependencies: - accepts "~1.3.5" - bytes "3.0.0" - compressible "~2.0.16" - debug "2.6.9" - on-headers "~1.0.2" - safe-buffer "5.1.2" - vary "~1.1.2" - -concat-map@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" - integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= - -configstore@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/configstore/-/configstore-5.0.1.tgz#d365021b5df4b98cdd187d6a3b0e3f6a7cc5ed96" - integrity sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA== - dependencies: - dot-prop "^5.2.0" - graceful-fs "^4.1.2" - make-dir "^3.0.0" - unique-string "^2.0.0" - write-file-atomic "^3.0.0" - xdg-basedir "^4.0.0" - -connect-history-api-fallback@^1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/connect-history-api-fallback/-/connect-history-api-fallback-1.6.0.tgz#8b32089359308d111115d81cad3fceab888f97bc" - integrity sha512-e54B99q/OUoH64zYYRf3HBP5z24G38h5D3qXu23JGRoigpX5Ss4r9ZnDk3g0Z8uQC2x2lPaJ+UlWBc1ZWBWdLg== - -consola@^2.15.0: - version "2.15.3" - resolved "https://registry.yarnpkg.com/consola/-/consola-2.15.3.tgz#2e11f98d6a4be71ff72e0bdf07bd23e12cb61550" - integrity sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw== - -content-disposition@0.5.2: - version "0.5.2" - resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.2.tgz#0cf68bb9ddf5f2be7961c3a85178cb85dba78cb4" - integrity sha1-DPaLud318r55YcOoUXjLhdunjLQ= - -content-disposition@0.5.3: - version "0.5.3" - resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd" - integrity sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g== - dependencies: - safe-buffer "5.1.2" - -content-type@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" - integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== - -convert-source-map@^1.7.0: - version "1.7.0" - resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.7.0.tgz#17a2cb882d7f77d3490585e2ce6c524424a3a442" - integrity sha512-4FJkXzKXEDB1snCFZlLP4gpC3JILicCpGbzG9f9G7tGqGCzETQ2hWPrcinA9oU4wtf2biUaEH5065UnMeR33oA== - dependencies: - safe-buffer "~5.1.1" - -cookie-signature@1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" - integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw= - -cookie@0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.0.tgz#beb437e7022b3b6d49019d088665303ebe9c14ba" - integrity sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg== - -copy-descriptor@^0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d" - integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40= - -copy-text-to-clipboard@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/copy-text-to-clipboard/-/copy-text-to-clipboard-3.0.1.tgz#8cbf8f90e0a47f12e4a24743736265d157bce69c" - integrity sha512-rvVsHrpFcL4F2P8ihsoLdFHmd404+CMg71S756oRSeQgqk51U3kicGdnvfkrxva0xXH92SjGS62B0XIJsbh+9Q== - -copy-webpack-plugin@^9.0.0: - version "9.1.0" - resolved "https://registry.yarnpkg.com/copy-webpack-plugin/-/copy-webpack-plugin-9.1.0.tgz#2d2c460c4c4695ec0a58afb2801a1205256c4e6b" - integrity sha512-rxnR7PaGigJzhqETHGmAcxKnLZSR5u1Y3/bcIv/1FnqXedcL/E2ewK7ZCNrArJKCiSv8yVXhTqetJh8inDvfsA== - dependencies: - fast-glob "^3.2.7" - glob-parent "^6.0.1" - globby "^11.0.3" - normalize-path "^3.0.0" - schema-utils "^3.1.1" - serialize-javascript "^6.0.0" - -core-js-compat@^3.18.0, core-js-compat@^3.19.1: - version "3.19.3" - resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.19.3.tgz#de75e5821c5ce924a0a1e7b7d5c2cb973ff388aa" - integrity sha512-59tYzuWgEEVU9r+SRgceIGXSSUn47JknoiXW6Oq7RW8QHjXWz3/vp8pa7dbtuVu40sewz3OP3JmQEcDdztrLhA== - dependencies: - browserslist "^4.18.1" - semver "7.0.0" - -core-js-pure@^3.19.0: - version "3.19.3" - resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.19.3.tgz#c69b2b36b58927317824994b532ec3f0f7e49607" - integrity sha512-N3JruInmCyt7EJj5mAq3csCgGYgiSqu7p7TQp2KOztr180/OAIxyIvL1FCjzgmQk/t3Yniua50Fsak7FShI9lA== - -core-js@^3.9.1: - version "3.19.3" - resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.19.3.tgz#6df8142a996337503019ff3235a7022d7cdf4559" - integrity sha512-LeLBMgEGSsG7giquSzvgBrTS7V5UL6ks3eQlUSbN8dJStlLFiRzUm5iqsRyzUB8carhfKjkJ2vzKqE6z1Vga9g== - -core-util-is@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" - integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= - -cosmiconfig@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.0.0.tgz#ef9b44d773959cae63ddecd122de23853b60f8d3" - integrity sha512-pondGvTuVYDk++upghXJabWzL6Kxu6f26ljFw64Swq9v6sQPUL3EUlVDV56diOjpCayKihL6hVe8exIACU4XcA== - dependencies: - "@types/parse-json" "^4.0.0" - import-fresh "^3.2.1" - parse-json "^5.0.0" - path-type "^4.0.0" - yaml "^1.10.0" - -cross-fetch@^3.0.4: - version "3.1.4" - resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.4.tgz#9723f3a3a247bf8b89039f3a380a9244e8fa2f39" - integrity sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ== - dependencies: - node-fetch "2.6.1" - -cross-spawn@7.0.3, cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== - dependencies: - path-key "^3.1.0" - shebang-command "^2.0.0" - which "^2.0.1" - -cross-spawn@^6.0.0: - version "6.0.5" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" - integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ== - dependencies: - nice-try "^1.0.4" - path-key "^2.0.1" - semver "^5.5.0" - shebang-command "^1.2.0" - which "^1.2.9" - -crypto-random-string@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" - integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA== - -css-color-names@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/css-color-names/-/css-color-names-1.0.1.tgz#6ff7ee81a823ad46e020fa2fd6ab40a887e2ba67" - integrity sha512-/loXYOch1qU1biStIFsHH8SxTmOseh1IJqFvy8IujXOm1h+QjUdDhkzOrR5HG8K8mlxREj0yfi8ewCHx0eMxzA== - -css-declaration-sorter@^6.0.3: - version "6.0.3" - resolved "https://registry.yarnpkg.com/css-declaration-sorter/-/css-declaration-sorter-6.0.3.tgz#9dfd8ea0df4cc7846827876fafb52314890c21a9" - integrity sha512-52P95mvW1SMzuRZegvpluT6yEv0FqQusydKQPZsNN5Q7hh8EwQvN8E2nwuJ16BBvNN6LcoIZXu/Bk58DAhrrxw== - dependencies: - timsort "^0.3.0" - -css-loader@^5.1.1: - version "5.2.4" - resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-5.2.4.tgz#e985dcbce339812cb6104ef3670f08f9893a1536" - integrity sha512-OFYGyINCKkdQsTrSYxzGSFnGS4gNjcXkKkQgWxK138jgnPt+lepxdjSZNc8sHAl5vP3DhsJUxufWIjOwI8PMMw== - dependencies: - camelcase "^6.2.0" - icss-utils "^5.1.0" - loader-utils "^2.0.0" - postcss "^8.2.10" - postcss-modules-extract-imports "^3.0.0" - postcss-modules-local-by-default "^4.0.0" - postcss-modules-scope "^3.0.0" - postcss-modules-values "^4.0.0" - postcss-value-parser "^4.1.0" - schema-utils "^3.0.0" - semver "^7.3.5" - -css-minimizer-webpack-plugin@^3.0.1: - version "3.2.0" - resolved "https://registry.yarnpkg.com/css-minimizer-webpack-plugin/-/css-minimizer-webpack-plugin-3.2.0.tgz#f59c56ec11137b37f000081bd19b450095094ad2" - integrity sha512-5q4myvkmm29jRlI73Fl8Mc008i6o6hCEKnV6/fOrzRVDWD6EFGwDRX+SM2qCVeZ7XiztRDKHpTGDUeUMAOOagg== - dependencies: - "@types/cssnano" "^4.0.1" - cssnano "^5.0.6" - jest-worker "^27.0.2" - postcss "^8.3.5" - schema-utils "^4.0.0" - serialize-javascript "^6.0.0" - source-map "^0.6.1" - -css-select-base-adapter@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/css-select-base-adapter/-/css-select-base-adapter-0.1.1.tgz#3b2ff4972cc362ab88561507a95408a1432135d7" - integrity sha512-jQVeeRG70QI08vSTwf1jHxp74JoZsr2XSgETae8/xC8ovSnL2WF87GTLO86Sbwdt2lK4Umg4HnnwMO4YF3Ce7w== - -css-select@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/css-select/-/css-select-2.1.0.tgz#6a34653356635934a81baca68d0255432105dbef" - integrity sha512-Dqk7LQKpwLoH3VovzZnkzegqNSuAziQyNZUcrdDM401iY+R5NkGBXGmtO05/yaXQziALuPogeG0b7UAgjnTJTQ== - dependencies: - boolbase "^1.0.0" - css-what "^3.2.1" - domutils "^1.7.0" - nth-check "^1.0.2" - -css-select@^3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/css-select/-/css-select-3.1.2.tgz#d52cbdc6fee379fba97fb0d3925abbd18af2d9d8" - integrity sha512-qmss1EihSuBNWNNhHjxzxSfJoFBM/lERB/Q4EnsJQQC62R2evJDW481091oAdOr9uh46/0n4nrg0It5cAnj1RA== - dependencies: - boolbase "^1.0.0" - css-what "^4.0.0" - domhandler "^4.0.0" - domutils "^2.4.3" - nth-check "^2.0.0" - -css-select@^4.1.3: - version "4.1.3" - resolved "https://registry.yarnpkg.com/css-select/-/css-select-4.1.3.tgz#a70440f70317f2669118ad74ff105e65849c7067" - integrity sha512-gT3wBNd9Nj49rAbmtFHj1cljIAOLYSX1nZ8CB7TBO3INYckygm5B7LISU/szY//YmdiSLbJvDLOx9VnMVpMBxA== - dependencies: - boolbase "^1.0.0" - css-what "^5.0.0" - domhandler "^4.2.0" - domutils "^2.6.0" - nth-check "^2.0.0" - -css-select@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858" - integrity sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg= - dependencies: - boolbase "~1.0.0" - css-what "2.1" - domutils "1.5.1" - nth-check "~1.0.1" - -css-tree@1.0.0-alpha.37: - version "1.0.0-alpha.37" - resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.0.0-alpha.37.tgz#98bebd62c4c1d9f960ec340cf9f7522e30709a22" - integrity sha512-DMxWJg0rnz7UgxKT0Q1HU/L9BeJI0M6ksor0OgqOnF+aRCDWg/N2641HmVyU9KVIu0OVVWOb2IpC9A+BJRnejg== - dependencies: - mdn-data "2.0.4" - source-map "^0.6.1" - -css-tree@^1.1.2, css-tree@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.1.3.tgz#eb4870fb6fd7707327ec95c2ff2ab09b5e8db91d" - integrity sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q== - dependencies: - mdn-data "2.0.14" - source-map "^0.6.1" - -css-what@2.1: - version "2.1.3" - resolved "https://registry.yarnpkg.com/css-what/-/css-what-2.1.3.tgz#a6d7604573365fe74686c3f311c56513d88285f2" - integrity sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg== - -css-what@^3.2.1: - version "3.4.2" - resolved "https://registry.yarnpkg.com/css-what/-/css-what-3.4.2.tgz#ea7026fcb01777edbde52124e21f327e7ae950e4" - integrity sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ== - -css-what@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/css-what/-/css-what-4.0.0.tgz#35e73761cab2eeb3d3661126b23d7aa0e8432233" - integrity sha512-teijzG7kwYfNVsUh2H/YN62xW3KK9YhXEgSlbxMlcyjPNvdKJqFx5lrwlJgoFP1ZHlB89iGDlo/JyshKeRhv5A== - -css-what@^5.0.0: - version "5.0.1" - resolved "https://registry.yarnpkg.com/css-what/-/css-what-5.0.1.tgz#3efa820131f4669a8ac2408f9c32e7c7de9f4cad" - integrity sha512-FYDTSHb/7KXsWICVsxdmiExPjCfRC4qRFBdVwv7Ax9hMnvMmEjP9RfxTEZ3qPZGmADDn2vAKSo9UcN1jKVYscg== - -cssesc@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-3.0.0.tgz#37741919903b868565e1c09ea747445cd18983ee" - integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg== - -cssnano-preset-advanced@^5.1.1: - version "5.1.8" - resolved "https://registry.yarnpkg.com/cssnano-preset-advanced/-/cssnano-preset-advanced-5.1.8.tgz#48678297b93d4964cd437ed1fa28d7f11ebf353c" - integrity sha512-QUR5/s3ZKX4hdrefOdTMdyzZ5RbBIt+GxcDh7bmu2yG21PqszQBtA02QG9ilsI1E7OCjkM4KlzaGtQ75GdGa3g== - dependencies: - autoprefixer "^10.3.7" - cssnano-preset-default "^5.1.8" - postcss-discard-unused "^5.0.1" - postcss-merge-idents "^5.0.1" - postcss-reduce-idents "^5.0.1" - postcss-zindex "^5.0.1" - -cssnano-preset-default@^5.1.4: - version "5.1.4" - resolved "https://registry.yarnpkg.com/cssnano-preset-default/-/cssnano-preset-default-5.1.4.tgz#359943bf00c5c8e05489f12dd25f3006f2c1cbd2" - integrity sha512-sPpQNDQBI3R/QsYxQvfB4mXeEcWuw0wGtKtmS5eg8wudyStYMgKOQT39G07EbW1LB56AOYrinRS9f0ig4Y3MhQ== - dependencies: - css-declaration-sorter "^6.0.3" - cssnano-utils "^2.0.1" - postcss-calc "^8.0.0" - postcss-colormin "^5.2.0" - postcss-convert-values "^5.0.1" - postcss-discard-comments "^5.0.1" - postcss-discard-duplicates "^5.0.1" - postcss-discard-empty "^5.0.1" - postcss-discard-overridden "^5.0.1" - postcss-merge-longhand "^5.0.2" - postcss-merge-rules "^5.0.2" - postcss-minify-font-values "^5.0.1" - postcss-minify-gradients "^5.0.2" - postcss-minify-params "^5.0.1" - postcss-minify-selectors "^5.1.0" - postcss-normalize-charset "^5.0.1" - postcss-normalize-display-values "^5.0.1" - postcss-normalize-positions "^5.0.1" - postcss-normalize-repeat-style "^5.0.1" - postcss-normalize-string "^5.0.1" - postcss-normalize-timing-functions "^5.0.1" - postcss-normalize-unicode "^5.0.1" - postcss-normalize-url "^5.0.2" - postcss-normalize-whitespace "^5.0.1" - postcss-ordered-values "^5.0.2" - postcss-reduce-initial "^5.0.1" - postcss-reduce-transforms "^5.0.1" - postcss-svgo "^5.0.2" - postcss-unique-selectors "^5.0.1" - -cssnano-preset-default@^5.1.8: - version "5.1.8" - resolved "https://registry.yarnpkg.com/cssnano-preset-default/-/cssnano-preset-default-5.1.8.tgz#7525feb1b72f7b06e57f55064cbdae341d79dea2" - integrity sha512-zWMlP0+AMPBVE852SqTrP0DnhTcTA2C1wAF92TKZ3Va+aUVqLIhkqKlnJIXXdqXD7RN+S1ujuWmNpvrJBiM/vg== - dependencies: - css-declaration-sorter "^6.0.3" - cssnano-utils "^2.0.1" - postcss-calc "^8.0.0" - postcss-colormin "^5.2.1" - postcss-convert-values "^5.0.2" - postcss-discard-comments "^5.0.1" - postcss-discard-duplicates "^5.0.1" - postcss-discard-empty "^5.0.1" - postcss-discard-overridden "^5.0.1" - postcss-merge-longhand "^5.0.4" - postcss-merge-rules "^5.0.3" - postcss-minify-font-values "^5.0.1" - postcss-minify-gradients "^5.0.3" - postcss-minify-params "^5.0.2" - postcss-minify-selectors "^5.1.0" - postcss-normalize-charset "^5.0.1" - postcss-normalize-display-values "^5.0.1" - postcss-normalize-positions "^5.0.1" - postcss-normalize-repeat-style "^5.0.1" - postcss-normalize-string "^5.0.1" - postcss-normalize-timing-functions "^5.0.1" - postcss-normalize-unicode "^5.0.1" - postcss-normalize-url "^5.0.3" - postcss-normalize-whitespace "^5.0.1" - postcss-ordered-values "^5.0.2" - postcss-reduce-initial "^5.0.2" - postcss-reduce-transforms "^5.0.1" - postcss-svgo "^5.0.3" - postcss-unique-selectors "^5.0.2" - -cssnano-utils@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/cssnano-utils/-/cssnano-utils-2.0.1.tgz#8660aa2b37ed869d2e2f22918196a9a8b6498ce2" - integrity sha512-i8vLRZTnEH9ubIyfdZCAdIdgnHAUeQeByEeQ2I7oTilvP9oHO6RScpeq3GsFUVqeB8uZgOQ9pw8utofNn32hhQ== - -cssnano@^5.0.4: - version "5.0.12" - resolved "https://registry.yarnpkg.com/cssnano/-/cssnano-5.0.12.tgz#2c083a1c786fc9dc2d5522bd3c0e331b7cd302ab" - integrity sha512-U38V4x2iJ3ijPdeWqUrEr4eKBB5PbEKsNP5T8xcik2Au3LeMtiMHX0i2Hu9k51FcKofNZumbrcdC6+a521IUHg== - dependencies: - cssnano-preset-default "^5.1.8" - is-resolvable "^1.1.0" - lilconfig "^2.0.3" - yaml "^1.10.2" - -cssnano@^5.0.6: - version "5.0.8" - resolved "https://registry.yarnpkg.com/cssnano/-/cssnano-5.0.8.tgz#39ad166256980fcc64faa08c9bb18bb5789ecfa9" - integrity sha512-Lda7geZU0Yu+RZi2SGpjYuQz4HI4/1Y+BhdD0jL7NXAQ5larCzVn+PUGuZbDMYz904AXXCOgO5L1teSvgu7aFg== - dependencies: - cssnano-preset-default "^5.1.4" - is-resolvable "^1.1.0" - lilconfig "^2.0.3" - yaml "^1.10.2" - -csso@^4.0.2, csso@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/csso/-/csso-4.2.0.tgz#ea3a561346e8dc9f546d6febedd50187cf389529" - integrity sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA== - dependencies: - css-tree "^1.1.2" - -debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.0: - version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" - integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== - dependencies: - ms "2.0.0" - -debug@^3.1.1, debug@^3.2.6: - version "3.2.7" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" - integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== - dependencies: - ms "^2.1.1" - -debug@^4.1.0, debug@^4.1.1: - version "4.3.1" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" - integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== - dependencies: - ms "2.1.2" - -decamelize@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" - integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= - -decode-uri-component@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" - integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU= - -decompress-response@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-3.3.0.tgz#80a4dd323748384bfa248083622aedec982adff3" - integrity sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M= - dependencies: - mimic-response "^1.0.0" - -deep-equal@^1.0.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.1.1.tgz#b5c98c942ceffaf7cb051e24e1434a25a2e6076a" - integrity sha512-yd9c5AdiqVcR+JjcwUQb9DkhJc8ngNr0MahEBGvDiJw8puWab2yZlh+nkasOnZP+EGTAP6rRp2JzJhJZzvNF8g== - dependencies: - is-arguments "^1.0.4" - is-date-object "^1.0.1" - is-regex "^1.0.4" - object-is "^1.0.1" - object-keys "^1.1.1" - regexp.prototype.flags "^1.2.0" - -deep-extend@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" - integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA== - -deepmerge@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955" - integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg== - -default-gateway@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/default-gateway/-/default-gateway-4.2.0.tgz#167104c7500c2115f6dd69b0a536bb8ed720552b" - integrity sha512-h6sMrVB1VMWVrW13mSc6ia/DwYYw5MN6+exNu1OaJeFac5aSAvwM7lZ0NVfTABuSkQelr4h5oebg3KB1XPdjgA== - dependencies: - execa "^1.0.0" - ip-regex "^2.1.0" - -defer-to-connect@^1.0.1: - version "1.1.3" - resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-1.1.3.tgz#331ae050c08dcf789f8c83a7b81f0ed94f4ac591" - integrity sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ== - -define-properties@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.3.tgz#cf88da6cbee26fe6db7094f61d870cbd84cee9f1" - integrity sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ== - dependencies: - object-keys "^1.0.12" - -define-property@^0.2.5: - version "0.2.5" - resolved "https://registry.yarnpkg.com/define-property/-/define-property-0.2.5.tgz#c35b1ef918ec3c990f9a5bc57be04aacec5c8116" - integrity sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY= - dependencies: - is-descriptor "^0.1.0" - -define-property@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/define-property/-/define-property-1.0.0.tgz#769ebaaf3f4a63aad3af9e8d304c9bbe79bfb0e6" - integrity sha1-dp66rz9KY6rTr56NMEybvnm/sOY= - dependencies: - is-descriptor "^1.0.0" - -define-property@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/define-property/-/define-property-2.0.2.tgz#d459689e8d654ba77e02a817f8710d702cb16e9d" - integrity sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ== - dependencies: - is-descriptor "^1.0.2" - isobject "^3.0.1" - -del@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/del/-/del-4.1.1.tgz#9e8f117222ea44a31ff3a156c049b99052a9f0b4" - integrity sha512-QwGuEUouP2kVwQenAsOof5Fv8K9t3D8Ca8NxcXKrIpEHjTXK5J2nXLdP+ALI1cgv8wj7KuwBhTwBkOZSJKM5XQ== - dependencies: - "@types/glob" "^7.1.1" - globby "^6.1.0" - is-path-cwd "^2.0.0" - is-path-in-cwd "^2.0.0" - p-map "^2.0.0" - pify "^4.0.1" - rimraf "^2.6.3" - -del@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/del/-/del-6.0.0.tgz#0b40d0332cea743f1614f818be4feb717714c952" - integrity sha512-1shh9DQ23L16oXSZKB2JxpL7iMy2E0S9d517ptA1P8iw0alkPtQcrKH7ru31rYtKwF499HkTu+DRzq3TCKDFRQ== - dependencies: - globby "^11.0.1" - graceful-fs "^4.2.4" - is-glob "^4.0.1" - is-path-cwd "^2.2.0" - is-path-inside "^3.0.2" - p-map "^4.0.0" - rimraf "^3.0.2" - slash "^3.0.0" - -depd@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" - integrity sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak= - -destroy@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" - integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= - -detab@2.0.4: - version "2.0.4" - resolved "https://registry.yarnpkg.com/detab/-/detab-2.0.4.tgz#b927892069aff405fbb9a186fe97a44a92a94b43" - integrity sha512-8zdsQA5bIkoRECvCrNKPla84lyoR7DSAyf7p0YgXzBO9PDJx8KntPUay7NS6yp+KdxdVtiE5SpHKtbp2ZQyA9g== - dependencies: - repeat-string "^1.5.4" - -detect-node@^2.0.4: - version "2.0.5" - resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.0.5.tgz#9d270aa7eaa5af0b72c4c9d9b814e7f4ce738b79" - integrity sha512-qi86tE6hRcFHy8jI1m2VG+LaPUR1LhqDa5G8tVjuUXmOrpuAgqsA1pN0+ldgr3aKUH+QLI9hCY/OcRYisERejw== - -detect-port-alt@1.1.6: - version "1.1.6" - resolved "https://registry.yarnpkg.com/detect-port-alt/-/detect-port-alt-1.1.6.tgz#24707deabe932d4a3cf621302027c2b266568275" - integrity sha512-5tQykt+LqfJFBEYaDITx7S7cR7mJ/zQmLXZ2qt5w04ainYZw6tBf9dBunMjVeVOdYVRUzUOE4HkY5J7+uttb5Q== - dependencies: - address "^1.0.1" - debug "^2.6.0" - -detect-port@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/detect-port/-/detect-port-1.3.0.tgz#d9c40e9accadd4df5cac6a782aefd014d573d1f1" - integrity sha512-E+B1gzkl2gqxt1IhUzwjrxBKRqx1UzC3WLONHinn8S3T6lwV/agVCyitiFOsGJ/eYuEUBvD71MZHy3Pv1G9doQ== - dependencies: - address "^1.0.1" - debug "^2.6.0" - -dir-glob@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" - integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA== - dependencies: - path-type "^4.0.0" - -dns-equal@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/dns-equal/-/dns-equal-1.0.0.tgz#b39e7f1da6eb0a75ba9c17324b34753c47e0654d" - integrity sha1-s55/HabrCnW6nBcySzR1PEfgZU0= - -dns-packet@^1.3.1: - version "1.3.4" - resolved "https://registry.yarnpkg.com/dns-packet/-/dns-packet-1.3.4.tgz#e3455065824a2507ba886c55a89963bb107dec6f" - integrity sha512-BQ6F4vycLXBvdrJZ6S3gZewt6rcrks9KBgM9vrhW+knGRqc8uEdT7fuCwloc7nny5xNoMJ17HGH0R/6fpo8ECA== - dependencies: - ip "^1.1.0" - safe-buffer "^5.0.1" - -dns-txt@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/dns-txt/-/dns-txt-2.0.2.tgz#b91d806f5d27188e4ab3e7d107d881a1cc4642b6" - integrity sha1-uR2Ab10nGI5Ks+fRB9iBocxGQrY= - dependencies: - buffer-indexof "^1.0.0" - -dom-converter@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/dom-converter/-/dom-converter-0.2.0.tgz#6721a9daee2e293682955b6afe416771627bb768" - integrity sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA== - dependencies: - utila "~0.4" - -dom-serializer@0: - version "0.2.2" - resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.2.2.tgz#1afb81f533717175d478655debc5e332d9f9bb51" - integrity sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g== - dependencies: - domelementtype "^2.0.1" - entities "^2.0.0" - -dom-serializer@^1.0.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-1.3.1.tgz#d845a1565d7c041a95e5dab62184ab41e3a519be" - integrity sha512-Pv2ZluG5ife96udGgEDovOOOA5UELkltfJpnIExPrAk1LTvecolUGn6lIaoLh86d83GiB86CjzciMd9BuRB71Q== - dependencies: - domelementtype "^2.0.1" - domhandler "^4.0.0" - entities "^2.0.0" - -dom-serializer@~0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0" - integrity sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA== - dependencies: - domelementtype "^1.3.0" - entities "^1.1.1" - -domelementtype@1, domelementtype@^1.3.0, domelementtype@^1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f" - integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w== - -domelementtype@^2.0.1, domelementtype@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.2.0.tgz#9a0b6c2782ed6a1c7323d42267183df9bd8b1d57" - integrity sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A== - -domhandler@^2.3.0: - version "2.4.2" - resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.4.2.tgz#8805097e933d65e85546f726d60f5eb88b44f803" - integrity sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA== - dependencies: - domelementtype "1" - -domhandler@^4.0.0, domhandler@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.2.0.tgz#f9768a5f034be60a89a27c2e4d0f74eba0d8b059" - integrity sha512-zk7sgt970kzPks2Bf+dwT/PLzghLnsivb9CcxkvR8Mzr66Olr0Ofd8neSbglHJHaHa2MadfoSdNlKYAaafmWfA== - dependencies: - domelementtype "^2.2.0" - -domutils@1.5.1: - version "1.5.1" - resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf" - integrity sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8= - dependencies: - dom-serializer "0" - domelementtype "1" - -domutils@^1.5.1, domutils@^1.7.0: - version "1.7.0" - resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a" - integrity sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg== - dependencies: - dom-serializer "0" - domelementtype "1" - -domutils@^2.4.3: - version "2.6.0" - resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.6.0.tgz#2e15c04185d43fb16ae7057cb76433c6edb938b7" - integrity sha512-y0BezHuy4MDYxh6OvolXYsH+1EMGmFbwv5FKW7ovwMG6zTPWqNPq3WF9ayZssFq+UlKdffGLbOEaghNdaOm1WA== - dependencies: - dom-serializer "^1.0.1" - domelementtype "^2.2.0" - domhandler "^4.2.0" - -domutils@^2.5.2, domutils@^2.6.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.7.0.tgz#8ebaf0c41ebafcf55b0b72ec31c56323712c5442" - integrity sha512-8eaHa17IwJUPAiB+SoTYBo5mCdeMgdcAoXJ59m6DT1vw+5iLS3gNoqYaRowaBKtGVrOF1Jz4yDTgYKLK2kvfJg== - dependencies: - dom-serializer "^1.0.1" - domelementtype "^2.2.0" - domhandler "^4.2.0" - -dot-case@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/dot-case/-/dot-case-3.0.4.tgz#9b2b670d00a431667a8a75ba29cd1b98809ce751" - integrity sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w== - dependencies: - no-case "^3.0.4" - tslib "^2.0.3" - -dot-prop@^5.2.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-5.3.0.tgz#90ccce708cd9cd82cc4dc8c3ddd9abdd55b20e88" - integrity sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q== - dependencies: - is-obj "^2.0.0" - -duplexer3@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" - integrity sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI= - -duplexer@^0.1.1, duplexer@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6" - integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg== - -ee-first@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" - integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= - -electron-to-chromium@^1.3.564: - version "1.4.15" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.15.tgz#4bd144d9d13f8b375c65e1a593e7f4a90bd01b90" - integrity sha512-WDw2IUL3k4QpbzInV3JZK+Zd1NjWJPDZ28oUSchWb/kf6AVj7/niaAlgcJlvojFa1d7pJSyQ/KSZsEtq5W7aGQ== - -electron-to-chromium@^1.3.719: - version "1.3.723" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.723.tgz#52769a75635342a4db29af5f1e40bd3dad02c877" - integrity sha512-L+WXyXI7c7+G1V8ANzRsPI5giiimLAUDC6Zs1ojHHPhYXb3k/iTABFmWjivEtsWrRQymjnO66/rO2ZTABGdmWg== - -electron-to-chromium@^1.3.723: - version "1.3.725" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.725.tgz#04fc83f9189169aff50f0a00c6b4090b910cba85" - integrity sha512-2BbeAESz7kc6KBzs7WVrMc1BY5waUphk4D4DX5dSQXJhsc3tP5ZFaiyuL0AB7vUKzDYpIeYwTYlEfxyjsGUrhw== - -electron-to-chromium@^1.3.896: - version "1.4.14" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.14.tgz#b0aa41fbfbf2eff8c2c6f7a871c03075250f8956" - integrity sha512-RsGkAN9JEAYMObS72kzUsPPcPGMqX1rBqGuXi9aa4TBKLzICoLf+DAAtd0fVFzrniJqYzpby47gthCUoObfs0Q== - -emoji-regex@^7.0.1: - version "7.0.3" - resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156" - integrity sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA== - -emoji-regex@^8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" - integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== - -emojis-list@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-3.0.0.tgz#5570662046ad29e2e916e71aae260abdff4f6a78" - integrity sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q== - -emoticon@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/emoticon/-/emoticon-3.2.0.tgz#c008ca7d7620fac742fe1bf4af8ff8fed154ae7f" - integrity sha512-SNujglcLTTg+lDAcApPNgEdudaqQFiAbJCqzjNxJkvN9vAwCGi0uu8IUVvx+f16h+V44KCY6Y2yboroc9pilHg== - -encodeurl@~1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" - integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= - -end-of-stream@^1.1.0: - version "1.4.4" - resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" - integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== - dependencies: - once "^1.4.0" - -enhanced-resolve@^5.8.3: - version "5.8.3" - resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.8.3.tgz#6d552d465cce0423f5b3d718511ea53826a7b2f0" - integrity sha512-EGAbGvH7j7Xt2nc0E7D99La1OiEs8LnyimkRgwExpUMScN6O+3x9tIWs7PLQZVNx4YD+00skHXPXi1yQHpAmZA== - dependencies: - graceful-fs "^4.2.4" - tapable "^2.2.0" - -entities@^1.1.1, entities@~1.1.1: - version "1.1.2" - resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56" - integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w== - -entities@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" - integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== - -errno@^0.1.3: - version "0.1.8" - resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.8.tgz#8bb3e9c7d463be4976ff888f76b4809ebc2e811f" - integrity sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A== - dependencies: - prr "~1.0.1" - -error-ex@^1.3.1: - version "1.3.2" - resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" - integrity sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g== - dependencies: - is-arrayish "^0.2.1" - -es-abstract@^1.17.2, es-abstract@^1.18.0-next.2: - version "1.18.0" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.0.tgz#ab80b359eecb7ede4c298000390bc5ac3ec7b5a4" - integrity sha512-LJzK7MrQa8TS0ja2w3YNLzUgJCGPdPOV1yVvezjNnS89D+VR08+Szt2mz3YB2Dck/+w5tfIq/RoUAFqJJGM2yw== - dependencies: - call-bind "^1.0.2" - es-to-primitive "^1.2.1" - function-bind "^1.1.1" - get-intrinsic "^1.1.1" - has "^1.0.3" - has-symbols "^1.0.2" - is-callable "^1.2.3" - is-negative-zero "^2.0.1" - is-regex "^1.1.2" - is-string "^1.0.5" - object-inspect "^1.9.0" - object-keys "^1.1.1" - object.assign "^4.1.2" - string.prototype.trimend "^1.0.4" - string.prototype.trimstart "^1.0.4" - unbox-primitive "^1.0.0" - -es-module-lexer@^0.9.0: - version "0.9.3" - resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-0.9.3.tgz#6f13db00cc38417137daf74366f535c8eb438f19" - integrity sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ== - -es-to-primitive@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a" - integrity sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA== - dependencies: - is-callable "^1.1.4" - is-date-object "^1.0.1" - is-symbol "^1.0.2" - -escalade@^3.0.2, escalade@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" - integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== - -escape-goat@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675" - integrity sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q== - -escape-html@^1.0.3, escape-html@~1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" - integrity sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg= - -escape-string-regexp@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz#a30304e99daa32e23b2fd20f51babd07cffca344" - integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w== - -escape-string-regexp@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" - integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= - -escape-string-regexp@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" - integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== - -eslint-scope@5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.1.1.tgz#e786e59a66cb92b3f6c1fb0d508aab174848f48c" - integrity sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw== - dependencies: - esrecurse "^4.3.0" - estraverse "^4.1.1" - -esprima@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" - integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== - -esrecurse@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921" - integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag== - dependencies: - estraverse "^5.2.0" - -estraverse@^4.1.1: - version "4.3.0" - resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.3.0.tgz#398ad3f3c5a24948be7725e83d11a7de28cdbd1d" - integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw== - -estraverse@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.2.0.tgz#307df42547e6cc7324d3cf03c155d5cdb8c53880" - integrity sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ== - -esutils@^2.0.2: - version "2.0.3" - resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" - integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== - -eta@^1.12.1: - version "1.12.3" - resolved "https://registry.yarnpkg.com/eta/-/eta-1.12.3.tgz#2982d08adfbef39f9fa50e2fbd42d7337e7338b1" - integrity sha512-qHixwbDLtekO/d51Yr4glcaUJCIjGVJyTzuqV4GPlgZo1YpgOKG+avQynErZIYrfM6JIJdtiG2Kox8tbb+DoGg== - -etag@~1.8.1: - version "1.8.1" - resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" - integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= - -eval@^0.1.4: - version "0.1.6" - resolved "https://registry.yarnpkg.com/eval/-/eval-0.1.6.tgz#9620d7d8c85515e97e6b47c5814f46ae381cb3cc" - integrity sha512-o0XUw+5OGkXw4pJZzQoXUk+H87DHuC+7ZE//oSrRGtatTmr12oTnLfg6QOq9DyTt0c/p4TwzgmkKrBzWTSizyQ== - dependencies: - require-like ">= 0.1.1" - -eventemitter3@^4.0.0: - version "4.0.7" - resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" - integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== - -events@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" - integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= - -events@^3.2.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" - integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== - -eventsource@^1.0.7: - version "1.1.0" - resolved "https://registry.yarnpkg.com/eventsource/-/eventsource-1.1.0.tgz#00e8ca7c92109e94b0ddf32dac677d841028cfaf" - integrity sha512-VSJjT5oCNrFvCS6igjzPAt5hBzQ2qPBFIbJ03zLI9SE0mxwZpMw6BfJrbFHm1a141AavMEB8JHmBhWAd66PfCg== - dependencies: - original "^1.0.0" - -execa@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8" - integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA== - dependencies: - cross-spawn "^6.0.0" - get-stream "^4.0.0" - is-stream "^1.1.0" - npm-run-path "^2.0.0" - p-finally "^1.0.0" - signal-exit "^3.0.0" - strip-eof "^1.0.0" - -execa@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-5.0.0.tgz#4029b0007998a841fbd1032e5f4de86a3c1e3376" - integrity sha512-ov6w/2LCiuyO4RLYGdpFGjkcs0wMTgGE8PrkTHikeUy5iJekXyPIKUjifk5CsE0pt7sMCrMZ3YNqoCj6idQOnQ== - dependencies: - cross-spawn "^7.0.3" - get-stream "^6.0.0" - human-signals "^2.1.0" - is-stream "^2.0.0" - merge-stream "^2.0.0" - npm-run-path "^4.0.1" - onetime "^5.1.2" - signal-exit "^3.0.3" - strip-final-newline "^2.0.0" - -expand-brackets@^2.1.4: - version "2.1.4" - resolved "https://registry.yarnpkg.com/expand-brackets/-/expand-brackets-2.1.4.tgz#b77735e315ce30f6b6eff0f83b04151a22449622" - integrity sha1-t3c14xXOMPa27/D4OwQVGiJEliI= - dependencies: - debug "^2.3.3" - define-property "^0.2.5" - extend-shallow "^2.0.1" - posix-character-classes "^0.1.0" - regex-not "^1.0.0" - snapdragon "^0.8.1" - to-regex "^3.0.1" - -express@^4.17.1: - version "4.17.1" - resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134" - integrity sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g== - dependencies: - accepts "~1.3.7" - array-flatten "1.1.1" - body-parser "1.19.0" - content-disposition "0.5.3" - content-type "~1.0.4" - cookie "0.4.0" - cookie-signature "1.0.6" - debug "2.6.9" - depd "~1.1.2" - encodeurl "~1.0.2" - escape-html "~1.0.3" - etag "~1.8.1" - finalhandler "~1.1.2" - fresh "0.5.2" - merge-descriptors "1.0.1" - methods "~1.1.2" - on-finished "~2.3.0" - parseurl "~1.3.3" - path-to-regexp "0.1.7" - proxy-addr "~2.0.5" - qs "6.7.0" - range-parser "~1.2.1" - safe-buffer "5.1.2" - send "0.17.1" - serve-static "1.14.1" - setprototypeof "1.1.1" - statuses "~1.5.0" - type-is "~1.6.18" - utils-merge "1.0.1" - vary "~1.1.2" - -extend-shallow@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/extend-shallow/-/extend-shallow-2.0.1.tgz#51af7d614ad9a9f610ea1bafbb989d6b1c56890f" - integrity sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8= - dependencies: - is-extendable "^0.1.0" - -extend-shallow@^3.0.0, extend-shallow@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/extend-shallow/-/extend-shallow-3.0.2.tgz#26a71aaf073b39fb2127172746131c2704028db8" - integrity sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg= - dependencies: - assign-symbols "^1.0.0" - is-extendable "^1.0.1" - -extend@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" - integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== - -extglob@^2.0.4: - version "2.0.4" - resolved "https://registry.yarnpkg.com/extglob/-/extglob-2.0.4.tgz#ad00fe4dc612a9232e8718711dc5cb5ab0285543" - integrity sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw== - dependencies: - array-unique "^0.3.2" - define-property "^1.0.0" - expand-brackets "^2.1.4" - extend-shallow "^2.0.1" - fragment-cache "^0.2.1" - regex-not "^1.0.0" - snapdragon "^0.8.1" - to-regex "^3.0.1" - -fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: - version "3.1.3" - resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" - integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== - -fast-glob@^3.1.1: - version "3.2.5" - resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.5.tgz#7939af2a656de79a4f1901903ee8adcaa7cb9661" - integrity sha512-2DtFcgT68wiTTiwZ2hNdJfcHNke9XOfnwmBRWXhmeKM8rF0TGwmC/Qto3S7RoZKp5cilZbxzO5iTNTQsJ+EeDg== - dependencies: - "@nodelib/fs.stat" "^2.0.2" - "@nodelib/fs.walk" "^1.2.3" - glob-parent "^5.1.0" - merge2 "^1.3.0" - micromatch "^4.0.2" - picomatch "^2.2.1" - -fast-glob@^3.2.7: - version "3.2.7" - resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.7.tgz#fd6cb7a2d7e9aa7a7846111e85a196d6b2f766a1" - integrity sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q== - dependencies: - "@nodelib/fs.stat" "^2.0.2" - "@nodelib/fs.walk" "^1.2.3" - glob-parent "^5.1.2" - merge2 "^1.3.0" - micromatch "^4.0.4" - -fast-json-stable-stringify@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" - integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== - -fast-url-parser@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/fast-url-parser/-/fast-url-parser-1.1.3.tgz#f4af3ea9f34d8a271cf58ad2b3759f431f0b318d" - integrity sha1-9K8+qfNNiicc9YrSs3WfQx8LMY0= - dependencies: - punycode "^1.3.2" - -fastq@^1.6.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.11.0.tgz#bb9fb955a07130a918eb63c1f5161cc32a5d0858" - integrity sha512-7Eczs8gIPDrVzT+EksYBcupqMyxSHXXrHOLRRxU2/DicV8789MRBRR8+Hc2uWzUupOs4YS4JzBmBxjjCVBxD/g== - dependencies: - reusify "^1.0.4" - -faye-websocket@^0.11.3: - version "0.11.3" - resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.11.3.tgz#5c0e9a8968e8912c286639fde977a8b209f2508e" - integrity sha512-D2y4bovYpzziGgbHYtGCMjlJM36vAl/y+xUyn1C+FVx8szd1E+86KwVw6XvYSzOP8iMpm1X0I4xJD+QtUb36OA== - dependencies: - websocket-driver ">=0.5.1" - -fbemitter@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/fbemitter/-/fbemitter-3.0.0.tgz#00b2a1af5411254aab416cd75f9e6289bee4bff3" - integrity sha512-KWKaceCwKQU0+HPoop6gn4eOHk50bBv/VxjJtGMfwmJt3D29JpN4H4eisCtIPA+a8GVBam+ldMMpMjJUvpDyHw== - dependencies: - fbjs "^3.0.0" - -fbjs-css-vars@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/fbjs-css-vars/-/fbjs-css-vars-1.0.2.tgz#216551136ae02fe255932c3ec8775f18e2c078b8" - integrity sha512-b2XGFAFdWZWg0phtAWLHCk836A1Xann+I+Dgd3Gk64MHKZO44FfoD1KxyvbSh0qZsIoXQGGlVztIY+oitJPpRQ== - -fbjs@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/fbjs/-/fbjs-3.0.0.tgz#0907067fb3f57a78f45d95f1eacffcacd623c165" - integrity sha512-dJd4PiDOFuhe7vk4F80Mba83Vr2QuK86FoxtgPmzBqEJahncp+13YCmfoa53KHCo6OnlXLG7eeMWPfB5CrpVKg== - dependencies: - cross-fetch "^3.0.4" - fbjs-css-vars "^1.0.0" - loose-envify "^1.0.0" - object-assign "^4.1.0" - promise "^7.1.1" - setimmediate "^1.0.5" - ua-parser-js "^0.7.18" - -feed@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/feed/-/feed-4.2.2.tgz#865783ef6ed12579e2c44bbef3c9113bc4956a7e" - integrity sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ== - dependencies: - xml-js "^1.6.11" - -figures@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" - integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== - dependencies: - escape-string-regexp "^1.0.5" - -file-loader@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/file-loader/-/file-loader-6.2.0.tgz#baef7cf8e1840df325e4390b4484879480eebe4d" - integrity sha512-qo3glqyTa61Ytg4u73GultjHGjdRyig3tG6lPtyX/jOEJvHif9uB0/OCI2Kif6ctF3caQTW2G5gym21oAsI4pw== - dependencies: - loader-utils "^2.0.0" - schema-utils "^3.0.0" - -file-uri-to-path@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" - integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== - -filesize@6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/filesize/-/filesize-6.1.0.tgz#e81bdaa780e2451d714d71c0d7a4f3238d37ad00" - integrity sha512-LpCHtPQ3sFx67z+uh2HnSyWSLLu5Jxo21795uRDuar/EOuYWXib5EmPaGIBuSnRqH2IODiKA2k5re/K9OnN/Yg== - -fill-range@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7" - integrity sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc= - dependencies: - extend-shallow "^2.0.1" - is-number "^3.0.0" - repeat-string "^1.6.1" - to-regex-range "^2.1.0" - -fill-range@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" - integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== - dependencies: - to-regex-range "^5.0.1" - -finalhandler@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d" - integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== - dependencies: - debug "2.6.9" - encodeurl "~1.0.2" - escape-html "~1.0.3" - on-finished "~2.3.0" - parseurl "~1.3.3" - statuses "~1.5.0" - unpipe "~1.0.0" - -find-cache-dir@^3.3.1: - version "3.3.1" - resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-3.3.1.tgz#89b33fad4a4670daa94f855f7fbe31d6d84fe880" - integrity sha512-t2GDMt3oGC/v+BMwzmllWDuJF/xcDtE5j/fCGbqDD7OLuJkj0cfh1YSA5VKPvwMeLFLNDBkwOKZ2X85jGLVftQ== - dependencies: - commondir "^1.0.1" - make-dir "^3.0.2" - pkg-dir "^4.1.0" - -find-up@4.1.0, find-up@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" - integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== - dependencies: - locate-path "^5.0.0" - path-exists "^4.0.0" - -find-up@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73" - integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg== - dependencies: - locate-path "^3.0.0" - -find-up@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" - integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== - dependencies: - locate-path "^6.0.0" - path-exists "^4.0.0" - -flux@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/flux/-/flux-4.0.1.tgz#7843502b02841d4aaa534af0b373034a1f75ee5c" - integrity sha512-emk4RCvJ8RzNP2lNpphKnG7r18q8elDYNAPx7xn+bDeOIo9FFfxEfIQ2y6YbQNmnsGD3nH1noxtLE64Puz1bRQ== - dependencies: - fbemitter "^3.0.0" - fbjs "^3.0.0" - -follow-redirects@^1.0.0, follow-redirects@^1.14.0: - version "1.14.7" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.7.tgz#2004c02eb9436eee9a21446a6477debf17e81685" - integrity sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ== - -for-in@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" - integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= - -fork-ts-checker-webpack-plugin@4.1.6: - version "4.1.6" - resolved "https://registry.yarnpkg.com/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-4.1.6.tgz#5055c703febcf37fa06405d400c122b905167fc5" - integrity sha512-DUxuQaKoqfNne8iikd14SAkh5uw4+8vNifp6gmA73yYNS6ywLIWSLD/n/mBzHQRpW3J7rbATEakmiA8JvkTyZw== - dependencies: - "@babel/code-frame" "^7.5.5" - chalk "^2.4.1" - micromatch "^3.1.10" - minimatch "^3.0.4" - semver "^5.6.0" - tapable "^1.0.0" - worker-rpc "^0.1.0" - -forwarded@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" - integrity sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ= - -fraction.js@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/fraction.js/-/fraction.js-4.1.1.tgz#ac4e520473dae67012d618aab91eda09bcb400ff" - integrity sha512-MHOhvvxHTfRFpF1geTK9czMIZ6xclsEor2wkIGYYq+PxcQqT7vStJqjhe6S1TenZrMZzo+wlqOufBDVepUEgPg== - -fragment-cache@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19" - integrity sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk= - dependencies: - map-cache "^0.2.2" - -fresh@0.5.2: - version "0.5.2" - resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" - integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac= - -fs-extra@^10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.0.0.tgz#9ff61b655dde53fb34a82df84bb214ce802e17c1" - integrity sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ== - dependencies: - graceful-fs "^4.2.0" - jsonfile "^6.0.1" - universalify "^2.0.0" - -fs.realpath@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" - integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= - -fsevents@^1.2.7: - version "1.2.13" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-1.2.13.tgz#f325cb0455592428bcf11b383370ef70e3bfcc38" - integrity sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw== - dependencies: - bindings "^1.5.0" - nan "^2.12.1" - -fsevents@~2.3.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" - integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== - -function-bind@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" - integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== - -gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2: - version "1.0.0-beta.2" - resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" - integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== - -get-caller-file@^2.0.1: - version "2.0.5" - resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" - integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== - -get-intrinsic@^1.0.2, get-intrinsic@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz#15f59f376f855c446963948f0d24cd3637b4abc6" - integrity sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q== - dependencies: - function-bind "^1.1.1" - has "^1.0.3" - has-symbols "^1.0.1" - -get-own-enumerable-property-symbols@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz#b5fde77f22cbe35f390b4e089922c50bce6ef664" - integrity sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g== - -get-stream@^4.0.0, get-stream@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5" - integrity sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w== - dependencies: - pump "^3.0.0" - -get-stream@^5.1.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" - integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== - dependencies: - pump "^3.0.0" - -get-stream@^6.0.0: - version "6.0.1" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" - integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== - -get-value@^2.0.3, get-value@^2.0.6: - version "2.0.6" - resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" - integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg= - -github-slugger@^1.3.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/github-slugger/-/github-slugger-1.4.0.tgz#206eb96cdb22ee56fdc53a28d5a302338463444e" - integrity sha512-w0dzqw/nt51xMVmlaV1+JRzN+oCa1KfcgGEWhxUG16wbdA+Xnt/yoFO8Z8x/V82ZcZ0wy6ln9QDup5avbhiDhQ== - -glob-parent@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-3.1.0.tgz#9e6af6299d8d3bd2bd40430832bd113df906c5ae" - integrity sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4= - dependencies: - is-glob "^3.1.0" - path-dirname "^1.0.0" - -glob-parent@^5.1.0, glob-parent@^5.1.2, glob-parent@~5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" - integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== - dependencies: - is-glob "^4.0.1" - -glob-parent@^6.0.1: - version "6.0.2" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-6.0.2.tgz#6d237d99083950c79290f24c7642a3de9a28f9e3" - integrity sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A== - dependencies: - is-glob "^4.0.3" - -glob-to-regexp@^0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" - integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== - -glob@^7.0.0, glob@^7.0.3, glob@^7.1.3: - version "7.2.0" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" - integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - -global-dirs@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-3.0.0.tgz#70a76fe84ea315ab37b1f5576cbde7d48ef72686" - integrity sha512-v8ho2DS5RiCjftj1nD9NmnfaOzTdud7RRnVd9kFNOjqZbISlx5DQ+OrTkywgd0dIt7oFCvKetZSHoHcP3sDdiA== - dependencies: - ini "2.0.0" - -global-modules@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" - integrity sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A== - dependencies: - global-prefix "^3.0.0" - -global-prefix@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/global-prefix/-/global-prefix-3.0.0.tgz#fc85f73064df69f50421f47f883fe5b913ba9b97" - integrity sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg== - dependencies: - ini "^1.3.5" - kind-of "^6.0.2" - which "^1.3.1" - -globals@^11.1.0: - version "11.12.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" - integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== - -globby@11.0.1: - version "11.0.1" - resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.1.tgz#9a2bf107a068f3ffeabc49ad702c79ede8cfd357" - integrity sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ== - dependencies: - array-union "^2.1.0" - dir-glob "^3.0.1" - fast-glob "^3.1.1" - ignore "^5.1.4" - merge2 "^1.3.0" - slash "^3.0.0" - -globby@^11.0.1, globby@^11.0.2, globby@^11.0.3: - version "11.0.4" - resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.4.tgz#2cbaff77c2f2a62e71e9b2813a67b97a3a3001a5" - integrity sha512-9O4MVG9ioZJ08ffbcyVYyLOJLk5JQ688pJ4eMGLpdWLHq/Wr1D9BlriLQyL0E+jbkuePVZXYFj47QM/v093wHg== - dependencies: - array-union "^2.1.0" - dir-glob "^3.0.1" - fast-glob "^3.1.1" - ignore "^5.1.4" - merge2 "^1.3.0" - slash "^3.0.0" - -globby@^6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/globby/-/globby-6.1.0.tgz#f5a6d70e8395e21c858fb0489d64df02424d506c" - integrity sha1-9abXDoOV4hyFj7BInWTfAkJNUGw= - dependencies: - array-union "^1.0.1" - glob "^7.0.3" - object-assign "^4.0.1" - pify "^2.0.0" - pinkie-promise "^2.0.0" - -got@^9.6.0: - version "9.6.0" - resolved "https://registry.yarnpkg.com/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85" - integrity sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q== - dependencies: - "@sindresorhus/is" "^0.14.0" - "@szmarczak/http-timer" "^1.1.2" - cacheable-request "^6.0.0" - decompress-response "^3.3.0" - duplexer3 "^0.1.4" - get-stream "^4.1.0" - lowercase-keys "^1.0.1" - mimic-response "^1.0.1" - p-cancelable "^1.0.0" - to-readable-stream "^1.0.0" - url-parse-lax "^3.0.0" - -graceful-fs@^4.1.11: - version "4.2.8" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.8.tgz#e412b8d33f5e006593cbd3cee6df9f2cebbe802a" - integrity sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg== - -graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4: - version "4.2.6" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee" - integrity sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ== - -gray-matter@^4.0.3: - version "4.0.3" - resolved "https://registry.yarnpkg.com/gray-matter/-/gray-matter-4.0.3.tgz#e893c064825de73ea1f5f7d88c7a9f7274288798" - integrity sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q== - dependencies: - js-yaml "^3.13.1" - kind-of "^6.0.2" - section-matter "^1.0.0" - strip-bom-string "^1.0.0" - -gzip-size@5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/gzip-size/-/gzip-size-5.1.1.tgz#cb9bee692f87c0612b232840a873904e4c135274" - integrity sha512-FNHi6mmoHvs1mxZAds4PpdCS6QG8B4C1krxJsMutgxl5t3+GlRTzzI3NEkifXx2pVsOvJdOGSmIgDhQ55FwdPA== - dependencies: - duplexer "^0.1.1" - pify "^4.0.1" - -gzip-size@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/gzip-size/-/gzip-size-6.0.0.tgz#065367fd50c239c0671cbcbad5be3e2eeb10e462" - integrity sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q== - dependencies: - duplexer "^0.1.2" - -handle-thing@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/handle-thing/-/handle-thing-2.0.1.tgz#857f79ce359580c340d43081cc648970d0bb234e" - integrity sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg== - -has-bigints@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.1.tgz#64fe6acb020673e3b78db035a5af69aa9d07b113" - integrity sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA== - -has-flag@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" - integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= - -has-flag@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" - integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== - -has-symbols@^1.0.1, has-symbols@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.2.tgz#165d3070c00309752a1236a479331e3ac56f1423" - integrity sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw== - -has-value@^0.3.1: - version "0.3.1" - resolved "https://registry.yarnpkg.com/has-value/-/has-value-0.3.1.tgz#7b1f58bada62ca827ec0a2078025654845995e1f" - integrity sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8= - dependencies: - get-value "^2.0.3" - has-values "^0.1.4" - isobject "^2.0.0" - -has-value@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/has-value/-/has-value-1.0.0.tgz#18b281da585b1c5c51def24c930ed29a0be6b177" - integrity sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc= - dependencies: - get-value "^2.0.6" - has-values "^1.0.0" - isobject "^3.0.0" - -has-values@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/has-values/-/has-values-0.1.4.tgz#6d61de95d91dfca9b9a02089ad384bff8f62b771" - integrity sha1-bWHeldkd/Km5oCCJrThL/49it3E= - -has-values@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/has-values/-/has-values-1.0.0.tgz#95b0b63fec2146619a6fe57fe75628d5a39efe4f" - integrity sha1-lbC2P+whRmGab+V/51Yo1aOe/k8= - dependencies: - is-number "^3.0.0" - kind-of "^4.0.0" - -has-yarn@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/has-yarn/-/has-yarn-2.1.0.tgz#137e11354a7b5bf11aa5cb649cf0c6f3ff2b2e77" - integrity sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw== - -has@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" - integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== - dependencies: - function-bind "^1.1.1" - -hast-to-hyperscript@^9.0.0: - version "9.0.1" - resolved "https://registry.yarnpkg.com/hast-to-hyperscript/-/hast-to-hyperscript-9.0.1.tgz#9b67fd188e4c81e8ad66f803855334173920218d" - integrity sha512-zQgLKqF+O2F72S1aa4y2ivxzSlko3MAvxkwG8ehGmNiqd98BIN3JM1rAJPmplEyLmGLO2QZYJtIneOSZ2YbJuA== - dependencies: - "@types/unist" "^2.0.3" - comma-separated-tokens "^1.0.0" - property-information "^5.3.0" - space-separated-tokens "^1.0.0" - style-to-object "^0.3.0" - unist-util-is "^4.0.0" - web-namespaces "^1.0.0" - -hast-util-from-parse5@^5.0.0: - version "5.0.3" - resolved "https://registry.yarnpkg.com/hast-util-from-parse5/-/hast-util-from-parse5-5.0.3.tgz#3089dc0ee2ccf6ec8bc416919b51a54a589e097c" - integrity sha512-gOc8UB99F6eWVWFtM9jUikjN7QkWxB3nY0df5Z0Zq1/Nkwl5V4hAAsl0tmwlgWl/1shlTF8DnNYLO8X6wRV9pA== - dependencies: - ccount "^1.0.3" - hastscript "^5.0.0" - property-information "^5.0.0" - web-namespaces "^1.1.2" - xtend "^4.0.1" - -hast-util-from-parse5@^6.0.0: - version "6.0.1" - resolved "https://registry.yarnpkg.com/hast-util-from-parse5/-/hast-util-from-parse5-6.0.1.tgz#554e34abdeea25ac76f5bd950a1f0180e0b3bc2a" - integrity sha512-jeJUWiN5pSxW12Rh01smtVkZgZr33wBokLzKLwinYOUfSzm1Nl/c3GUGebDyOKjdsRgMvoVbV0VpAcpjF4NrJA== - dependencies: - "@types/parse5" "^5.0.0" - hastscript "^6.0.0" - property-information "^5.0.0" - vfile "^4.0.0" - vfile-location "^3.2.0" - web-namespaces "^1.0.0" - -hast-util-parse-selector@^2.0.0: - version "2.2.5" - resolved "https://registry.yarnpkg.com/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz#d57c23f4da16ae3c63b3b6ca4616683313499c3a" - integrity sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ== - -hast-util-raw@6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/hast-util-raw/-/hast-util-raw-6.0.1.tgz#973b15930b7529a7b66984c98148b46526885977" - integrity sha512-ZMuiYA+UF7BXBtsTBNcLBF5HzXzkyE6MLzJnL605LKE8GJylNjGc4jjxazAHUtcwT5/CEt6afRKViYB4X66dig== - dependencies: - "@types/hast" "^2.0.0" - hast-util-from-parse5 "^6.0.0" - hast-util-to-parse5 "^6.0.0" - html-void-elements "^1.0.0" - parse5 "^6.0.0" - unist-util-position "^3.0.0" - vfile "^4.0.0" - web-namespaces "^1.0.0" - xtend "^4.0.0" - zwitch "^1.0.0" - -hast-util-to-parse5@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/hast-util-to-parse5/-/hast-util-to-parse5-6.0.0.tgz#1ec44650b631d72952066cea9b1445df699f8479" - integrity sha512-Lu5m6Lgm/fWuz8eWnrKezHtVY83JeRGaNQ2kn9aJgqaxvVkFCZQBEhgodZUDUvoodgyROHDb3r5IxAEdl6suJQ== - dependencies: - hast-to-hyperscript "^9.0.0" - property-information "^5.0.0" - web-namespaces "^1.0.0" - xtend "^4.0.0" - zwitch "^1.0.0" - -hastscript@^5.0.0: - version "5.1.2" - resolved "https://registry.yarnpkg.com/hastscript/-/hastscript-5.1.2.tgz#bde2c2e56d04c62dd24e8c5df288d050a355fb8a" - integrity sha512-WlztFuK+Lrvi3EggsqOkQ52rKbxkXL3RwB6t5lwoa8QLMemoWfBuL43eDrwOamJyR7uKQKdmKYaBH1NZBiIRrQ== - dependencies: - comma-separated-tokens "^1.0.0" - hast-util-parse-selector "^2.0.0" - property-information "^5.0.0" - space-separated-tokens "^1.0.0" - -hastscript@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/hastscript/-/hastscript-6.0.0.tgz#e8768d7eac56c3fdeac8a92830d58e811e5bf640" - integrity sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w== - dependencies: - "@types/hast" "^2.0.0" - comma-separated-tokens "^1.0.0" - hast-util-parse-selector "^2.0.0" - property-information "^5.0.0" - space-separated-tokens "^1.0.0" - -he@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" - integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== - -history@^4.9.0: - version "4.10.1" - resolved "https://registry.yarnpkg.com/history/-/history-4.10.1.tgz#33371a65e3a83b267434e2b3f3b1b4c58aad4cf3" - integrity sha512-36nwAD620w12kuzPAsyINPWJqlNbij+hpK1k9XRloDtym8mxzGYl2c17LnV6IAGB2Dmg4tEa7G7DlawS0+qjew== - dependencies: - "@babel/runtime" "^7.1.2" - loose-envify "^1.2.0" - resolve-pathname "^3.0.0" - tiny-invariant "^1.0.2" - tiny-warning "^1.0.0" - value-equal "^1.0.1" - -hoist-non-react-statics@^3.1.0: - version "3.3.2" - resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45" - integrity sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw== - dependencies: - react-is "^16.7.0" - -hpack.js@^2.1.6: - version "2.1.6" - resolved "https://registry.yarnpkg.com/hpack.js/-/hpack.js-2.1.6.tgz#87774c0949e513f42e84575b3c45681fade2a0b2" - integrity sha1-h3dMCUnlE/QuhFdbPEVoH63ioLI= - dependencies: - inherits "^2.0.1" - obuf "^1.0.0" - readable-stream "^2.0.1" - wbuf "^1.1.0" - -html-entities@^1.3.1: - version "1.4.0" - resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-1.4.0.tgz#cfbd1b01d2afaf9adca1b10ae7dffab98c71d2dc" - integrity sha512-8nxjcBcd8wovbeKx7h3wTji4e6+rhaVuPNpMqwWgnHh+N9ToqsCs6XztWRBPQ+UtzsoMAdKZtUENoVzU/EMtZA== - -html-minifier-terser@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/html-minifier-terser/-/html-minifier-terser-5.1.1.tgz#922e96f1f3bb60832c2634b79884096389b1f054" - integrity sha512-ZPr5MNObqnV/T9akshPKbVgyOqLmy+Bxo7juKCfTfnjNniTAMdy4hz21YQqoofMBJD2kdREaqPPdThoR78Tgxg== - dependencies: - camel-case "^4.1.1" - clean-css "^4.2.3" - commander "^4.1.1" - he "^1.2.0" - param-case "^3.0.3" - relateurl "^0.2.7" - terser "^4.6.3" - -html-minifier-terser@^6.0.2: - version "6.0.2" - resolved "https://registry.yarnpkg.com/html-minifier-terser/-/html-minifier-terser-6.0.2.tgz#14059ad64b69bf9f8b8a33f25b53411d8321e75d" - integrity sha512-AgYO3UGhMYQx2S/FBJT3EM0ZYcKmH6m9XL9c1v77BeK/tYJxGPxT1/AtsdUi4FcP8kZGmqqnItCcjFPcX9hk6A== - dependencies: - camel-case "^4.1.2" - clean-css "^5.1.5" - commander "^8.1.0" - he "^1.2.0" - param-case "^3.0.4" - relateurl "^0.2.7" - terser "^5.7.2" - -html-tags@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/html-tags/-/html-tags-3.1.0.tgz#7b5e6f7e665e9fb41f30007ed9e0d41e97fb2140" - integrity sha512-1qYz89hW3lFDEazhjW0yVAV87lw8lVkrJocr72XmBkMKsoSVJCQx3W8BXsC7hO2qAt8BoVjYjtAcZ9perqGnNg== - -html-void-elements@^1.0.0: - version "1.0.5" - resolved "https://registry.yarnpkg.com/html-void-elements/-/html-void-elements-1.0.5.tgz#ce9159494e86d95e45795b166c2021c2cfca4483" - integrity sha512-uE/TxKuyNIcx44cIWnjr/rfIATDH7ZaOMmstu0CwhFG1Dunhlp4OC6/NMbhiwoq5BpW0ubi303qnEk/PZj614w== - -html-webpack-plugin@^5.3.2: - version "5.5.0" - resolved "https://registry.yarnpkg.com/html-webpack-plugin/-/html-webpack-plugin-5.5.0.tgz#c3911936f57681c1f9f4d8b68c158cd9dfe52f50" - integrity sha512-sy88PC2cRTVxvETRgUHFrL4No3UxvcH8G1NepGhqaTT+GXN2kTamqasot0inS5hXeg1cMbFDt27zzo9p35lZVw== - dependencies: - "@types/html-minifier-terser" "^6.0.0" - html-minifier-terser "^6.0.2" - lodash "^4.17.21" - pretty-error "^4.0.0" - tapable "^2.0.0" - -htmlparser2@^3.9.1: - version "3.10.1" - resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f" - integrity sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ== - dependencies: - domelementtype "^1.3.1" - domhandler "^2.3.0" - domutils "^1.5.1" - entities "^1.1.1" - inherits "^2.0.1" - readable-stream "^3.1.1" - -htmlparser2@^6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7" - integrity sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A== - dependencies: - domelementtype "^2.0.1" - domhandler "^4.0.0" - domutils "^2.5.2" - entities "^2.0.0" - -http-cache-semantics@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390" - integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ== - -http-deceiver@^1.2.7: - version "1.2.7" - resolved "https://registry.yarnpkg.com/http-deceiver/-/http-deceiver-1.2.7.tgz#fa7168944ab9a519d337cb0bec7284dc3e723d87" - integrity sha1-+nFolEq5pRnTN8sL7HKE3D5yPYc= - -http-errors@1.7.2: - version "1.7.2" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.2.tgz#4f5029cf13239f31036e5b2e55292bcfbcc85c8f" - integrity sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg== - dependencies: - depd "~1.1.2" - inherits "2.0.3" - setprototypeof "1.1.1" - statuses ">= 1.5.0 < 2" - toidentifier "1.0.0" - -http-errors@~1.6.2: - version "1.6.3" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.3.tgz#8b55680bb4be283a0b5bf4ea2e38580be1d9320d" - integrity sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0= - dependencies: - depd "~1.1.2" - inherits "2.0.3" - setprototypeof "1.1.0" - statuses ">= 1.4.0 < 2" - -http-errors@~1.7.2: - version "1.7.3" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" - integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw== - dependencies: - depd "~1.1.2" - inherits "2.0.4" - setprototypeof "1.1.1" - statuses ">= 1.5.0 < 2" - toidentifier "1.0.0" - -http-parser-js@>=0.5.1: - version "0.5.3" - resolved "https://registry.yarnpkg.com/http-parser-js/-/http-parser-js-0.5.3.tgz#01d2709c79d41698bb01d4decc5e9da4e4a033d9" - integrity sha512-t7hjvef/5HEK7RWTdUzVUhl8zkEu+LlaE0IYzdMuvbSDipxBRpOn4Uhw8ZyECEa808iVT8XCjzo6xmYt4CiLZg== - -http-proxy-middleware@0.19.1: - version "0.19.1" - resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-0.19.1.tgz#183c7dc4aa1479150306498c210cdaf96080a43a" - integrity sha512-yHYTgWMQO8VvwNS22eLLloAkvungsKdKTLO8AJlftYIKNfJr3GK3zK0ZCfzDDGUBttdGc8xFy1mCitvNKQtC3Q== - dependencies: - http-proxy "^1.17.0" - is-glob "^4.0.0" - lodash "^4.17.11" - micromatch "^3.1.10" - -http-proxy@^1.17.0: - version "1.18.1" - resolved "https://registry.yarnpkg.com/http-proxy/-/http-proxy-1.18.1.tgz#401541f0534884bbf95260334e72f88ee3976549" - integrity sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ== - dependencies: - eventemitter3 "^4.0.0" - follow-redirects "^1.0.0" - requires-port "^1.0.0" - -human-signals@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" - integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== - -iconv-lite@0.4.24: - version "0.4.24" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" - integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== - dependencies: - safer-buffer ">= 2.1.2 < 3" - -icss-utils@^5.0.0, icss-utils@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae" - integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA== - -ignore@^5.1.4: - version "5.1.8" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" - integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== - -immer@8.0.1: - version "8.0.1" - resolved "https://registry.yarnpkg.com/immer/-/immer-8.0.1.tgz#9c73db683e2b3975c424fb0572af5889877ae656" - integrity sha512-aqXhGP7//Gui2+UrEtvxZxSquQVXTpZ7KDxfCcKAF3Vysvw0CViVaW9RZ1j1xlIYqaaaipBoqdqeibkc18PNvA== - -import-fresh@^3.2.1, import-fresh@^3.2.2, import-fresh@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" - integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== - dependencies: - parent-module "^1.0.0" - resolve-from "^4.0.0" - -import-lazy@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/import-lazy/-/import-lazy-2.1.0.tgz#05698e3d45c88e8d7e9d92cb0584e77f096f3e43" - integrity sha1-BWmOPUXIjo1+nZLLBYTnfwlvPkM= - -import-local@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/import-local/-/import-local-2.0.0.tgz#55070be38a5993cf18ef6db7e961f5bee5c5a09d" - integrity sha512-b6s04m3O+s3CGSbqDIyP4R6aAwAeYlVq9+WUWep6iHa8ETRf9yei1U48C5MmfJmV9AiLYYBKPMq/W+/WRpQmCQ== - dependencies: - pkg-dir "^3.0.0" - resolve-cwd "^2.0.0" - -imurmurhash@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" - integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= - -indent-string@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" - integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== - -infima@0.2.0-alpha.26: - version "0.2.0-alpha.26" - resolved "https://registry.yarnpkg.com/infima/-/infima-0.2.0-alpha.26.tgz#8582d40ef01a09dbbde8f0e574f8c61d6bc0992b" - integrity sha512-0/Dt+89mf8xW+9/hKGmynK+WOAsiy0QydVJL0qie6WK57yGIQv+SjJrhMybKndnmkZBQ+Vlt0tWPnTakx8X2Qw== - -inflight@^1.0.4: - version "1.0.6" - resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" - integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= - dependencies: - once "^1.3.0" - wrappy "1" - -inherits@2, inherits@2.0.4, inherits@^2.0.0, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: - version "2.0.4" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" - integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== - -inherits@2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" - integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= - -ini@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ini/-/ini-2.0.0.tgz#e5fd556ecdd5726be978fa1001862eacb0a94bc5" - integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== - -ini@^1.3.5, ini@~1.3.0: - version "1.3.8" - resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" - integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== - -inline-style-parser@0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/inline-style-parser/-/inline-style-parser-0.1.1.tgz#ec8a3b429274e9c0a1f1c4ffa9453a7fef72cea1" - integrity sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q== - -internal-ip@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/internal-ip/-/internal-ip-4.3.0.tgz#845452baad9d2ca3b69c635a137acb9a0dad0907" - integrity sha512-S1zBo1D6zcsyuC6PMmY5+55YMILQ9av8lotMx447Bq6SAgo/sDK6y6uUKmuYhW7eacnIhFfsPmCNYdDzsnnDCg== - dependencies: - default-gateway "^4.2.0" - ipaddr.js "^1.9.0" - -interpret@^1.0.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e" - integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA== - -ip-regex@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" - integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk= - -ip@^1.1.0, ip@^1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a" - integrity sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo= - -ipaddr.js@1.9.1, ipaddr.js@^1.9.0: - version "1.9.1" - resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" - integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== - -is-absolute-url@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/is-absolute-url/-/is-absolute-url-3.0.3.tgz#96c6a22b6a23929b11ea0afb1836c36ad4a5d698" - integrity sha512-opmNIX7uFnS96NtPmhWQgQx6/NYFgsUXYMllcfzwWKUMwfo8kku1TvE6hkNcH+Q1ts5cMVrsY7j0bxXQDciu9Q== - -is-accessor-descriptor@^0.1.6: - version "0.1.6" - resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6" - integrity sha1-qeEss66Nh2cn7u84Q/igiXtcmNY= - dependencies: - kind-of "^3.0.2" - -is-accessor-descriptor@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz#169c2f6d3df1f992618072365c9b0ea1f6878656" - integrity sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ== - dependencies: - kind-of "^6.0.0" - -is-alphabetical@1.0.4, is-alphabetical@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-1.0.4.tgz#9e7d6b94916be22153745d184c298cbf986a686d" - integrity sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg== - -is-alphanumerical@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz#7eb9a2431f855f6b1ef1a78e326df515696c4dbf" - integrity sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A== - dependencies: - is-alphabetical "^1.0.0" - is-decimal "^1.0.0" - -is-arguments@^1.0.4: - version "1.1.0" - resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.1.0.tgz#62353031dfbee07ceb34656a6bde59efecae8dd9" - integrity sha512-1Ij4lOMPl/xB5kBDn7I+b2ttPMKa8szhEIrXDuXQD/oe3HJLTLhqhgGspwgyGd6MOywBUqVvYicF72lkgDnIHg== - dependencies: - call-bind "^1.0.0" - -is-arrayish@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" - integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0= - -is-bigint@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.1.tgz#6923051dfcbc764278540b9ce0e6b3213aa5ebc2" - integrity sha512-J0ELF4yHFxHy0cmSxZuheDOz2luOdVvqjwmEcj8H/L1JHeuEDSDbeRP+Dk9kFVk5RTFzbucJ2Kb9F7ixY2QaCg== - -is-binary-path@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-1.0.1.tgz#75f16642b480f187a711c814161fd3a4a7655898" - integrity sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg= - dependencies: - binary-extensions "^1.0.0" - -is-binary-path@~2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09" - integrity sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw== - dependencies: - binary-extensions "^2.0.0" - -is-boolean-object@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.1.0.tgz#e2aaad3a3a8fca34c28f6eee135b156ed2587ff0" - integrity sha512-a7Uprx8UtD+HWdyYwnD1+ExtTgqQtD2k/1yJgtXP6wnMm8byhkoTZRl+95LLThpzNZJ5aEvi46cdH+ayMFRwmA== - dependencies: - call-bind "^1.0.0" - -is-buffer@^1.1.5: - version "1.1.6" - resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" - integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== - -is-buffer@^2.0.0: - version "2.0.5" - resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.5.tgz#ebc252e400d22ff8d77fa09888821a24a658c191" - integrity sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ== - -is-callable@^1.1.4, is-callable@^1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.3.tgz#8b1e0500b73a1d76c70487636f368e519de8db8e" - integrity sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ== - -is-ci@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-2.0.0.tgz#6bc6334181810e04b5c22b3d589fdca55026404c" - integrity sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w== - dependencies: - ci-info "^2.0.0" - -is-core-module@^2.2.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.3.0.tgz#d341652e3408bca69c4671b79a0954a3d349f887" - integrity sha512-xSphU2KG9867tsYdLD4RWQ1VqdFl4HTO9Thf3I/3dLEfr0dbPTWKsuCKrgqMljg4nPE+Gq0VCnzT3gr0CyBmsw== - dependencies: - has "^1.0.3" - -is-core-module@^2.8.0: - version "2.8.1" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.8.1.tgz#f59fdfca701d5879d0a6b100a40aa1560ce27211" - integrity sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA== - dependencies: - has "^1.0.3" - -is-data-descriptor@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56" - integrity sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y= - dependencies: - kind-of "^3.0.2" - -is-data-descriptor@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz#d84876321d0e7add03990406abbbbd36ba9268c7" - integrity sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ== - dependencies: - kind-of "^6.0.0" - -is-date-object@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.2.tgz#bda736f2cd8fd06d32844e7743bfa7494c3bfd7e" - integrity sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g== - -is-decimal@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-1.0.4.tgz#65a3a5958a1c5b63a706e1b333d7cd9f630d3fa5" - integrity sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw== - -is-descriptor@^0.1.0: - version "0.1.6" - resolved "https://registry.yarnpkg.com/is-descriptor/-/is-descriptor-0.1.6.tgz#366d8240dde487ca51823b1ab9f07a10a78251ca" - integrity sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg== - dependencies: - is-accessor-descriptor "^0.1.6" - is-data-descriptor "^0.1.4" - kind-of "^5.0.0" - -is-descriptor@^1.0.0, is-descriptor@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-descriptor/-/is-descriptor-1.0.2.tgz#3b159746a66604b04f8c81524ba365c5f14d86ec" - integrity sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg== - dependencies: - is-accessor-descriptor "^1.0.0" - is-data-descriptor "^1.0.0" - kind-of "^6.0.2" - -is-docker@^2.0.0: - version "2.2.1" - resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.2.1.tgz#33eeabe23cfe86f14bde4408a02c0cfb853acdaa" - integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ== - -is-extendable@^0.1.0, is-extendable@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89" - integrity sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik= - -is-extendable@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-1.0.1.tgz#a7470f9e426733d81bd81e1155264e3a3507cab4" - integrity sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA== - dependencies: - is-plain-object "^2.0.4" - -is-extglob@^2.1.0, is-extglob@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" - integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI= - -is-fullwidth-code-point@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" - integrity sha1-o7MKXE8ZkYMWeqq5O+764937ZU8= - -is-fullwidth-code-point@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" - integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== - -is-glob@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-3.1.0.tgz#7ba5ae24217804ac70707b96922567486cc3e84a" - integrity sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo= - dependencies: - is-extglob "^2.1.0" - -is-glob@^4.0.0, is-glob@^4.0.3: - version "4.0.3" - resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" - integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== - dependencies: - is-extglob "^2.1.1" - -is-glob@^4.0.1, is-glob@~4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc" - integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg== - dependencies: - is-extglob "^2.1.1" - -is-hexadecimal@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz#cc35c97588da4bd49a8eedd6bc4082d44dcb23a7" - integrity sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw== - -is-installed-globally@^0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/is-installed-globally/-/is-installed-globally-0.4.0.tgz#9a0fd407949c30f86eb6959ef1b7994ed0b7b520" - integrity sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ== - dependencies: - global-dirs "^3.0.0" - is-path-inside "^3.0.2" - -is-negative-zero@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.1.tgz#3de746c18dda2319241a53675908d8f766f11c24" - integrity sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w== - -is-npm@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-5.0.0.tgz#43e8d65cc56e1b67f8d47262cf667099193f45a8" - integrity sha512-WW/rQLOazUq+ST/bCAVBp/2oMERWLsR7OrKyt052dNDk4DHcDE0/7QSXITlmi+VBcV13DfIbysG3tZJm5RfdBA== - -is-number-object@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.4.tgz#36ac95e741cf18b283fc1ddf5e83da798e3ec197" - integrity sha512-zohwelOAur+5uXtk8O3GPQ1eAcu4ZX3UwxQhUlfFFMNpUd83gXgjbhJh6HmB6LUNV/ieOLQuDwJO3dWJosUeMw== - -is-number@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195" - integrity sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU= - dependencies: - kind-of "^3.0.2" - -is-number@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" - integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== - -is-obj@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-1.0.1.tgz#3e4729ac1f5fde025cd7d83a896dab9f4f67db0f" - integrity sha1-PkcprB9f3gJc19g6iW2rn09n2w8= - -is-obj@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-2.0.0.tgz#473fb05d973705e3fd9620545018ca8e22ef4982" - integrity sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w== - -is-path-cwd@^2.0.0, is-path-cwd@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/is-path-cwd/-/is-path-cwd-2.2.0.tgz#67d43b82664a7b5191fd9119127eb300048a9fdb" - integrity sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ== - -is-path-in-cwd@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-path-in-cwd/-/is-path-in-cwd-2.1.0.tgz#bfe2dca26c69f397265a4009963602935a053acb" - integrity sha512-rNocXHgipO+rvnP6dk3zI20RpOtrAM/kzbB258Uw5BWr3TpXi861yzjo16Dn4hUox07iw5AyeMLHWsujkjzvRQ== - dependencies: - is-path-inside "^2.1.0" - -is-path-inside@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-2.1.0.tgz#7c9810587d659a40d27bcdb4d5616eab059494b2" - integrity sha512-wiyhTzfDWsvwAW53OBWF5zuvaOGlZ6PwYxAbPVDhpm+gM09xKQGjBq/8uYN12aDvMxnAnq3dxTyoSoRNmg5YFg== - dependencies: - path-is-inside "^1.0.2" - -is-path-inside@^3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" - integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== - -is-plain-obj@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz#45e42e37fccf1f40da8e5f76ee21515840c09287" - integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA== - -is-plain-object@^2.0.3, is-plain-object@^2.0.4: - version "2.0.4" - resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677" - integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og== - dependencies: - isobject "^3.0.1" - -is-regex@^1.0.4, is-regex@^1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.2.tgz#81c8ebde4db142f2cf1c53fc86d6a45788266251" - integrity sha512-axvdhb5pdhEVThqJzYXwMlVuZwC+FF2DpcOhTS+y/8jVq4trxyPgfcwIxIKiyeuLlSQYKkmUaPQJ8ZE4yNKXDg== - dependencies: - call-bind "^1.0.2" - has-symbols "^1.0.1" - -is-regexp@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-regexp/-/is-regexp-1.0.0.tgz#fd2d883545c46bac5a633e7b9a09e87fa2cb5069" - integrity sha1-/S2INUXEa6xaYz57mgnof6LLUGk= - -is-resolvable@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/is-resolvable/-/is-resolvable-1.1.0.tgz#fb18f87ce1feb925169c9a407c19318a3206ed88" - integrity sha512-qgDYXFSR5WvEfuS5dMj6oTMEbrrSaM0CrFk2Yiq/gXnBvD9pMa2jGXxyhGLfvhZpuMZe18CJpFxAt3CRs42NMg== - -is-root@2.1.0, is-root@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-root/-/is-root-2.1.0.tgz#809e18129cf1129644302a4f8544035d51984a9c" - integrity sha512-AGOriNp96vNBd3HtU+RzFEc75FfR5ymiYv8E553I71SCeXBiMsVDUtdio1OEFvrPyLIQ9tVR5RxXIFe5PUFjMg== - -is-stream@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" - integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ= - -is-stream@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3" - integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw== - -is-string@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.5.tgz#40493ed198ef3ff477b8c7f92f644ec82a5cd3a6" - integrity sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ== - -is-symbol@^1.0.2, is-symbol@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.3.tgz#38e1014b9e6329be0de9d24a414fd7441ec61937" - integrity sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ== - dependencies: - has-symbols "^1.0.1" - -is-typedarray@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" - integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= - -is-whitespace-character@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-whitespace-character/-/is-whitespace-character-1.0.4.tgz#0858edd94a95594c7c9dd0b5c174ec6e45ee4aa7" - integrity sha512-SDweEzfIZM0SJV0EUga669UTKlmL0Pq8Lno0QDQsPnvECB3IM2aP0gdx5TrU0A01MAPfViaZiI2V1QMZLaKK5w== - -is-windows@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" - integrity sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA== - -is-word-character@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-word-character/-/is-word-character-1.0.4.tgz#ce0e73216f98599060592f62ff31354ddbeb0230" - integrity sha512-5SMO8RVennx3nZrqtKwCGyyetPE9VDba5ugvKLaD4KopPG5kR4mQ7tNt/r7feL5yt5h3lpuBbIUmCOG2eSzXHA== - -is-wsl@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-1.1.0.tgz#1f16e4aa22b04d1336b66188a66af3c600c3a66d" - integrity sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0= - -is-wsl@^2.1.1: - version "2.2.0" - resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-2.2.0.tgz#74a4c76e77ca9fd3f932f290c17ea326cd157271" - integrity sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww== - dependencies: - is-docker "^2.0.0" - -is-yarn-global@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/is-yarn-global/-/is-yarn-global-0.3.0.tgz#d502d3382590ea3004893746754c89139973e232" - integrity sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw== - -isarray@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" - integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8= - -isarray@1.0.0, isarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= - -isexe@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" - integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= - -isobject@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89" - integrity sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk= - dependencies: - isarray "1.0.0" - -isobject@^3.0.0, isobject@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" - integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8= - -jest-worker@^27.0.2: - version "27.0.2" - resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-27.0.2.tgz#4ebeb56cef48b3e7514552f80d0d80c0129f0b05" - integrity sha512-EoBdilOTTyOgmHXtw/cPc+ZrCA0KJMrkXzkrPGNwLmnvvlN1nj7MPrxpT7m+otSv2e1TLaVffzDnE/LB14zJMg== - dependencies: - "@types/node" "*" - merge-stream "^2.0.0" - supports-color "^8.0.0" - -jest-worker@^27.0.6: - version "27.2.5" - resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-27.2.5.tgz#ed42865661959488aa020e8a325df010597c36d4" - integrity sha512-HTjEPZtcNKZ4LnhSp02NEH4vE+5OpJ0EsOWYvGQpHgUMLngydESAAMH5Wd/asPf29+XUDQZszxpLg1BkIIA2aw== - dependencies: - "@types/node" "*" - merge-stream "^2.0.0" - supports-color "^8.0.0" - -joi@^17.3.0: - version "17.5.0" - resolved "https://registry.yarnpkg.com/joi/-/joi-17.5.0.tgz#7e66d0004b5045d971cf416a55fb61d33ac6e011" - integrity sha512-R7hR50COp7StzLnDi4ywOXHrBrgNXuUUfJWIR5lPY5Bm/pOD3jZaTwpluUXVLRWcoWZxkrHBBJ5hLxgnlehbdw== - dependencies: - "@hapi/hoek" "^9.0.0" - "@hapi/topo" "^5.0.0" - "@sideway/address" "^4.1.3" - "@sideway/formula" "^3.0.0" - "@sideway/pinpoint" "^2.0.0" - -joi@^17.4.0: - version "17.4.0" - resolved "https://registry.yarnpkg.com/joi/-/joi-17.4.0.tgz#b5c2277c8519e016316e49ababd41a1908d9ef20" - integrity sha512-F4WiW2xaV6wc1jxete70Rw4V/VuMd6IN+a5ilZsxG4uYtUXWu2kq9W5P2dz30e7Gmw8RCbY/u/uk+dMPma9tAg== - dependencies: - "@hapi/hoek" "^9.0.0" - "@hapi/topo" "^5.0.0" - "@sideway/address" "^4.1.0" - "@sideway/formula" "^3.0.0" - "@sideway/pinpoint" "^2.0.0" - -"js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" - integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== - -js-yaml@^3.13.1: - version "3.14.1" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537" - integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== - dependencies: - argparse "^1.0.7" - esprima "^4.0.0" - -js-yaml@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" - integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== - dependencies: - argparse "^2.0.1" - -jsesc@^2.5.1: - version "2.5.2" - resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" - integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== - -jsesc@~0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" - integrity sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0= - -json-buffer@3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898" - integrity sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg= - -json-parse-better-errors@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" - integrity sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw== - -json-parse-even-better-errors@^2.3.0: - version "2.3.1" - resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" - integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== - -json-schema-traverse@^0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" - integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== - -json-schema-traverse@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2" - integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== - -json3@^3.3.3: - version "3.3.3" - resolved "https://registry.yarnpkg.com/json3/-/json3-3.3.3.tgz#7fc10e375fc5ae42c4705a5cc0aa6f62be305b81" - integrity sha512-c7/8mbUsKigAbLkD5B010BK4D9LZm7A1pNItkEwiUZRpIN66exu/e7YQWysGun+TRKaJp8MhemM+VkfWv42aCA== - -json5@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe" - integrity sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow== - dependencies: - minimist "^1.2.0" - -json5@^2.1.2: - version "2.2.0" - resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3" - integrity sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA== - dependencies: - minimist "^1.2.5" - -jsonfile@^6.0.1: - version "6.1.0" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" - integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== - dependencies: - universalify "^2.0.0" - optionalDependencies: - graceful-fs "^4.1.6" - -keyv@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/keyv/-/keyv-3.1.0.tgz#ecc228486f69991e49e9476485a5be1e8fc5c4d9" - integrity sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA== - dependencies: - json-buffer "3.0.0" - -killable@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/killable/-/killable-1.0.1.tgz#4c8ce441187a061c7474fb87ca08e2a638194892" - integrity sha512-LzqtLKlUwirEUyl/nicirVmNiPvYs7l5n8wOPP7fyJVpUPkvCnW/vuiXGpylGUlnPDnB7311rARzAt3Mhswpjg== - -kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: - version "3.2.2" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" - integrity sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ= - dependencies: - is-buffer "^1.1.5" - -kind-of@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-4.0.0.tgz#20813df3d712928b207378691a45066fae72dd57" - integrity sha1-IIE989cSkosgc3hpGkUGb65y3Vc= - dependencies: - is-buffer "^1.1.5" - -kind-of@^5.0.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-5.1.0.tgz#729c91e2d857b7a419a1f9aa65685c4c33f5845d" - integrity sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw== - -kind-of@^6.0.0, kind-of@^6.0.2: - version "6.0.3" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" - integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== - -kleur@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" - integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== - -klona@^2.0.4: - version "2.0.4" - resolved "https://registry.yarnpkg.com/klona/-/klona-2.0.4.tgz#7bb1e3affb0cb8624547ef7e8f6708ea2e39dfc0" - integrity sha512-ZRbnvdg/NxqzC7L9Uyqzf4psi1OM4Cuc+sJAkQPjO6XkQIJTNbfK2Rsmbw8fx1p2mkZdp2FZYo2+LwXYY/uwIA== - -latest-version@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-5.1.0.tgz#119dfe908fe38d15dfa43ecd13fa12ec8832face" - integrity sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA== - dependencies: - package-json "^6.3.0" - -leven@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2" - integrity sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A== - -lilconfig@^2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-2.0.3.tgz#68f3005e921dafbd2a2afb48379986aa6d2579fd" - integrity sha512-EHKqr/+ZvdKCifpNrJCKxBTgk5XupZA3y/aCPY9mxfgBzmgh93Mt/WqjjQ38oMxXuvDokaKiM3lAgvSH2sjtHg== - -lines-and-columns@^1.1.6: - version "1.1.6" - resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" - integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA= - -loader-runner@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.2.0.tgz#d7022380d66d14c5fb1d496b89864ebcfd478384" - integrity sha512-92+huvxMvYlMzMt0iIOukcwYBFpkYJdpl2xsZ7LrlayO7E8SOv+JJUEK17B/dJIHAOLMfh2dZZ/Y18WgmGtYNw== - -loader-utils@2.0.0, loader-utils@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.0.tgz#e4cace5b816d425a166b5f097e10cd12b36064b0" - integrity sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ== - dependencies: - big.js "^5.2.2" - emojis-list "^3.0.0" - json5 "^2.1.2" - -loader-utils@^1.2.3, loader-utils@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.0.tgz#c579b5e34cb34b1a74edc6c1fb36bfa371d5a613" - integrity sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA== - dependencies: - big.js "^5.2.2" - emojis-list "^3.0.0" - json5 "^1.0.1" - -locate-path@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-3.0.0.tgz#dbec3b3ab759758071b58fe59fc41871af21400e" - integrity sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A== - dependencies: - p-locate "^3.0.0" - path-exists "^3.0.0" - -locate-path@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0" - integrity sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g== - dependencies: - p-locate "^4.1.0" - -locate-path@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" - integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== - dependencies: - p-locate "^5.0.0" - -lodash.assignin@^4.0.9: - version "4.2.0" - resolved "https://registry.yarnpkg.com/lodash.assignin/-/lodash.assignin-4.2.0.tgz#ba8df5fb841eb0a3e8044232b0e263a8dc6a28a2" - integrity sha1-uo31+4QesKPoBEIysOJjqNxqKKI= - -lodash.bind@^4.1.4: - version "4.2.1" - resolved "https://registry.yarnpkg.com/lodash.bind/-/lodash.bind-4.2.1.tgz#7ae3017e939622ac31b7d7d7dcb1b34db1690d35" - integrity sha1-euMBfpOWIqwxt9fX3LGzTbFpDTU= - -lodash.curry@^4.0.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/lodash.curry/-/lodash.curry-4.1.1.tgz#248e36072ede906501d75966200a86dab8b23170" - integrity sha1-JI42By7ekGUB11lmIAqG2riyMXA= - -lodash.debounce@^4.0.8: - version "4.0.8" - resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" - integrity sha1-gteb/zCmfEAF/9XiUVMArZyk168= - -lodash.defaults@^4.0.1: - version "4.2.0" - resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" - integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw= - -lodash.filter@^4.4.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/lodash.filter/-/lodash.filter-4.6.0.tgz#668b1d4981603ae1cc5a6fa760143e480b4c4ace" - integrity sha1-ZosdSYFgOuHMWm+nYBQ+SAtMSs4= - -lodash.flatten@^4.2.0: - version "4.4.0" - resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f" - integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8= - -lodash.flow@^3.3.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/lodash.flow/-/lodash.flow-3.5.0.tgz#87bf40292b8cf83e4e8ce1a3ae4209e20071675a" - integrity sha1-h79AKSuM+D5OjOGjrkIJ4gBxZ1o= - -lodash.foreach@^4.3.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.foreach/-/lodash.foreach-4.5.0.tgz#1a6a35eace401280c7f06dddec35165ab27e3e53" - integrity sha1-Gmo16s5AEoDH8G3d7DUWWrJ+PlM= - -lodash.map@^4.4.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/lodash.map/-/lodash.map-4.6.0.tgz#771ec7839e3473d9c4cde28b19394c3562f4f6d3" - integrity sha1-dx7Hg540c9nEzeKLGTlMNWL09tM= - -lodash.memoize@^4.1.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe" - integrity sha1-vMbEmkKihA7Zl/Mj6tpezRguC/4= - -lodash.merge@^4.4.0: - version "4.6.2" - resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" - integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== - -lodash.pick@^4.2.1: - version "4.4.0" - resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3" - integrity sha1-UvBWEP/53tQiYRRB7R/BI6AwAbM= - -lodash.reduce@^4.4.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/lodash.reduce/-/lodash.reduce-4.6.0.tgz#f1ab6b839299ad48f784abbf476596f03b914d3b" - integrity sha1-8atrg5KZrUj3hKu/R2WW8DuRTTs= - -lodash.reject@^4.4.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/lodash.reject/-/lodash.reject-4.6.0.tgz#80d6492dc1470864bbf583533b651f42a9f52415" - integrity sha1-gNZJLcFHCGS79YNTO2UfQqn1JBU= - -lodash.some@^4.4.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/lodash.some/-/lodash.some-4.6.0.tgz#1bb9f314ef6b8baded13b549169b2a945eb68e4d" - integrity sha1-G7nzFO9ri63tE7VJFpsqlF62jk0= - -lodash.toarray@^4.4.0: - version "4.4.0" - resolved "https://registry.yarnpkg.com/lodash.toarray/-/lodash.toarray-4.4.0.tgz#24c4bfcd6b2fba38bfd0594db1179d8e9b656561" - integrity sha1-JMS/zWsvuji/0FlNsRedjptlZWE= - -lodash.uniq@4.5.0, lodash.uniq@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" - integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M= - -lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21: - version "4.17.21" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" - integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== - -loglevel@^1.6.8: - version "1.8.0" - resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.8.0.tgz#e7ec73a57e1e7b419cb6c6ac06bf050b67356114" - integrity sha512-G6A/nJLRgWOuuwdNuA6koovfEV1YpqqAG4pRUlFaz3jj2QNZ8M4vBqnVA+HBTmU/AMNUtlOsMmSpF6NyOjztbA== - -loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.2.0, loose-envify@^1.3.1, loose-envify@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" - integrity sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q== - dependencies: - js-tokens "^3.0.0 || ^4.0.0" - -lower-case@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-2.0.2.tgz#6fa237c63dbdc4a82ca0fd882e4722dc5e634e28" - integrity sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg== - dependencies: - tslib "^2.0.3" - -lowercase-keys@^1.0.0, lowercase-keys@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f" - integrity sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA== - -lowercase-keys@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479" - integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA== - -lru-cache@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" - integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== - dependencies: - yallist "^4.0.0" - -make-dir@^3.0.0, make-dir@^3.0.2, make-dir@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" - integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw== - dependencies: - semver "^6.0.0" - -map-cache@^0.2.2: - version "0.2.2" - resolved "https://registry.yarnpkg.com/map-cache/-/map-cache-0.2.2.tgz#c32abd0bd6525d9b051645bb4f26ac5dc98a0dbf" - integrity sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8= - -map-visit@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/map-visit/-/map-visit-1.0.0.tgz#ecdca8f13144e660f1b5bd41f12f3479d98dfb8f" - integrity sha1-7Nyo8TFE5mDxtb1B8S80edmN+48= - dependencies: - object-visit "^1.0.0" - -markdown-escapes@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/markdown-escapes/-/markdown-escapes-1.0.4.tgz#c95415ef451499d7602b91095f3c8e8975f78535" - integrity sha512-8z4efJYk43E0upd0NbVXwgSTQs6cT3T06etieCMEg7dRbzCbxUCK/GHlX8mhHRDcp+OLlHkPKsvqQTCvsRl2cg== - -mdast-squeeze-paragraphs@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/mdast-squeeze-paragraphs/-/mdast-squeeze-paragraphs-4.0.0.tgz#7c4c114679c3bee27ef10b58e2e015be79f1ef97" - integrity sha512-zxdPn69hkQ1rm4J+2Cs2j6wDEv7O17TfXTJ33tl/+JPIoEmtV9t2ZzBM5LPHE8QlHsmVD8t3vPKCyY3oH+H8MQ== - dependencies: - unist-util-remove "^2.0.0" - -mdast-util-definitions@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/mdast-util-definitions/-/mdast-util-definitions-4.0.0.tgz#c5c1a84db799173b4dcf7643cda999e440c24db2" - integrity sha512-k8AJ6aNnUkB7IE+5azR9h81O5EQ/cTDXtWdMq9Kk5KcEW/8ritU5CeLg/9HhOC++nALHBlaogJ5jz0Ybk3kPMQ== - dependencies: - unist-util-visit "^2.0.0" - -mdast-util-to-hast@10.0.1: - version "10.0.1" - resolved "https://registry.yarnpkg.com/mdast-util-to-hast/-/mdast-util-to-hast-10.0.1.tgz#0cfc82089494c52d46eb0e3edb7a4eb2aea021eb" - integrity sha512-BW3LM9SEMnjf4HXXVApZMt8gLQWVNXc3jryK0nJu/rOXPOnlkUjmdkDlmxMirpbU9ILncGFIwLH/ubnWBbcdgA== - dependencies: - "@types/mdast" "^3.0.0" - "@types/unist" "^2.0.0" - mdast-util-definitions "^4.0.0" - mdurl "^1.0.0" - unist-builder "^2.0.0" - unist-util-generated "^1.0.0" - unist-util-position "^3.0.0" - unist-util-visit "^2.0.0" - -mdast-util-to-string@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-2.0.0.tgz#b8cfe6a713e1091cb5b728fc48885a4767f8b97b" - integrity sha512-AW4DRS3QbBayY/jJmD8437V1Gombjf8RSOUCMFBuo5iHi58AGEgVCKQ+ezHkZZDpAQS75hcBMpLqjpJTjtUL7w== - -mdn-data@2.0.14: - version "2.0.14" - resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50" - integrity sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow== - -mdn-data@2.0.4: - version "2.0.4" - resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.4.tgz#699b3c38ac6f1d728091a64650b65d388502fd5b" - integrity sha512-iV3XNKw06j5Q7mi6h+9vbx23Tv7JkjEVgKHW4pimwyDGWm0OIQntJJ+u1C6mg6mK1EaTv42XQ7w76yuzH7M2cA== - -mdurl@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e" - integrity sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4= - -media-typer@0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" - integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g= - -memory-fs@^0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552" - integrity sha1-OpoguEYlI+RHz7x+i7gO1me/xVI= - dependencies: - errno "^0.1.3" - readable-stream "^2.0.1" - -merge-descriptors@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" - integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= - -merge-stream@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" - integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== - -merge2@^1.3.0: - version "1.4.1" - resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" - integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== - -methods@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" - integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= - -microevent.ts@~0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/microevent.ts/-/microevent.ts-0.1.1.tgz#70b09b83f43df5172d0205a63025bce0f7357fa0" - integrity sha512-jo1OfR4TaEwd5HOrt5+tAZ9mqT4jmpNAusXtyfNzqVm9uiSYFZlKM1wYL4oU7azZW/PxQW53wM0S6OR1JHNa2g== - -micromatch@^3.1.10, micromatch@^3.1.4: - version "3.1.10" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-3.1.10.tgz#70859bc95c9840952f359a068a3fc49f9ecfac23" - integrity sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg== - dependencies: - arr-diff "^4.0.0" - array-unique "^0.3.2" - braces "^2.3.1" - define-property "^2.0.2" - extend-shallow "^3.0.2" - extglob "^2.0.4" - fragment-cache "^0.2.1" - kind-of "^6.0.2" - nanomatch "^1.2.9" - object.pick "^1.3.0" - regex-not "^1.0.0" - snapdragon "^0.8.1" - to-regex "^3.0.2" - -micromatch@^4.0.2, micromatch@^4.0.4: - version "4.0.4" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.4.tgz#896d519dfe9db25fce94ceb7a500919bf881ebf9" - integrity sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg== - dependencies: - braces "^3.0.1" - picomatch "^2.2.3" - -mime-db@1.47.0, "mime-db@>= 1.43.0 < 2": - version "1.47.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.47.0.tgz#8cb313e59965d3c05cfbf898915a267af46a335c" - integrity sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw== - -mime-db@~1.33.0: - version "1.33.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.33.0.tgz#a3492050a5cb9b63450541e39d9788d2272783db" - integrity sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ== - -mime-types@2.1.18: - version "2.1.18" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.18.tgz#6f323f60a83d11146f831ff11fd66e2fe5503bb8" - integrity sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ== - dependencies: - mime-db "~1.33.0" - -mime-types@^2.1.27, mime-types@~2.1.17, mime-types@~2.1.24: - version "2.1.30" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.30.tgz#6e7be8b4c479825f85ed6326695db73f9305d62d" - integrity sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg== - dependencies: - mime-db "1.47.0" - -mime@1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" - integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== - -mime@^2.3.1: - version "2.5.2" - resolved "https://registry.yarnpkg.com/mime/-/mime-2.5.2.tgz#6e3dc6cc2b9510643830e5f19d5cb753da5eeabe" - integrity sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg== - -mime@^2.4.4: - version "2.6.0" - resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367" - integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg== - -mimic-fn@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" - integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== - -mimic-response@^1.0.0, mimic-response@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" - integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== - -mini-create-react-context@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/mini-create-react-context/-/mini-create-react-context-0.4.1.tgz#072171561bfdc922da08a60c2197a497cc2d1d5e" - integrity sha512-YWCYEmd5CQeHGSAKrYvXgmzzkrvssZcuuQDDeqkT+PziKGMgE+0MCCtcKbROzocGBG1meBLl2FotlRwf4gAzbQ== - dependencies: - "@babel/runtime" "^7.12.1" - tiny-warning "^1.0.3" - -mini-css-extract-plugin@^1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-1.6.0.tgz#b4db2525af2624899ed64a23b0016e0036411893" - integrity sha512-nPFKI7NSy6uONUo9yn2hIfb9vyYvkFu95qki0e21DQ9uaqNKDP15DGpK0KnV6wDroWxPHtExrdEwx/yDQ8nVRw== - dependencies: - loader-utils "^2.0.0" - schema-utils "^3.0.0" - webpack-sources "^1.1.0" - -minimalistic-assert@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz#2e194de044626d4a10e7f7fbc00ce73e83e4d5c7" - integrity sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A== - -minimatch@3.0.4, minimatch@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" - integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== - dependencies: - brace-expansion "^1.1.7" - -minimist@^1.2.0, minimist@^1.2.5: - version "1.2.5" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" - integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== - -mixin-deep@^1.2.0: - version "1.3.2" - resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.2.tgz#1120b43dc359a785dce65b55b82e257ccf479566" - integrity sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA== - dependencies: - for-in "^1.0.2" - is-extendable "^1.0.1" - -mkdirp@^0.5.1, mkdirp@^0.5.5, mkdirp@~0.5.1: - version "0.5.5" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" - integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== - dependencies: - minimist "^1.2.5" - -module-alias@^2.2.2: - version "2.2.2" - resolved "https://registry.yarnpkg.com/module-alias/-/module-alias-2.2.2.tgz#151cdcecc24e25739ff0aa6e51e1c5716974c0e0" - integrity sha512-A/78XjoX2EmNvppVWEhM2oGk3x4lLxnkEA4jTbaK97QKSDjkIoOsKQlfylt/d3kKKi596Qy3NP5XrXJ6fZIC9Q== - -ms@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" - integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= - -ms@2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" - integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== - -ms@2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" - integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== - -ms@^2.1.1: - version "2.1.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" - integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== - -multicast-dns-service-types@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz#899f11d9686e5e05cb91b35d5f0e63b773cfc901" - integrity sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE= - -multicast-dns@^6.0.1: - version "6.2.3" - resolved "https://registry.yarnpkg.com/multicast-dns/-/multicast-dns-6.2.3.tgz#a0ec7bd9055c4282f790c3c82f4e28db3b31b229" - integrity sha512-ji6J5enbMyGRHIAkAOu3WdV8nggqviKCEKtXcOqfphZZtQrmHKycfynJ2V7eVPUA4NhJ6V7Wf4TmGbTwKE9B6g== - dependencies: - dns-packet "^1.3.1" - thunky "^1.0.2" - -nan@^2.12.1: - version "2.15.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.15.0.tgz#3f34a473ff18e15c1b5626b62903b5ad6e665fee" - integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ== - -nanoid@^3.1.28, nanoid@^3.1.30: - version "3.2.0" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.2.0.tgz#62667522da6673971cca916a6d3eff3f415ff80c" - integrity sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA== - -nanomatch@^1.2.9: - version "1.2.13" - resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.13.tgz#b87a8aa4fc0de8fe6be88895b38983ff265bd119" - integrity sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA== - dependencies: - arr-diff "^4.0.0" - array-unique "^0.3.2" - define-property "^2.0.2" - extend-shallow "^3.0.2" - fragment-cache "^0.2.1" - is-windows "^1.0.2" - kind-of "^6.0.2" - object.pick "^1.3.0" - regex-not "^1.0.0" - snapdragon "^0.8.1" - to-regex "^3.0.1" - -negotiator@0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" - integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== - -neo-async@^2.6.2: - version "2.6.2" - resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" - integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== - -nice-try@^1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" - integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== - -no-case@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/no-case/-/no-case-3.0.4.tgz#d361fd5c9800f558551a8369fc0dcd4662b6124d" - integrity sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg== - dependencies: - lower-case "^2.0.2" - tslib "^2.0.3" - -node-emoji@^1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.10.0.tgz#8886abd25d9c7bb61802a658523d1f8d2a89b2da" - integrity sha512-Yt3384If5H6BYGVHiHwTL+99OzJKHhgp82S8/dktEK73T26BazdgZ4JZh92xSVtGNJvz9UbXdNAc5hcrXV42vw== - dependencies: - lodash.toarray "^4.4.0" - -node-fetch@2.6.1: - version "2.6.1" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" - integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== - -node-forge@^0.10.0: - version "0.10.0" - resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" - integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA== - -node-releases@^1.1.61: - version "1.1.77" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.77.tgz#50b0cfede855dd374e7585bf228ff34e57c1c32e" - integrity sha512-rB1DUFUNAN4Gn9keO2K1efO35IDK7yKHCdCaIMvFO7yUYmmZYeDjnGKle26G4rwj+LKRQpjyUUvMkPglwGCYNQ== - -node-releases@^1.1.71: - version "1.1.71" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.71.tgz#cb1334b179896b1c89ecfdd4b725fb7bbdfc7dbb" - integrity sha512-zR6HoT6LrLCRBwukmrVbHv0EpEQjksO6GmFcZQQuCAy139BEsoVKPYnf3jongYW83fAa1torLGYwxxky/p28sg== - -node-releases@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.1.tgz#3d1d395f204f1f2f29a54358b9fb678765ad2fc5" - integrity sha512-CqyzN6z7Q6aMeF/ktcMVTzhAHCEpf8SOarwpzpf8pNBY2k5/oM34UHldUwp8VKI7uxct2HxSRdJjBaZeESzcxA== - -normalize-path@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.1.1.tgz#1ab28b556e198363a8c1a6f7e6fa20137fe6aed9" - integrity sha1-GrKLVW4Zg2Oowab35vogE3/mrtk= - dependencies: - remove-trailing-separator "^1.0.1" - -normalize-path@^3.0.0, normalize-path@~3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" - integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== - -normalize-range@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/normalize-range/-/normalize-range-0.1.2.tgz#2d10c06bdfd312ea9777695a4d28439456b75942" - integrity sha1-LRDAa9/TEuqXd2laTShDlFa3WUI= - -normalize-url@^4.1.0: - version "4.5.1" - resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.1.tgz#0dd90cf1288ee1d1313b87081c9a5932ee48518a" - integrity sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA== - -normalize-url@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.0.1.tgz#a4f27f58cf8c7b287b440b8a8201f42d0b00d256" - integrity sha512-VU4pzAuh7Kip71XEmO9aNREYAdMHFGTVj/i+CaTImS8x0i1d3jUZkXhqluy/PRgjPLMgsLQulYY3PJ/aSbSjpQ== - -npm-run-path@^2.0.0: - version "2.0.2" - resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f" - integrity sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8= - dependencies: - path-key "^2.0.0" - -npm-run-path@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea" - integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw== - dependencies: - path-key "^3.0.0" - -nprogress@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/nprogress/-/nprogress-0.2.0.tgz#cb8f34c53213d895723fcbab907e9422adbcafb1" - integrity sha1-y480xTIT2JVyP8urkH6UIq28r7E= - -nth-check@^1.0.2, nth-check@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.2.tgz#b2bd295c37e3dd58a3bf0700376663ba4d9cf05c" - integrity sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg== - dependencies: - boolbase "~1.0.0" - -nth-check@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.0.0.tgz#1bb4f6dac70072fc313e8c9cd1417b5074c0a125" - integrity sha512-i4sc/Kj8htBrAiH1viZ0TgU8Y5XqCaV/FziYK6TBczxmeKm3AEFWqqF3195yKudrarqy7Zu80Ra5dobFjn9X/Q== - dependencies: - boolbase "^1.0.0" - -object-assign@^4.0.1, object-assign@^4.1.0, object-assign@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= - -object-copy@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/object-copy/-/object-copy-0.1.0.tgz#7e7d858b781bd7c991a41ba975ed3812754e998c" - integrity sha1-fn2Fi3gb18mRpBupde04EnVOmYw= - dependencies: - copy-descriptor "^0.1.0" - define-property "^0.2.5" - kind-of "^3.0.3" - -object-inspect@^1.9.0: - version "1.10.2" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.10.2.tgz#b6385a3e2b7cae0b5eafcf90cddf85d128767f30" - integrity sha512-gz58rdPpadwztRrPjZE9DZLOABUpTGdcANUgOwBFO1C+HZZhePoP83M65WGDmbpwFYJSWqavbl4SgDn4k8RYTA== - -object-is@^1.0.1: - version "1.1.5" - resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.5.tgz#b9deeaa5fc7f1846a0faecdceec138e5778f53ac" - integrity sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" - -object-keys@^1.0.12, object-keys@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" - integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== - -object-visit@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/object-visit/-/object-visit-1.0.1.tgz#f79c4493af0c5377b59fe39d395e41042dd045bb" - integrity sha1-95xEk68MU3e1n+OdOV5BBC3QRbs= - dependencies: - isobject "^3.0.0" - -object.assign@^4.1.0, object.assign@^4.1.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.2.tgz#0ed54a342eceb37b38ff76eb831a0e788cb63940" - integrity sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ== - dependencies: - call-bind "^1.0.0" - define-properties "^1.1.3" - has-symbols "^1.0.1" - object-keys "^1.1.1" - -object.getownpropertydescriptors@^2.1.0: - version "2.1.2" - resolved "https://registry.yarnpkg.com/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.2.tgz#1bd63aeacf0d5d2d2f31b5e393b03a7c601a23f7" - integrity sha512-WtxeKSzfBjlzL+F9b7M7hewDzMwy+C8NRssHd1YrNlzHzIDrXcXiNOMrezdAEM4UXixgV+vvnyBeN7Rygl2ttQ== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" - es-abstract "^1.18.0-next.2" - -object.pick@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/object.pick/-/object.pick-1.3.0.tgz#87a10ac4c1694bd2e1cbf53591a66141fb5dd747" - integrity sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c= - dependencies: - isobject "^3.0.1" - -object.values@^1.1.0: - version "1.1.3" - resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.3.tgz#eaa8b1e17589f02f698db093f7c62ee1699742ee" - integrity sha512-nkF6PfDB9alkOUxpf1HNm/QlkeW3SReqL5WXeBLpEJJnlPSvRaDQpW3gQTksTN3fgJX4hL42RzKyOin6ff3tyw== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" - es-abstract "^1.18.0-next.2" - has "^1.0.3" - -obuf@^1.0.0, obuf@^1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/obuf/-/obuf-1.1.2.tgz#09bea3343d41859ebd446292d11c9d4db619084e" - integrity sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg== - -on-finished@~2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" - integrity sha1-IPEzZIGwg811M3mSoWlxqi2QaUc= - dependencies: - ee-first "1.1.1" - -on-headers@~1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.0.2.tgz#772b0ae6aaa525c399e489adfad90c403eb3c28f" - integrity sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA== - -once@^1.3.0, once@^1.3.1, once@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" - integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= - dependencies: - wrappy "1" - -onetime@^5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" - integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== - dependencies: - mimic-fn "^2.1.0" - -open@^7.0.2: - version "7.4.2" - resolved "https://registry.yarnpkg.com/open/-/open-7.4.2.tgz#b8147e26dcf3e426316c730089fd71edd29c2321" - integrity sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q== - dependencies: - is-docker "^2.0.0" - is-wsl "^2.1.1" - -opener@^1.5.2: - version "1.5.2" - resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598" - integrity sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A== - -opn@^5.5.0: - version "5.5.0" - resolved "https://registry.yarnpkg.com/opn/-/opn-5.5.0.tgz#fc7164fab56d235904c51c3b27da6758ca3b9bfc" - integrity sha512-PqHpggC9bLV0VeWcdKhkpxY+3JTzetLSqTCWL/z/tFIbI6G8JCjondXklT1JinczLz2Xib62sSp0T/gKT4KksA== - dependencies: - is-wsl "^1.1.0" - -original@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/original/-/original-1.0.2.tgz#e442a61cffe1c5fd20a65f3261c26663b303f25f" - integrity sha512-hyBVl6iqqUOJ8FqRe+l/gS8H+kKYjrEndd5Pm1MfBtsEKA038HkkdbAl/72EAXGyonD/PFsvmVG+EvcIpliMBg== - dependencies: - url-parse "^1.4.3" - -p-cancelable@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc" - integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw== - -p-finally@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" - integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4= - -p-limit@^2.0.0, p-limit@^2.2.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" - integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== - dependencies: - p-try "^2.0.0" - -p-limit@^3.0.2, p-limit@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" - integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== - dependencies: - yocto-queue "^0.1.0" - -p-locate@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4" - integrity sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ== - dependencies: - p-limit "^2.0.0" - -p-locate@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-4.1.0.tgz#a3428bb7088b3a60292f66919278b7c297ad4f07" - integrity sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A== - dependencies: - p-limit "^2.2.0" - -p-locate@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" - integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== - dependencies: - p-limit "^3.0.2" - -p-map@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" - integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== - -p-map@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b" - integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ== - dependencies: - aggregate-error "^3.0.0" - -p-retry@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-3.0.1.tgz#316b4c8893e2c8dc1cfa891f406c4b422bebf328" - integrity sha512-XE6G4+YTTkT2a0UWb2kjZe8xNwf8bIbnqpc/IS/idOBVhyves0mK5OJgeocjx7q5pvX/6m23xuzVPYT1uGM73w== - dependencies: - retry "^0.12.0" - -p-try@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" - integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== - -package-json@^6.3.0: - version "6.5.0" - resolved "https://registry.yarnpkg.com/package-json/-/package-json-6.5.0.tgz#6feedaca35e75725876d0b0e64974697fed145b0" - integrity sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ== - dependencies: - got "^9.6.0" - registry-auth-token "^4.0.0" - registry-url "^5.0.0" - semver "^6.2.0" - -param-case@^3.0.3, param-case@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/param-case/-/param-case-3.0.4.tgz#7d17fe4aa12bde34d4a77d91acfb6219caad01c5" - integrity sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A== - dependencies: - dot-case "^3.0.4" - tslib "^2.0.3" - -parent-module@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" - integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g== - dependencies: - callsites "^3.0.0" - -parse-entities@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/parse-entities/-/parse-entities-2.0.0.tgz#53c6eb5b9314a1f4ec99fa0fdf7ce01ecda0cbe8" - integrity sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ== - dependencies: - character-entities "^1.0.0" - character-entities-legacy "^1.0.0" - character-reference-invalid "^1.0.0" - is-alphanumerical "^1.0.0" - is-decimal "^1.0.0" - is-hexadecimal "^1.0.0" - -parse-json@^5.0.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd" - integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg== - dependencies: - "@babel/code-frame" "^7.0.0" - error-ex "^1.3.1" - json-parse-even-better-errors "^2.3.0" - lines-and-columns "^1.1.6" - -parse-numeric-range@^1.2.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/parse-numeric-range/-/parse-numeric-range-1.3.0.tgz#7c63b61190d61e4d53a1197f0c83c47bb670ffa3" - integrity sha512-twN+njEipszzlMJd4ONUYgSfZPDxgHhT9Ahed5uTigpQn90FggW4SA/AIPq/6a149fTbE9qBEcSwE3FAEp6wQQ== - -parse5@^5.0.0: - version "5.1.1" - resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178" - integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug== - -parse5@^6.0.0: - version "6.0.1" - resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b" - integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== - -parseurl@~1.3.2, parseurl@~1.3.3: - version "1.3.3" - resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" - integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== - -pascal-case@^3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/pascal-case/-/pascal-case-3.1.2.tgz#b48e0ef2b98e205e7c1dae747d0b1508237660eb" - integrity sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g== - dependencies: - no-case "^3.0.4" - tslib "^2.0.3" - -pascalcase@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/pascalcase/-/pascalcase-0.1.1.tgz#b363e55e8006ca6fe21784d2db22bd15d7917f14" - integrity sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ= - -path-dirname@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/path-dirname/-/path-dirname-1.0.2.tgz#cc33d24d525e099a5388c0336c6e32b9160609e0" - integrity sha1-zDPSTVJeCZpTiMAzbG4yuRYGCeA= - -path-exists@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" - integrity sha1-zg6+ql94yxiSXqfYENe1mwEP1RU= - -path-exists@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" - integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== - -path-is-absolute@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" - integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= - -path-is-inside@1.0.2, path-is-inside@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/path-is-inside/-/path-is-inside-1.0.2.tgz#365417dede44430d1c11af61027facf074bdfc53" - integrity sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM= - -path-key@^2.0.0, path-key@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" - integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A= - -path-key@^3.0.0, path-key@^3.1.0: - version "3.1.1" - resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" - integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== - -path-parse@^1.0.6, path-parse@^1.0.7: - version "1.0.7" - resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" - integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== - -path-to-regexp@0.1.7: - version "0.1.7" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" - integrity sha1-32BBeABfUi8V60SQ5yR6G/qmf4w= - -path-to-regexp@2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-2.2.1.tgz#90b617025a16381a879bc82a38d4e8bdeb2bcf45" - integrity sha512-gu9bD6Ta5bwGrrU8muHzVOBFFREpp2iRkVfhBJahwJ6p6Xw20SjT0MxLnwkjOibQmGSYhiUnf2FLe7k+jcFmGQ== - -path-to-regexp@^1.7.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.8.0.tgz#887b3ba9d84393e87a0a0b9f4cb756198b53548a" - integrity sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA== - dependencies: - isarray "0.0.1" - -path-type@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" - integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== - -picocolors@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-0.2.1.tgz#570670f793646851d1ba135996962abad587859f" - integrity sha512-cMlDqaLEqfSaW8Z7N5Jw+lyIW869EzT73/F5lhtY9cLGoVxSXznfgfXMO0Z5K0o0Q2TkTXq+0KFsdnSe3jDViA== - -picocolors@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" - integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== - -picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3: - version "2.2.3" - resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.3.tgz#465547f359ccc206d3c48e46a1bcb89bf7ee619d" - integrity sha512-KpELjfwcCDUb9PeigTs2mBJzXUPzAuP2oPcA989He8Rte0+YUAjw1JVedDhuTKPkHjSYzMN3npC9luThGYEKdg== - -pify@^2.0.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" - integrity sha1-7RQaasBDqEnqWISY59yosVMw6Qw= - -pify@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/pify/-/pify-4.0.1.tgz#4b2cd25c50d598735c50292224fd8c6df41e3231" - integrity sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g== - -pinkie-promise@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz#2135d6dfa7a358c069ac9b178776288228450ffa" - integrity sha1-ITXW36ejWMBprJsXh3YogihFD/o= - dependencies: - pinkie "^2.0.0" - -pinkie@^2.0.0: - version "2.0.4" - resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870" - integrity sha1-clVrgM+g1IqXToDnckjoDtT3+HA= - -pkg-dir@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3" - integrity sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw== - dependencies: - find-up "^3.0.0" - -pkg-dir@^4.1.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" - integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== - dependencies: - find-up "^4.0.0" - -pkg-up@3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/pkg-up/-/pkg-up-3.1.0.tgz#100ec235cc150e4fd42519412596a28512a0def5" - integrity sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA== - dependencies: - find-up "^3.0.0" - -portfinder@^1.0.26: - version "1.0.28" - resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.28.tgz#67c4622852bd5374dd1dd900f779f53462fac778" - integrity sha512-Se+2isanIcEqf2XMHjyUKskczxbPH7dQnlMjXX6+dybayyHvAf/TCgyMRlzf/B6QDhAEFOGes0pzRo3by4AbMA== - dependencies: - async "^2.6.2" - debug "^3.1.1" - mkdirp "^0.5.5" - -posix-character-classes@^0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab" - integrity sha1-AerA/jta9xoqbAL+q7jB/vfgDqs= - -postcss-calc@^8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/postcss-calc/-/postcss-calc-8.0.0.tgz#a05b87aacd132740a5db09462a3612453e5df90a" - integrity sha512-5NglwDrcbiy8XXfPM11F3HeC6hoT9W7GUH/Zi5U/p7u3Irv4rHhdDcIZwG0llHXV4ftsBjpfWMXAnXNl4lnt8g== - dependencies: - postcss-selector-parser "^6.0.2" - postcss-value-parser "^4.0.2" - -postcss-colormin@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/postcss-colormin/-/postcss-colormin-5.2.0.tgz#2b620b88c0ff19683f3349f4cf9e24ebdafb2c88" - integrity sha512-+HC6GfWU3upe5/mqmxuqYZ9B2Wl4lcoUUNkoaX59nEWV4EtADCMiBqui111Bu8R8IvaZTmqmxrqOAqjbHIwXPw== - dependencies: - browserslist "^4.16.6" - caniuse-api "^3.0.0" - colord "^2.0.1" - postcss-value-parser "^4.1.0" - -postcss-colormin@^5.2.1: - version "5.2.1" - resolved "https://registry.yarnpkg.com/postcss-colormin/-/postcss-colormin-5.2.1.tgz#6e444a806fd3c578827dbad022762df19334414d" - integrity sha512-VVwMrEYLcHYePUYV99Ymuoi7WhKrMGy/V9/kTS0DkCoJYmmjdOMneyhzYUxcNgteKDVbrewOkSM7Wje/MFwxzA== - dependencies: - browserslist "^4.16.6" - caniuse-api "^3.0.0" - colord "^2.9.1" - postcss-value-parser "^4.1.0" - -postcss-convert-values@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-convert-values/-/postcss-convert-values-5.0.1.tgz#4ec19d6016534e30e3102fdf414e753398645232" - integrity sha512-C3zR1Do2BkKkCgC0g3sF8TS0koF2G+mN8xxayZx3f10cIRmTaAnpgpRQZjNekTZxM2ciSPoh2IWJm0VZx8NoQg== - dependencies: - postcss-value-parser "^4.1.0" - -postcss-convert-values@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-convert-values/-/postcss-convert-values-5.0.2.tgz#879b849dc3677c7d6bc94b6a2c1a3f0808798059" - integrity sha512-KQ04E2yadmfa1LqXm7UIDwW1ftxU/QWZmz6NKnHnUvJ3LEYbbcX6i329f/ig+WnEByHegulocXrECaZGLpL8Zg== - dependencies: - postcss-value-parser "^4.1.0" - -postcss-discard-comments@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-discard-comments/-/postcss-discard-comments-5.0.1.tgz#9eae4b747cf760d31f2447c27f0619d5718901fe" - integrity sha512-lgZBPTDvWrbAYY1v5GYEv8fEO/WhKOu/hmZqmCYfrpD6eyDWWzAOsl2rF29lpvziKO02Gc5GJQtlpkTmakwOWg== - -postcss-discard-duplicates@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-discard-duplicates/-/postcss-discard-duplicates-5.0.1.tgz#68f7cc6458fe6bab2e46c9f55ae52869f680e66d" - integrity sha512-svx747PWHKOGpAXXQkCc4k/DsWo+6bc5LsVrAsw+OU+Ibi7klFZCyX54gjYzX4TH+f2uzXjRviLARxkMurA2bA== - -postcss-discard-empty@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-discard-empty/-/postcss-discard-empty-5.0.1.tgz#ee136c39e27d5d2ed4da0ee5ed02bc8a9f8bf6d8" - integrity sha512-vfU8CxAQ6YpMxV2SvMcMIyF2LX1ZzWpy0lqHDsOdaKKLQVQGVP1pzhrI9JlsO65s66uQTfkQBKBD/A5gp9STFw== - -postcss-discard-overridden@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-discard-overridden/-/postcss-discard-overridden-5.0.1.tgz#454b41f707300b98109a75005ca4ab0ff2743ac6" - integrity sha512-Y28H7y93L2BpJhrdUR2SR2fnSsT+3TVx1NmVQLbcnZWwIUpJ7mfcTC6Za9M2PG6w8j7UQRfzxqn8jU2VwFxo3Q== - -postcss-discard-unused@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-discard-unused/-/postcss-discard-unused-5.0.1.tgz#63e35a74a154912f93d4e75a1e6ff3cc146f934b" - integrity sha512-tD6xR/xyZTwfhKYRw0ylfCY8wbfhrjpKAMnDKRTLMy2fNW5hl0hoV6ap5vo2JdCkuHkP3CHw72beO4Y8pzFdww== - dependencies: - postcss-selector-parser "^6.0.5" - -postcss-loader@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/postcss-loader/-/postcss-loader-5.3.0.tgz#1657f869e48d4fdb018a40771c235e499ee26244" - integrity sha512-/+Z1RAmssdiSLgIZwnJHwBMnlABPgF7giYzTN2NOfr9D21IJZ4mQC1R2miwp80zno9M4zMD/umGI8cR+2EL5zw== - dependencies: - cosmiconfig "^7.0.0" - klona "^2.0.4" - semver "^7.3.4" - -postcss-merge-idents@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-merge-idents/-/postcss-merge-idents-5.0.1.tgz#6b5856fc28f2571f28ecce49effb9b0e64be9437" - integrity sha512-xu8ueVU0RszbI2gKkxR6mluupsOSSLvt8q4gA2fcKFkA+x6SlH3cb4cFHpDvcRCNFbUmCR/VUub+Y6zPOjPx+Q== - dependencies: - cssnano-utils "^2.0.1" - postcss-value-parser "^4.1.0" - -postcss-merge-longhand@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-merge-longhand/-/postcss-merge-longhand-5.0.2.tgz#277ada51d9a7958e8ef8cf263103c9384b322a41" - integrity sha512-BMlg9AXSI5G9TBT0Lo/H3PfUy63P84rVz3BjCFE9e9Y9RXQZD3+h3YO1kgTNsNJy7bBc1YQp8DmSnwLIW5VPcw== - dependencies: - css-color-names "^1.0.1" - postcss-value-parser "^4.1.0" - stylehacks "^5.0.1" - -postcss-merge-longhand@^5.0.4: - version "5.0.4" - resolved "https://registry.yarnpkg.com/postcss-merge-longhand/-/postcss-merge-longhand-5.0.4.tgz#41f4f3270282ea1a145ece078b7679f0cef21c32" - integrity sha512-2lZrOVD+d81aoYkZDpWu6+3dTAAGkCKbV5DoRhnIR7KOULVrI/R7bcMjhrH9KTRy6iiHKqmtG+n/MMj1WmqHFw== - dependencies: - postcss-value-parser "^4.1.0" - stylehacks "^5.0.1" - -postcss-merge-rules@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-merge-rules/-/postcss-merge-rules-5.0.2.tgz#d6e4d65018badbdb7dcc789c4f39b941305d410a" - integrity sha512-5K+Md7S3GwBewfB4rjDeol6V/RZ8S+v4B66Zk2gChRqLTCC8yjnHQ601omj9TKftS19OPGqZ/XzoqpzNQQLwbg== - dependencies: - browserslist "^4.16.6" - caniuse-api "^3.0.0" - cssnano-utils "^2.0.1" - postcss-selector-parser "^6.0.5" - vendors "^1.0.3" - -postcss-merge-rules@^5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/postcss-merge-rules/-/postcss-merge-rules-5.0.3.tgz#b5cae31f53129812a77e3eb1eeee448f8cf1a1db" - integrity sha512-cEKTMEbWazVa5NXd8deLdCnXl+6cYG7m2am+1HzqH0EnTdy8fRysatkaXb2dEnR+fdaDxTvuZ5zoBdv6efF6hg== - dependencies: - browserslist "^4.16.6" - caniuse-api "^3.0.0" - cssnano-utils "^2.0.1" - postcss-selector-parser "^6.0.5" - -postcss-minify-font-values@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-minify-font-values/-/postcss-minify-font-values-5.0.1.tgz#a90cefbfdaa075bd3dbaa1b33588bb4dc268addf" - integrity sha512-7JS4qIsnqaxk+FXY1E8dHBDmraYFWmuL6cgt0T1SWGRO5bzJf8sUoelwa4P88LEWJZweHevAiDKxHlofuvtIoA== - dependencies: - postcss-value-parser "^4.1.0" - -postcss-minify-gradients@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-minify-gradients/-/postcss-minify-gradients-5.0.2.tgz#7c175c108f06a5629925d698b3c4cf7bd3864ee5" - integrity sha512-7Do9JP+wqSD6Prittitt2zDLrfzP9pqKs2EcLX7HJYxsxCOwrrcLt4x/ctQTsiOw+/8HYotAoqNkrzItL19SdQ== - dependencies: - colord "^2.6" - cssnano-utils "^2.0.1" - postcss-value-parser "^4.1.0" - -postcss-minify-gradients@^5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/postcss-minify-gradients/-/postcss-minify-gradients-5.0.3.tgz#f970a11cc71e08e9095e78ec3a6b34b91c19550e" - integrity sha512-Z91Ol22nB6XJW+5oe31+YxRsYooxOdFKcbOqY/V8Fxse1Y3vqlNRpi1cxCqoACZTQEhl+xvt4hsbWiV5R+XI9Q== - dependencies: - colord "^2.9.1" - cssnano-utils "^2.0.1" - postcss-value-parser "^4.1.0" - -postcss-minify-params@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-minify-params/-/postcss-minify-params-5.0.1.tgz#371153ba164b9d8562842fdcd929c98abd9e5b6c" - integrity sha512-4RUC4k2A/Q9mGco1Z8ODc7h+A0z7L7X2ypO1B6V8057eVK6mZ6xwz6QN64nHuHLbqbclkX1wyzRnIrdZehTEHw== - dependencies: - alphanum-sort "^1.0.2" - browserslist "^4.16.0" - cssnano-utils "^2.0.1" - postcss-value-parser "^4.1.0" - uniqs "^2.0.0" - -postcss-minify-params@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-minify-params/-/postcss-minify-params-5.0.2.tgz#1b644da903473fbbb18fbe07b8e239883684b85c" - integrity sha512-qJAPuBzxO1yhLad7h2Dzk/F7n1vPyfHfCCh5grjGfjhi1ttCnq4ZXGIW77GSrEbh9Hus9Lc/e/+tB4vh3/GpDg== - dependencies: - alphanum-sort "^1.0.2" - browserslist "^4.16.6" - cssnano-utils "^2.0.1" - postcss-value-parser "^4.1.0" - -postcss-minify-selectors@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-minify-selectors/-/postcss-minify-selectors-5.1.0.tgz#4385c845d3979ff160291774523ffa54eafd5a54" - integrity sha512-NzGBXDa7aPsAcijXZeagnJBKBPMYLaJJzB8CQh6ncvyl2sIndLVWfbcDi0SBjRWk5VqEjXvf8tYwzoKf4Z07og== - dependencies: - alphanum-sort "^1.0.2" - postcss-selector-parser "^6.0.5" - -postcss-modules-extract-imports@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-extract-imports/-/postcss-modules-extract-imports-3.0.0.tgz#cda1f047c0ae80c97dbe28c3e76a43b88025741d" - integrity sha512-bdHleFnP3kZ4NYDhuGlVK+CMrQ/pqUm8bx/oGL93K6gVwiclvX5x0n76fYMKuIGKzlABOy13zsvqjb0f92TEXw== - -postcss-modules-local-by-default@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-local-by-default/-/postcss-modules-local-by-default-4.0.0.tgz#ebbb54fae1598eecfdf691a02b3ff3b390a5a51c" - integrity sha512-sT7ihtmGSF9yhm6ggikHdV0hlziDTX7oFoXtuVWeDd3hHObNkcHRo9V3yg7vCAY7cONyxJC/XXCmmiHHcvX7bQ== - dependencies: - icss-utils "^5.0.0" - postcss-selector-parser "^6.0.2" - postcss-value-parser "^4.1.0" - -postcss-modules-scope@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-scope/-/postcss-modules-scope-3.0.0.tgz#9ef3151456d3bbfa120ca44898dfca6f2fa01f06" - integrity sha512-hncihwFA2yPath8oZ15PZqvWGkWf+XUfQgUGamS4LqoP1anQLOsOJw0vr7J7IwLpoY9fatA2qiGUGmuZL0Iqlg== - dependencies: - postcss-selector-parser "^6.0.4" - -postcss-modules-values@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-values/-/postcss-modules-values-4.0.0.tgz#d7c5e7e68c3bb3c9b27cbf48ca0bb3ffb4602c9c" - integrity sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ== - dependencies: - icss-utils "^5.0.0" - -postcss-normalize-charset@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-charset/-/postcss-normalize-charset-5.0.1.tgz#121559d1bebc55ac8d24af37f67bd4da9efd91d0" - integrity sha512-6J40l6LNYnBdPSk+BHZ8SF+HAkS4q2twe5jnocgd+xWpz/mx/5Sa32m3W1AA8uE8XaXN+eg8trIlfu8V9x61eg== - -postcss-normalize-display-values@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-display-values/-/postcss-normalize-display-values-5.0.1.tgz#62650b965981a955dffee83363453db82f6ad1fd" - integrity sha512-uupdvWk88kLDXi5HEyI9IaAJTE3/Djbcrqq8YgjvAVuzgVuqIk3SuJWUisT2gaJbZm1H9g5k2w1xXilM3x8DjQ== - dependencies: - cssnano-utils "^2.0.1" - postcss-value-parser "^4.1.0" - -postcss-normalize-positions@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-positions/-/postcss-normalize-positions-5.0.1.tgz#868f6af1795fdfa86fbbe960dceb47e5f9492fe5" - integrity sha512-rvzWAJai5xej9yWqlCb1OWLd9JjW2Ex2BCPzUJrbaXmtKtgfL8dBMOOMTX6TnvQMtjk3ei1Lswcs78qKO1Skrg== - dependencies: - postcss-value-parser "^4.1.0" - -postcss-normalize-repeat-style@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-repeat-style/-/postcss-normalize-repeat-style-5.0.1.tgz#cbc0de1383b57f5bb61ddd6a84653b5e8665b2b5" - integrity sha512-syZ2itq0HTQjj4QtXZOeefomckiV5TaUO6ReIEabCh3wgDs4Mr01pkif0MeVwKyU/LHEkPJnpwFKRxqWA/7O3w== - dependencies: - cssnano-utils "^2.0.1" - postcss-value-parser "^4.1.0" - -postcss-normalize-string@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-string/-/postcss-normalize-string-5.0.1.tgz#d9eafaa4df78c7a3b973ae346ef0e47c554985b0" - integrity sha512-Ic8GaQ3jPMVl1OEn2U//2pm93AXUcF3wz+OriskdZ1AOuYV25OdgS7w9Xu2LO5cGyhHCgn8dMXh9bO7vi3i9pA== - dependencies: - postcss-value-parser "^4.1.0" - -postcss-normalize-timing-functions@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-timing-functions/-/postcss-normalize-timing-functions-5.0.1.tgz#8ee41103b9130429c6cbba736932b75c5e2cb08c" - integrity sha512-cPcBdVN5OsWCNEo5hiXfLUnXfTGtSFiBU9SK8k7ii8UD7OLuznzgNRYkLZow11BkQiiqMcgPyh4ZqXEEUrtQ1Q== - dependencies: - cssnano-utils "^2.0.1" - postcss-value-parser "^4.1.0" - -postcss-normalize-unicode@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-unicode/-/postcss-normalize-unicode-5.0.1.tgz#82d672d648a411814aa5bf3ae565379ccd9f5e37" - integrity sha512-kAtYD6V3pK0beqrU90gpCQB7g6AOfP/2KIPCVBKJM2EheVsBQmx/Iof+9zR9NFKLAx4Pr9mDhogB27pmn354nA== - dependencies: - browserslist "^4.16.0" - postcss-value-parser "^4.1.0" - -postcss-normalize-url@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-normalize-url/-/postcss-normalize-url-5.0.2.tgz#ddcdfb7cede1270740cf3e4dfc6008bd96abc763" - integrity sha512-k4jLTPUxREQ5bpajFQZpx8bCF2UrlqOTzP9kEqcEnOfwsRshWs2+oAFIHfDQB8GO2PaUaSE0NlTAYtbluZTlHQ== - dependencies: - is-absolute-url "^3.0.3" - normalize-url "^6.0.1" - postcss-value-parser "^4.1.0" - -postcss-normalize-url@^5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/postcss-normalize-url/-/postcss-normalize-url-5.0.3.tgz#42eca6ede57fe69075fab0f88ac8e48916ef931c" - integrity sha512-qWiUMbvkRx3kc1Dp5opzUwc7MBWZcSDK2yofCmdvFBCpx+zFPkxBC1FASQ59Pt+flYfj/nTZSkmF56+XG5elSg== - dependencies: - is-absolute-url "^3.0.3" - normalize-url "^6.0.1" - postcss-value-parser "^4.1.0" - -postcss-normalize-whitespace@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-whitespace/-/postcss-normalize-whitespace-5.0.1.tgz#b0b40b5bcac83585ff07ead2daf2dcfbeeef8e9a" - integrity sha512-iPklmI5SBnRvwceb/XH568yyzK0qRVuAG+a1HFUsFRf11lEJTiQQa03a4RSCQvLKdcpX7XsI1Gen9LuLoqwiqA== - dependencies: - postcss-value-parser "^4.1.0" - -postcss-ordered-values@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-ordered-values/-/postcss-ordered-values-5.0.2.tgz#1f351426977be00e0f765b3164ad753dac8ed044" - integrity sha512-8AFYDSOYWebJYLyJi3fyjl6CqMEG/UVworjiyK1r573I56kb3e879sCJLGvR3merj+fAdPpVplXKQZv+ey6CgQ== - dependencies: - cssnano-utils "^2.0.1" - postcss-value-parser "^4.1.0" - -postcss-reduce-idents@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-reduce-idents/-/postcss-reduce-idents-5.0.1.tgz#99b49ce8ee6f9c179447671cc9693e198e877bb7" - integrity sha512-6Rw8iIVFbqtaZExgWK1rpVgP7DPFRPh0DDFZxJ/ADNqPiH10sPCoq5tgo6kLiTyfh9sxjKYjXdc8udLEcPOezg== - dependencies: - postcss-value-parser "^4.1.0" - -postcss-reduce-initial@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-reduce-initial/-/postcss-reduce-initial-5.0.1.tgz#9d6369865b0f6f6f6b165a0ef5dc1a4856c7e946" - integrity sha512-zlCZPKLLTMAqA3ZWH57HlbCjkD55LX9dsRyxlls+wfuRfqCi5mSlZVan0heX5cHr154Dq9AfbH70LyhrSAezJw== - dependencies: - browserslist "^4.16.0" - caniuse-api "^3.0.0" - -postcss-reduce-initial@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-reduce-initial/-/postcss-reduce-initial-5.0.2.tgz#fa424ce8aa88a89bc0b6d0f94871b24abe94c048" - integrity sha512-v/kbAAQ+S1V5v9TJvbGkV98V2ERPdU6XvMcKMjqAlYiJ2NtsHGlKYLPjWWcXlaTKNxooId7BGxeraK8qXvzKtw== - dependencies: - browserslist "^4.16.6" - caniuse-api "^3.0.0" - -postcss-reduce-transforms@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-reduce-transforms/-/postcss-reduce-transforms-5.0.1.tgz#93c12f6a159474aa711d5269923e2383cedcf640" - integrity sha512-a//FjoPeFkRuAguPscTVmRQUODP+f3ke2HqFNgGPwdYnpeC29RZdCBvGRGTsKpMURb/I3p6jdKoBQ2zI+9Q7kA== - dependencies: - cssnano-utils "^2.0.1" - postcss-value-parser "^4.1.0" - -postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4: - version "6.0.5" - resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.5.tgz#042d74e137db83e6f294712096cb413f5aa612c4" - integrity sha512-aFYPoYmXbZ1V6HZaSvat08M97A8HqO6Pjz+PiNpw/DhuRrC72XWAdp3hL6wusDCN31sSmcZyMGa2hZEuX+Xfhg== - dependencies: - cssesc "^3.0.0" - util-deprecate "^1.0.2" - -postcss-selector-parser@^6.0.5: - version "6.0.6" - resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.6.tgz#2c5bba8174ac2f6981ab631a42ab0ee54af332ea" - integrity sha512-9LXrvaaX3+mcv5xkg5kFwqSzSH1JIObIx51PrndZwlmznwXRfxMddDvo9gve3gVR8ZTKgoFDdWkbRFmEhT4PMg== - dependencies: - cssesc "^3.0.0" - util-deprecate "^1.0.2" - -postcss-sort-media-queries@^3.10.11: - version "3.12.13" - resolved "https://registry.yarnpkg.com/postcss-sort-media-queries/-/postcss-sort-media-queries-3.12.13.tgz#213b1422e7efc29e0c2519b6ab259ff7d50b2156" - integrity sha512-bFbR1+P6HhZWXcT5DVV2pBH5Y2U5daKbFd0j+kcwKdzrxkbmgFu0GhI2JfFUyy5KQIeW+YJGP+vwNDOS5hIn2g== - dependencies: - sort-css-media-queries "2.0.4" - -postcss-svgo@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-svgo/-/postcss-svgo-5.0.2.tgz#bc73c4ea4c5a80fbd4b45e29042c34ceffb9257f" - integrity sha512-YzQuFLZu3U3aheizD+B1joQ94vzPfE6BNUcSYuceNxlVnKKsOtdo6hL9/zyC168Q8EwfLSgaDSalsUGa9f2C0A== - dependencies: - postcss-value-parser "^4.1.0" - svgo "^2.3.0" - -postcss-svgo@^5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/postcss-svgo/-/postcss-svgo-5.0.3.tgz#d945185756e5dfaae07f9edb0d3cae7ff79f9b30" - integrity sha512-41XZUA1wNDAZrQ3XgWREL/M2zSw8LJPvb5ZWivljBsUQAGoEKMYm6okHsTjJxKYI4M75RQEH4KYlEM52VwdXVA== - dependencies: - postcss-value-parser "^4.1.0" - svgo "^2.7.0" - -postcss-unique-selectors@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-unique-selectors/-/postcss-unique-selectors-5.0.1.tgz#3be5c1d7363352eff838bd62b0b07a0abad43bfc" - integrity sha512-gwi1NhHV4FMmPn+qwBNuot1sG1t2OmacLQ/AX29lzyggnjd+MnVD5uqQmpXO3J17KGL2WAxQruj1qTd3H0gG/w== - dependencies: - alphanum-sort "^1.0.2" - postcss-selector-parser "^6.0.5" - uniqs "^2.0.0" - -postcss-unique-selectors@^5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/postcss-unique-selectors/-/postcss-unique-selectors-5.0.2.tgz#5d6893daf534ae52626708e0d62250890108c0c1" - integrity sha512-w3zBVlrtZm7loQWRPVC0yjUwwpty7OM6DnEHkxcSQXO1bMS3RJ+JUS5LFMSDZHJcvGsRwhZinCWVqn8Kej4EDA== - dependencies: - alphanum-sort "^1.0.2" - postcss-selector-parser "^6.0.5" - -postcss-value-parser@^4.0.2, postcss-value-parser@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.1.0.tgz#443f6a20ced6481a2bda4fa8532a6e55d789a2cb" - integrity sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ== - -postcss-zindex@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/postcss-zindex/-/postcss-zindex-5.0.1.tgz#c585724beb69d356af8c7e68847b28d6298ece03" - integrity sha512-nwgtJJys+XmmSGoYCcgkf/VczP8Mp/0OfSv3v0+fw0uABY4yxw+eFs0Xp9nAZHIKnS5j+e9ywQ+RD+ONyvl5pA== - -"postcss@5 - 7": - version "7.0.39" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.39.tgz#9624375d965630e2e1f2c02a935c82a59cb48309" - integrity sha512-yioayjNbHn6z1/Bywyb2Y4s3yvDAeXGOyxqD+LnVOinq6Mdmd++SW2wUNVzavyyHxd6+DxzWGIuosg6P1Rj8uA== - dependencies: - picocolors "^0.2.1" - source-map "^0.6.1" - -postcss@^8.2.10, postcss@^8.3.5: - version "8.3.9" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.9.tgz#98754caa06c4ee9eb59cc48bd073bb6bd3437c31" - integrity sha512-f/ZFyAKh9Dnqytx5X62jgjhhzttjZS7hMsohcI7HEI5tjELX/HxCy3EFhsRxyzGvrzFF+82XPvCS8T9TFleVJw== - dependencies: - nanoid "^3.1.28" - picocolors "^0.2.1" - source-map-js "^0.6.2" - -postcss@^8.2.15, postcss@^8.3.11: - version "8.4.4" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.4.tgz#d53d4ec6a75fd62557a66bb41978bf47ff0c2869" - integrity sha512-joU6fBsN6EIer28Lj6GDFoC/5yOZzLCfn0zHAn/MYXI7aPt4m4hK5KC5ovEZXy+lnCjmYIbQWngvju2ddyEr8Q== - dependencies: - nanoid "^3.1.30" - picocolors "^1.0.0" - source-map-js "^1.0.1" - -prepend-http@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897" - integrity sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc= - -pretty-error@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/pretty-error/-/pretty-error-4.0.0.tgz#90a703f46dd7234adb46d0f84823e9d1cb8f10d6" - integrity sha512-AoJ5YMAcXKYxKhuJGdcvse+Voc6v1RgnsR3nWcYU7q4t6z0Q6T86sv5Zq8VIRbOWWFpvdGE83LtdSMNd+6Y0xw== - dependencies: - lodash "^4.17.20" - renderkid "^3.0.0" - -pretty-time@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/pretty-time/-/pretty-time-1.1.0.tgz#ffb7429afabb8535c346a34e41873adf3d74dd0e" - integrity sha512-28iF6xPQrP8Oa6uxE6a1biz+lWeTOAPKggvjB8HAs6nVMKZwf5bG++632Dx614hIWgUPkgivRfG+a8uAXGTIbA== - -prism-react-renderer@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/prism-react-renderer/-/prism-react-renderer-1.2.1.tgz#392460acf63540960e5e3caa699d851264e99b89" - integrity sha512-w23ch4f75V1Tnz8DajsYKvY5lF7H1+WvzvLUcF0paFxkTHSp42RS0H5CttdN2Q8RR3DRGZ9v5xD/h3n8C8kGmg== - -prismjs@^1.23.0: - version "1.25.0" - resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.25.0.tgz#6f822df1bdad965734b310b315a23315cf999756" - integrity sha512-WCjJHl1KEWbnkQom1+SzftbtXMKQoezOCYs5rECqMN+jP+apI7ftoflyqigqzopSO3hMhTEb0mFClA8lkolgEg== - -process-nextick-args@~2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" - integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== - -promise@^7.1.1: - version "7.3.1" - resolved "https://registry.yarnpkg.com/promise/-/promise-7.3.1.tgz#064b72602b18f90f29192b8b1bc418ffd1ebd3bf" - integrity sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg== - dependencies: - asap "~2.0.3" - -prompts@2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.0.tgz#4aa5de0723a231d1ee9121c40fdf663df73f61d7" - integrity sha512-awZAKrk3vN6CroQukBL+R9051a4R3zCZBlJm/HBfrSZ8iTpYix3VX1vU4mveiLpiwmOJT4wokTF9m6HUk4KqWQ== - dependencies: - kleur "^3.0.3" - sisteransi "^1.0.5" - -prompts@^2.4.1: - version "2.4.2" - resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069" - integrity sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q== - dependencies: - kleur "^3.0.3" - sisteransi "^1.0.5" - -prop-types@^15.5.0, prop-types@^15.6.2, prop-types@^15.7.2: - version "15.7.2" - resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.7.2.tgz#52c41e75b8c87e72b9d9360e0206b99dcbffa6c5" - integrity sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ== - dependencies: - loose-envify "^1.4.0" - object-assign "^4.1.1" - react-is "^16.8.1" - -property-information@^5.0.0, property-information@^5.3.0: - version "5.6.0" - resolved "https://registry.yarnpkg.com/property-information/-/property-information-5.6.0.tgz#61675545fb23002f245c6540ec46077d4da3ed69" - integrity sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA== - dependencies: - xtend "^4.0.0" - -proxy-addr@~2.0.5: - version "2.0.6" - resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.6.tgz#fdc2336505447d3f2f2c638ed272caf614bbb2bf" - integrity sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw== - dependencies: - forwarded "~0.1.2" - ipaddr.js "1.9.1" - -prr@~1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" - integrity sha1-0/wRS6BplaRexok/SEzrHXj19HY= - -pump@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" - integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== - dependencies: - end-of-stream "^1.1.0" - once "^1.3.1" - -punycode@1.3.2: - version "1.3.2" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" - integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= - -punycode@^1.3.2: - version "1.4.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" - integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= - -punycode@^2.1.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" - integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== - -pupa@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.1.1.tgz#f5e8fd4afc2c5d97828faa523549ed8744a20d62" - integrity sha512-l1jNAspIBSFqbT+y+5FosojNpVpF94nlI+wDUpqP9enwOTfHx9f0gh5nB96vl+6yTpsJsypeNrwfzPrKuHB41A== - dependencies: - escape-goat "^2.0.0" - -pure-color@^1.2.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/pure-color/-/pure-color-1.3.0.tgz#1fe064fb0ac851f0de61320a8bf796836422f33e" - integrity sha1-H+Bk+wrIUfDeYTIKi/eWg2Qi8z4= - -q@^1.1.2: - version "1.5.1" - resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" - integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc= - -qs@6.7.0: - version "6.7.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" - integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== - -querystring@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" - integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= - -querystringify@^2.1.1: - version "2.2.0" - resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" - integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ== - -queue-microtask@^1.2.2: - version "1.2.3" - resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" - integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== - -randombytes@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" - integrity sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ== - dependencies: - safe-buffer "^5.1.0" - -range-parser@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.0.tgz#f49be6b487894ddc40dcc94a322f611092e00d5e" - integrity sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4= - -range-parser@^1.2.1, range-parser@~1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" - integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== - -raw-body@2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.0.tgz#a1ce6fb9c9bc356ca52e89256ab59059e13d0332" - integrity sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q== - dependencies: - bytes "3.1.0" - http-errors "1.7.2" - iconv-lite "0.4.24" - unpipe "1.0.0" - -rc@^1.2.8: - version "1.2.8" - resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" - integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== - dependencies: - deep-extend "^0.6.0" - ini "~1.3.0" - minimist "^1.2.0" - strip-json-comments "~2.0.1" - -react-base16-styling@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/react-base16-styling/-/react-base16-styling-0.6.0.tgz#ef2156d66cf4139695c8a167886cb69ea660792c" - integrity sha1-7yFW1mz0E5aVyKFniGy2nqZgeSw= - dependencies: - base16 "^1.0.0" - lodash.curry "^4.0.1" - lodash.flow "^3.3.0" - pure-color "^1.2.0" - -react-dev-utils@^11.0.1: - version "11.0.4" - resolved "https://registry.yarnpkg.com/react-dev-utils/-/react-dev-utils-11.0.4.tgz#a7ccb60257a1ca2e0efe7a83e38e6700d17aa37a" - integrity sha512-dx0LvIGHcOPtKbeiSUM4jqpBl3TcY7CDjZdfOIcKeznE7BWr9dg0iPG90G5yfVQ+p/rGNMXdbfStvzQZEVEi4A== - dependencies: - "@babel/code-frame" "7.10.4" - address "1.1.2" - browserslist "4.14.2" - chalk "2.4.2" - cross-spawn "7.0.3" - detect-port-alt "1.1.6" - escape-string-regexp "2.0.0" - filesize "6.1.0" - find-up "4.1.0" - fork-ts-checker-webpack-plugin "4.1.6" - global-modules "2.0.0" - globby "11.0.1" - gzip-size "5.1.1" - immer "8.0.1" - is-root "2.1.0" - loader-utils "2.0.0" - open "^7.0.2" - pkg-up "3.1.0" - prompts "2.4.0" - react-error-overlay "^6.0.9" - recursive-readdir "2.2.2" - shell-quote "1.7.2" - strip-ansi "6.0.0" - text-table "0.2.0" - -react-dom@^17.0.2: - version "17.0.2" - resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-17.0.2.tgz#ecffb6845e3ad8dbfcdc498f0d0a939736502c23" - integrity sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA== - dependencies: - loose-envify "^1.1.0" - object-assign "^4.1.1" - scheduler "^0.20.2" - -react-error-overlay@^6.0.9: - version "6.0.9" - resolved "https://registry.yarnpkg.com/react-error-overlay/-/react-error-overlay-6.0.9.tgz#3c743010c9359608c375ecd6bc76f35d93995b0a" - integrity sha512-nQTTcUu+ATDbrSD1BZHr5kgSD4oF8OFjxun8uAaL8RwPBacGBNPf/yAuVVdx17N8XNzRDMrZ9XcKZHCjPW+9ew== - -react-fast-compare@^3.1.1: - version "3.2.0" - resolved "https://registry.yarnpkg.com/react-fast-compare/-/react-fast-compare-3.2.0.tgz#641a9da81b6a6320f270e89724fb45a0b39e43bb" - integrity sha512-rtGImPZ0YyLrscKI9xTpV8psd6I8VAtjKCzQDlzyDvqJA8XOW78TXYQwNRNd8g8JZnDu8q9Fu/1v4HPAVwVdHA== - -react-helmet@^6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/react-helmet/-/react-helmet-6.1.0.tgz#a750d5165cb13cf213e44747502652e794468726" - integrity sha512-4uMzEY9nlDlgxr61NL3XbKRy1hEkXmKNXhjbAIOVw5vcFrsdYbH2FEwcNyWvWinl103nXgzYNlns9ca+8kFiWw== - dependencies: - object-assign "^4.1.1" - prop-types "^15.7.2" - react-fast-compare "^3.1.1" - react-side-effect "^2.1.0" - -react-is@^16.6.0, react-is@^16.7.0, react-is@^16.8.1: - version "16.13.1" - resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4" - integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== - -react-json-view@^1.21.3: - version "1.21.3" - resolved "https://registry.yarnpkg.com/react-json-view/-/react-json-view-1.21.3.tgz#f184209ee8f1bf374fb0c41b0813cff54549c475" - integrity sha512-13p8IREj9/x/Ye4WI/JpjhoIwuzEgUAtgJZNBJckfzJt1qyh24BdTm6UQNGnyTq9dapQdrqvquZTo3dz1X6Cjw== - dependencies: - flux "^4.0.1" - react-base16-styling "^0.6.0" - react-lifecycles-compat "^3.0.4" - react-textarea-autosize "^8.3.2" - -react-lifecycles-compat@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz#4f1a273afdfc8f3488a8c516bfda78f872352362" - integrity sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA== - -react-loadable-ssr-addon-v5-slorber@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/react-loadable-ssr-addon-v5-slorber/-/react-loadable-ssr-addon-v5-slorber-1.0.1.tgz#2cdc91e8a744ffdf9e3556caabeb6e4278689883" - integrity sha512-lq3Lyw1lGku8zUEJPDxsNm1AfYHBrO9Y1+olAYwpUJ2IGFBskM0DMKok97A6LWUpHm+o7IvQBOWu9MLenp9Z+A== - dependencies: - "@babel/runtime" "^7.10.3" - -react-loadable@^5.5.0: - version "5.5.0" - resolved "https://registry.yarnpkg.com/react-loadable/-/react-loadable-5.5.0.tgz#582251679d3da86c32aae2c8e689c59f1196d8c4" - integrity sha512-C8Aui0ZpMd4KokxRdVAm2bQtI03k2RMRNzOB+IipV3yxFTSVICv7WoUr5L9ALB5BmKO1iHgZtWM8EvYG83otdg== - dependencies: - prop-types "^15.5.0" - -react-router-config@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/react-router-config/-/react-router-config-5.1.1.tgz#0f4263d1a80c6b2dc7b9c1902c9526478194a988" - integrity sha512-DuanZjaD8mQp1ppHjgnnUnyOlqYXZVjnov/JzFhjLEwd3Z4dYjMSnqrEzzGThH47vpCOqPPwJM2FtthLeJ8Pbg== - dependencies: - "@babel/runtime" "^7.1.2" - -react-router-dom@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-5.2.0.tgz#9e65a4d0c45e13289e66c7b17c7e175d0ea15662" - integrity sha512-gxAmfylo2QUjcwxI63RhQ5G85Qqt4voZpUXSEqCwykV0baaOTQDR1f0PmY8AELqIyVc0NEZUj0Gov5lNGcXgsA== - dependencies: - "@babel/runtime" "^7.1.2" - history "^4.9.0" - loose-envify "^1.3.1" - prop-types "^15.6.2" - react-router "5.2.0" - tiny-invariant "^1.0.2" - tiny-warning "^1.0.0" - -react-router@5.2.0, react-router@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-5.2.0.tgz#424e75641ca8747fbf76e5ecca69781aa37ea293" - integrity sha512-smz1DUuFHRKdcJC0jobGo8cVbhO3x50tCL4icacOlcwDOEQPq4TMqwx3sY1TP+DvtTgz4nm3thuo7A+BK2U0Dw== - dependencies: - "@babel/runtime" "^7.1.2" - history "^4.9.0" - hoist-non-react-statics "^3.1.0" - loose-envify "^1.3.1" - mini-create-react-context "^0.4.0" - path-to-regexp "^1.7.0" - prop-types "^15.6.2" - react-is "^16.6.0" - tiny-invariant "^1.0.2" - tiny-warning "^1.0.0" - -react-side-effect@^2.1.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/react-side-effect/-/react-side-effect-2.1.1.tgz#66c5701c3e7560ab4822a4ee2742dee215d72eb3" - integrity sha512-2FoTQzRNTncBVtnzxFOk2mCpcfxQpenBMbk5kSVBg5UcPqV9fRbgY2zhb7GTWWOlpFmAxhClBDlIq8Rsubz1yQ== - -react-textarea-autosize@^8.3.2: - version "8.3.2" - resolved "https://registry.yarnpkg.com/react-textarea-autosize/-/react-textarea-autosize-8.3.2.tgz#4f9374d357b0a6f6469956726722549124a1b2db" - integrity sha512-JrMWVgQSaExQByP3ggI1eA8zF4mF0+ddVuX7acUeK2V7bmrpjVOY72vmLz2IXFJSAXoY3D80nEzrn0GWajWK3Q== - dependencies: - "@babel/runtime" "^7.10.2" - use-composed-ref "^1.0.0" - use-latest "^1.0.0" - -react@^17.0.2: - version "17.0.2" - resolved "https://registry.yarnpkg.com/react/-/react-17.0.2.tgz#d0b5cc516d29eb3eee383f75b62864cfb6800037" - integrity sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA== - dependencies: - loose-envify "^1.1.0" - object-assign "^4.1.1" - -readable-stream@^2.0.1, readable-stream@^2.0.2: - version "2.3.7" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" - integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.3" - isarray "~1.0.0" - process-nextick-args "~2.0.0" - safe-buffer "~5.1.1" - string_decoder "~1.1.1" - util-deprecate "~1.0.1" - -readable-stream@^3.0.6, readable-stream@^3.1.1: - version "3.6.0" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" - integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== - dependencies: - inherits "^2.0.3" - string_decoder "^1.1.1" - util-deprecate "^1.0.1" - -readdirp@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-2.2.1.tgz#0e87622a3325aa33e892285caf8b4e846529a525" - integrity sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ== - dependencies: - graceful-fs "^4.1.11" - micromatch "^3.1.10" - readable-stream "^2.0.2" - -readdirp@~3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" - integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA== - dependencies: - picomatch "^2.2.1" - -reading-time@^1.3.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/reading-time/-/reading-time-1.5.0.tgz#d2a7f1b6057cb2e169beaf87113cc3411b5bc5bb" - integrity sha512-onYyVhBNr4CmAxFsKS7bz+uTLRakypIe4R+5A824vBSkQy/hB3fZepoVEf8OVAxzLvK+H/jm9TzpI3ETSm64Kg== - -rechoir@^0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" - integrity sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q= - dependencies: - resolve "^1.1.6" - -recursive-readdir@2.2.2: - version "2.2.2" - resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.2.tgz#9946fb3274e1628de6e36b2f6714953b4845094f" - integrity sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg== - dependencies: - minimatch "3.0.4" - -regenerate-unicode-properties@^8.2.0: - version "8.2.0" - resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-8.2.0.tgz#e5de7111d655e7ba60c057dbe9ff37c87e65cdec" - integrity sha512-F9DjY1vKLo/tPePDycuH3dn9H1OTPIkVD9Kz4LODu+F2C75mgjAJ7x/gwy6ZcSNRAAkhNlJSOHRe8k3p+K9WhA== - dependencies: - regenerate "^1.4.0" - -regenerate@^1.4.0: - version "1.4.2" - resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.4.2.tgz#b9346d8827e8f5a32f7ba29637d398b69014848a" - integrity sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A== - -regenerator-runtime@^0.13.4: - version "0.13.7" - resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz#cac2dacc8a1ea675feaabaeb8ae833898ae46f55" - integrity sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew== - -regenerator-transform@^0.14.2: - version "0.14.5" - resolved "https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.14.5.tgz#c98da154683671c9c4dcb16ece736517e1b7feb4" - integrity sha512-eOf6vka5IO151Jfsw2NO9WpGX58W6wWmefK3I1zEGr0lOD0u8rwPaNqQL1aRxUaxLeKO3ArNh3VYg1KbaD+FFw== - dependencies: - "@babel/runtime" "^7.8.4" - -regex-not@^1.0.0, regex-not@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c" - integrity sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A== - dependencies: - extend-shallow "^3.0.2" - safe-regex "^1.1.0" - -regexp.prototype.flags@^1.2.0: - version "1.3.1" - resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.3.1.tgz#7ef352ae8d159e758c0eadca6f8fcb4eef07be26" - integrity sha512-JiBdRBq91WlY7uRJ0ds7R+dU02i6LKi8r3BuQhNXn+kmeLN+EfHhfjqMRis1zJxnlu88hq/4dx0P2OP3APRTOA== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" - -regexpu-core@^4.7.1: - version "4.7.1" - resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-4.7.1.tgz#2dea5a9a07233298fbf0db91fa9abc4c6e0f8ad6" - integrity sha512-ywH2VUraA44DZQuRKzARmw6S66mr48pQVva4LBeRhcOltJ6hExvWly5ZjFLYo67xbIxb6W1q4bAGtgfEl20zfQ== - dependencies: - regenerate "^1.4.0" - regenerate-unicode-properties "^8.2.0" - regjsgen "^0.5.1" - regjsparser "^0.6.4" - unicode-match-property-ecmascript "^1.0.4" - unicode-match-property-value-ecmascript "^1.2.0" - -registry-auth-token@^4.0.0: - version "4.2.1" - resolved "https://registry.yarnpkg.com/registry-auth-token/-/registry-auth-token-4.2.1.tgz#6d7b4006441918972ccd5fedcd41dc322c79b250" - integrity sha512-6gkSb4U6aWJB4SF2ZvLb76yCBjcvufXBqvvEx1HbmKPkutswjW1xNVRY0+daljIYRbogN7O0etYSlbiaEQyMyw== - dependencies: - rc "^1.2.8" - -registry-url@^5.0.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/registry-url/-/registry-url-5.1.0.tgz#e98334b50d5434b81136b44ec638d9c2009c5009" - integrity sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw== - dependencies: - rc "^1.2.8" - -regjsgen@^0.5.1: - version "0.5.2" - resolved "https://registry.yarnpkg.com/regjsgen/-/regjsgen-0.5.2.tgz#92ff295fb1deecbf6ecdab2543d207e91aa33733" - integrity sha512-OFFT3MfrH90xIW8OOSyUrk6QHD5E9JOTeGodiJeBS3J6IwlgzJMNE/1bZklWz5oTg+9dCMyEetclvCVXOPoN3A== - -regjsparser@^0.6.4: - version "0.6.9" - resolved "https://registry.yarnpkg.com/regjsparser/-/regjsparser-0.6.9.tgz#b489eef7c9a2ce43727627011429cf833a7183e6" - integrity sha512-ZqbNRz1SNjLAiYuwY0zoXW8Ne675IX5q+YHioAGbCw4X96Mjl2+dcX9B2ciaeyYjViDAfvIjFpQjJgLttTEERQ== - dependencies: - jsesc "~0.5.0" - -rehype-parse@^6.0.2: - version "6.0.2" - resolved "https://registry.yarnpkg.com/rehype-parse/-/rehype-parse-6.0.2.tgz#aeb3fdd68085f9f796f1d3137ae2b85a98406964" - integrity sha512-0S3CpvpTAgGmnz8kiCyFLGuW5yA4OQhyNTm/nwPopZ7+PI11WnGl1TTWTGv/2hPEe/g2jRLlhVVSsoDH8waRug== - dependencies: - hast-util-from-parse5 "^5.0.0" - parse5 "^5.0.0" - xtend "^4.0.0" - -relateurl@^0.2.7: - version "0.2.7" - resolved "https://registry.yarnpkg.com/relateurl/-/relateurl-0.2.7.tgz#54dbf377e51440aca90a4cd274600d3ff2d888a9" - integrity sha1-VNvzd+UUQKypCkzSdGANP/LYiKk= - -remark-admonitions@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/remark-admonitions/-/remark-admonitions-1.2.1.tgz#87caa1a442aa7b4c0cafa04798ed58a342307870" - integrity sha512-Ji6p68VDvD+H1oS95Fdx9Ar5WA2wcDA4kwrrhVU7fGctC6+d3uiMICu7w7/2Xld+lnU7/gi+432+rRbup5S8ow== - dependencies: - rehype-parse "^6.0.2" - unified "^8.4.2" - unist-util-visit "^2.0.1" - -remark-emoji@^2.1.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/remark-emoji/-/remark-emoji-2.2.0.tgz#1c702090a1525da5b80e15a8f963ef2c8236cac7" - integrity sha512-P3cj9s5ggsUvWw5fS2uzCHJMGuXYRb0NnZqYlNecewXt8QBU9n5vW3DUUKOhepS8F9CwdMx9B8a3i7pqFWAI5w== - dependencies: - emoticon "^3.2.0" - node-emoji "^1.10.0" - unist-util-visit "^2.0.3" - -remark-footnotes@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/remark-footnotes/-/remark-footnotes-2.0.0.tgz#9001c4c2ffebba55695d2dd80ffb8b82f7e6303f" - integrity sha512-3Clt8ZMH75Ayjp9q4CorNeyjwIxHFcTkaektplKGl2A1jNGEUey8cKL0ZC5vJwfcD5GFGsNLImLG/NGzWIzoMQ== - -remark-mdx@1.6.22: - version "1.6.22" - resolved "https://registry.yarnpkg.com/remark-mdx/-/remark-mdx-1.6.22.tgz#06a8dab07dcfdd57f3373af7f86bd0e992108bbd" - integrity sha512-phMHBJgeV76uyFkH4rvzCftLfKCr2RZuF+/gmVcaKrpsihyzmhXjA0BEMDaPTXG5y8qZOKPVo83NAOX01LPnOQ== - dependencies: - "@babel/core" "7.12.9" - "@babel/helper-plugin-utils" "7.10.4" - "@babel/plugin-proposal-object-rest-spread" "7.12.1" - "@babel/plugin-syntax-jsx" "7.12.1" - "@mdx-js/util" "1.6.22" - is-alphabetical "1.0.4" - remark-parse "8.0.3" - unified "9.2.0" - -remark-parse@8.0.3: - version "8.0.3" - resolved "https://registry.yarnpkg.com/remark-parse/-/remark-parse-8.0.3.tgz#9c62aa3b35b79a486454c690472906075f40c7e1" - integrity sha512-E1K9+QLGgggHxCQtLt++uXltxEprmWzNfg+MxpfHsZlrddKzZ/hZyWHDbK3/Ap8HJQqYJRXP+jHczdL6q6i85Q== - dependencies: - ccount "^1.0.0" - collapse-white-space "^1.0.2" - is-alphabetical "^1.0.0" - is-decimal "^1.0.0" - is-whitespace-character "^1.0.0" - is-word-character "^1.0.0" - markdown-escapes "^1.0.0" - parse-entities "^2.0.0" - repeat-string "^1.5.4" - state-toggle "^1.0.0" - trim "0.0.1" - trim-trailing-lines "^1.0.0" - unherit "^1.0.4" - unist-util-remove-position "^2.0.0" - vfile-location "^3.0.0" - xtend "^4.0.1" - -remark-squeeze-paragraphs@4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/remark-squeeze-paragraphs/-/remark-squeeze-paragraphs-4.0.0.tgz#76eb0e085295131c84748c8e43810159c5653ead" - integrity sha512-8qRqmL9F4nuLPIgl92XUuxI3pFxize+F1H0e/W3llTk0UsjJaj01+RrirkMw7P21RKe4X6goQhYRSvNWX+70Rw== - dependencies: - mdast-squeeze-paragraphs "^4.0.0" - -remove-trailing-separator@^1.0.1: - version "1.1.0" - resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef" - integrity sha1-wkvOKig62tW8P1jg1IJJuSN52O8= - -renderkid@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/renderkid/-/renderkid-3.0.0.tgz#5fd823e4d6951d37358ecc9a58b1f06836b6268a" - integrity sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg== - dependencies: - css-select "^4.1.3" - dom-converter "^0.2.0" - htmlparser2 "^6.1.0" - lodash "^4.17.21" - strip-ansi "^6.0.1" - -repeat-element@^1.1.2: - version "1.1.4" - resolved "https://registry.yarnpkg.com/repeat-element/-/repeat-element-1.1.4.tgz#be681520847ab58c7568ac75fbfad28ed42d39e9" - integrity sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ== - -repeat-string@^1.5.4, repeat-string@^1.6.1: - version "1.6.1" - resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" - integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= - -require-directory@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" - integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I= - -require-from-string@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" - integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== - -"require-like@>= 0.1.1": - version "0.1.2" - resolved "https://registry.yarnpkg.com/require-like/-/require-like-0.1.2.tgz#ad6f30c13becd797010c468afa775c0c0a6b47fa" - integrity sha1-rW8wwTvs15cBDEaK+ndcDAprR/o= - -require-main-filename@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" - integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg== - -requires-port@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" - integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8= - -resolve-cwd@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-2.0.0.tgz#00a9f7387556e27038eae232caa372a6a59b665a" - integrity sha1-AKn3OHVW4nA46uIyyqNypqWbZlo= - dependencies: - resolve-from "^3.0.0" - -resolve-from@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-3.0.0.tgz#b22c7af7d9d6881bc8b6e653335eebcb0a188748" - integrity sha1-six699nWiBvItuZTM17rywoYh0g= - -resolve-from@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" - integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== - -resolve-pathname@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/resolve-pathname/-/resolve-pathname-3.0.0.tgz#99d02224d3cf263689becbb393bc560313025dcd" - integrity sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng== - -resolve-url@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" - integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo= - -resolve@^1.1.6: - version "1.21.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.21.0.tgz#b51adc97f3472e6a5cf4444d34bc9d6b9037591f" - integrity sha512-3wCbTpk5WJlyE4mSOtDLhqQmGFi0/TD9VPwmiolnk8U0wRgMEktqCXd3vy5buTO3tljvalNvKrjHEfrd2WpEKA== - dependencies: - is-core-module "^2.8.0" - path-parse "^1.0.7" - supports-preserve-symlinks-flag "^1.0.0" - -resolve@^1.14.2, resolve@^1.3.2: - version "1.20.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975" - integrity sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A== - dependencies: - is-core-module "^2.2.0" - path-parse "^1.0.6" - -responselike@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/responselike/-/responselike-1.0.2.tgz#918720ef3b631c5642be068f15ade5a46f4ba1e7" - integrity sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec= - dependencies: - lowercase-keys "^1.0.0" - -ret@~0.1.10: - version "0.1.15" - resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" - integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg== - -retry@^0.12.0: - version "0.12.0" - resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b" - integrity sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs= - -reusify@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" - integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== - -rimraf@^2.6.3: - version "2.7.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" - integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== - dependencies: - glob "^7.1.3" - -rimraf@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" - integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== - dependencies: - glob "^7.1.3" - -rtl-detect@^1.0.3: - version "1.0.4" - resolved "https://registry.yarnpkg.com/rtl-detect/-/rtl-detect-1.0.4.tgz#40ae0ea7302a150b96bc75af7d749607392ecac6" - integrity sha512-EBR4I2VDSSYr7PkBmFy04uhycIpDKp+21p/jARYXlCSjQksTBQcJ0HFUPOO79EPPH5JS6VAhiIQbycf0O3JAxQ== - -rtlcss@^3.1.2: - version "3.5.0" - resolved "https://registry.yarnpkg.com/rtlcss/-/rtlcss-3.5.0.tgz#c9eb91269827a102bac7ae3115dd5d049de636c3" - integrity sha512-wzgMaMFHQTnyi9YOwsx9LjOxYXJPzS8sYnFaKm6R5ysvTkwzHiB0vxnbHwchHQT65PTdBjDG21/kQBWI7q9O7A== - dependencies: - find-up "^5.0.0" - picocolors "^1.0.0" - postcss "^8.3.11" - strip-json-comments "^3.1.1" - -run-parallel@^1.1.9: - version "1.2.0" - resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" - integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA== - dependencies: - queue-microtask "^1.2.2" - -rxjs@^6.6.3: - version "6.6.7" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.7.tgz#90ac018acabf491bf65044235d5863c4dab804c9" - integrity sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ== - dependencies: - tslib "^1.9.0" - -safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: - version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" - integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== - -safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@~5.2.0: - version "5.2.1" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" - integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== - -safe-regex@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/safe-regex/-/safe-regex-1.1.0.tgz#40a3669f3b077d1e943d44629e157dd48023bf2e" - integrity sha1-QKNmnzsHfR6UPURinhV91IAjvy4= - dependencies: - ret "~0.1.10" - -"safer-buffer@>= 2.1.2 < 3": - version "2.1.2" - resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" - integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== - -sax@^1.2.4, sax@~1.2.4: - version "1.2.4" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" - integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== - -scheduler@^0.20.2: - version "0.20.2" - resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.20.2.tgz#4baee39436e34aa93b4874bddcbf0fe8b8b50e91" - integrity sha512-2eWfGgAqqWFGqtdMmcL5zCMK1U8KlXv8SQFGglL3CEtd0aDVDWgeF/YoCmvln55m5zSk3J/20hTaSBeSObsQDQ== - dependencies: - loose-envify "^1.1.0" - object-assign "^4.1.1" - -schema-utils@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-1.0.0.tgz#0b79a93204d7b600d4b2850d1f66c2a34951c770" - integrity sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g== - dependencies: - ajv "^6.1.0" - ajv-errors "^1.0.0" - ajv-keywords "^3.1.0" - -schema-utils@^2.6.5: - version "2.7.1" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-2.7.1.tgz#1ca4f32d1b24c590c203b8e7a50bf0ea4cd394d7" - integrity sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg== - dependencies: - "@types/json-schema" "^7.0.5" - ajv "^6.12.4" - ajv-keywords "^3.5.2" - -schema-utils@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.0.0.tgz#67502f6aa2b66a2d4032b4279a2944978a0913ef" - integrity sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA== - dependencies: - "@types/json-schema" "^7.0.6" - ajv "^6.12.5" - ajv-keywords "^3.5.2" - -schema-utils@^3.1.0, schema-utils@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.1.1.tgz#bc74c4b6b6995c1d88f76a8b77bea7219e0c8281" - integrity sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw== - dependencies: - "@types/json-schema" "^7.0.8" - ajv "^6.12.5" - ajv-keywords "^3.5.2" - -schema-utils@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-4.0.0.tgz#60331e9e3ae78ec5d16353c467c34b3a0a1d3df7" - integrity sha512-1edyXKgh6XnJsJSQ8mKWXnN/BVaIbFMLpouRUrXgVq7WYne5kw3MW7UPhO44uRXQSIpTSXoJbmrR2X0w9kUTyg== - dependencies: - "@types/json-schema" "^7.0.9" - ajv "^8.8.0" - ajv-formats "^2.1.1" - ajv-keywords "^5.0.0" - -section-matter@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/section-matter/-/section-matter-1.0.0.tgz#e9041953506780ec01d59f292a19c7b850b84167" - integrity sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA== - dependencies: - extend-shallow "^2.0.1" - kind-of "^6.0.0" - -select-hose@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" - integrity sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo= - -selfsigned@^1.10.8: - version "1.10.11" - resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-1.10.11.tgz#24929cd906fe0f44b6d01fb23999a739537acbe9" - integrity sha512-aVmbPOfViZqOZPgRBT0+3u4yZFHpmnIghLMlAcb5/xhp5ZtB/RVnKhz5vl2M32CLXAqR4kha9zfhNg0Lf/sxKA== - dependencies: - node-forge "^0.10.0" - -semver-diff@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-3.1.1.tgz#05f77ce59f325e00e2706afd67bb506ddb1ca32b" - integrity sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg== - dependencies: - semver "^6.3.0" - -semver@7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" - integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== - -semver@^5.4.1, semver@^5.5.0, semver@^5.6.0: - version "5.7.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" - integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== - -semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" - integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== - -semver@^7.3.4, semver@^7.3.5: - version "7.3.5" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" - integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ== - dependencies: - lru-cache "^6.0.0" - -send@0.17.1: - version "0.17.1" - resolved "https://registry.yarnpkg.com/send/-/send-0.17.1.tgz#c1d8b059f7900f7466dd4938bdc44e11ddb376c8" - integrity sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg== - dependencies: - debug "2.6.9" - depd "~1.1.2" - destroy "~1.0.4" - encodeurl "~1.0.2" - escape-html "~1.0.3" - etag "~1.8.1" - fresh "0.5.2" - http-errors "~1.7.2" - mime "1.6.0" - ms "2.1.1" - on-finished "~2.3.0" - range-parser "~1.2.1" - statuses "~1.5.0" - -serialize-javascript@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.0.tgz#efae5d88f45d7924141da8b5c3a7a7e663fefeb8" - integrity sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag== - dependencies: - randombytes "^2.1.0" - -serve-handler@^6.1.3: - version "6.1.3" - resolved "https://registry.yarnpkg.com/serve-handler/-/serve-handler-6.1.3.tgz#1bf8c5ae138712af55c758477533b9117f6435e8" - integrity sha512-FosMqFBNrLyeiIDvP1zgO6YoTzFYHxLDEIavhlmQ+knB2Z7l1t+kGLHkZIDN7UVWqQAmKI3D20A6F6jo3nDd4w== - dependencies: - bytes "3.0.0" - content-disposition "0.5.2" - fast-url-parser "1.1.3" - mime-types "2.1.18" - minimatch "3.0.4" - path-is-inside "1.0.2" - path-to-regexp "2.2.1" - range-parser "1.2.0" - -serve-index@^1.9.1: - version "1.9.1" - resolved "https://registry.yarnpkg.com/serve-index/-/serve-index-1.9.1.tgz#d3768d69b1e7d82e5ce050fff5b453bea12a9239" - integrity sha1-03aNabHn2C5c4FD/9bRTvqEqkjk= - dependencies: - accepts "~1.3.4" - batch "0.6.1" - debug "2.6.9" - escape-html "~1.0.3" - http-errors "~1.6.2" - mime-types "~2.1.17" - parseurl "~1.3.2" - -serve-static@1.14.1: - version "1.14.1" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.1.tgz#666e636dc4f010f7ef29970a88a674320898b2f9" - integrity sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg== - dependencies: - encodeurl "~1.0.2" - escape-html "~1.0.3" - parseurl "~1.3.3" - send "0.17.1" - -set-blocking@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" - integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= - -set-value@^2.0.0, set-value@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b" - integrity sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw== - dependencies: - extend-shallow "^2.0.1" - is-extendable "^0.1.1" - is-plain-object "^2.0.3" - split-string "^3.0.1" - -setimmediate@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" - integrity sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU= - -setprototypeof@1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656" - integrity sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ== - -setprototypeof@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" - integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw== - -shallow-clone@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3" - integrity sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA== - dependencies: - kind-of "^6.0.2" - -shebang-command@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" - integrity sha1-RKrGW2lbAzmJaMOfNj/uXer98eo= - dependencies: - shebang-regex "^1.0.0" - -shebang-command@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" - integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== - dependencies: - shebang-regex "^3.0.0" - -shebang-regex@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" - integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM= - -shebang-regex@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" - integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== - -shell-quote@1.7.2: - version "1.7.2" - resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.2.tgz#67a7d02c76c9da24f99d20808fcaded0e0e04be2" - integrity sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg== - -shelljs@^0.8.4: - version "0.8.5" - resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.5.tgz#de055408d8361bed66c669d2f000538ced8ee20c" - integrity sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow== - dependencies: - glob "^7.0.0" - interpret "^1.0.0" - rechoir "^0.6.2" - -signal-exit@^3.0.0: - version "3.0.6" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.6.tgz#24e630c4b0f03fea446a2bd299e62b4a6ca8d0af" - integrity sha512-sDl4qMFpijcGw22U5w63KmD3cZJfBuFlVNbVMKje2keoKML7X2UzWbc4XrmEbDwg0NXJc3yv4/ox7b+JWb57kQ== - -signal-exit@^3.0.2, signal-exit@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" - integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== - -sirv@^1.0.7: - version "1.0.11" - resolved "https://registry.yarnpkg.com/sirv/-/sirv-1.0.11.tgz#81c19a29202048507d6ec0d8ba8910fda52eb5a4" - integrity sha512-SR36i3/LSWja7AJNRBz4fF/Xjpn7lQFI30tZ434dIy+bitLYSP+ZEenHg36i23V2SGEz+kqjksg0uOGZ5LPiqg== - dependencies: - "@polka/url" "^1.0.0-next.9" - mime "^2.3.1" - totalist "^1.0.0" - -sisteransi@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" - integrity sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg== - -sitemap@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-7.0.0.tgz#022bef4df8cba42e38e1fe77039f234cab0372b6" - integrity sha512-Ud0jrRQO2k7fEtPAM+cQkBKoMvxQyPKNXKDLn8tRVHxRCsdDQ2JZvw+aZ5IRYYQVAV9iGxEar6boTwZzev+x3g== - dependencies: - "@types/node" "^15.0.1" - "@types/sax" "^1.2.1" - arg "^5.0.0" - sax "^1.2.4" - -slash@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" - integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== - -snapdragon-node@^2.0.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b" - integrity sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw== - dependencies: - define-property "^1.0.0" - isobject "^3.0.0" - snapdragon-util "^3.0.1" - -snapdragon-util@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/snapdragon-util/-/snapdragon-util-3.0.1.tgz#f956479486f2acd79700693f6f7b805e45ab56e2" - integrity sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ== - dependencies: - kind-of "^3.2.0" - -snapdragon@^0.8.1: - version "0.8.2" - resolved "https://registry.yarnpkg.com/snapdragon/-/snapdragon-0.8.2.tgz#64922e7c565b0e14204ba1aa7d6964278d25182d" - integrity sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg== - dependencies: - base "^0.11.1" - debug "^2.2.0" - define-property "^0.2.5" - extend-shallow "^2.0.1" - map-cache "^0.2.2" - source-map "^0.5.6" - source-map-resolve "^0.5.0" - use "^3.1.0" - -sockjs-client@^1.5.0: - version "1.5.2" - resolved "https://registry.yarnpkg.com/sockjs-client/-/sockjs-client-1.5.2.tgz#4bc48c2da9ce4769f19dc723396b50f5c12330a3" - integrity sha512-ZzRxPBISQE7RpzlH4tKJMQbHM9pabHluk0WBaxAQ+wm/UieeBVBou0p4wVnSQGN9QmpAZygQ0cDIypWuqOFmFQ== - dependencies: - debug "^3.2.6" - eventsource "^1.0.7" - faye-websocket "^0.11.3" - inherits "^2.0.4" - json3 "^3.3.3" - url-parse "^1.5.3" - -sockjs@^0.3.21: - version "0.3.21" - resolved "https://registry.yarnpkg.com/sockjs/-/sockjs-0.3.21.tgz#b34ffb98e796930b60a0cfa11904d6a339a7d417" - integrity sha512-DhbPFGpxjc6Z3I+uX07Id5ZO2XwYsWOrYjaSeieES78cq+JaJvVe5q/m1uvjIQhXinhIeCFRH6JgXe+mvVMyXw== - dependencies: - faye-websocket "^0.11.3" - uuid "^3.4.0" - websocket-driver "^0.7.4" - -sort-css-media-queries@2.0.4: - version "2.0.4" - resolved "https://registry.yarnpkg.com/sort-css-media-queries/-/sort-css-media-queries-2.0.4.tgz#b2badfa519cb4a938acbc6d3aaa913d4949dc908" - integrity sha512-PAIsEK/XupCQwitjv7XxoMvYhT7EAfyzI3hsy/MyDgTvc+Ft55ctdkctJLOy6cQejaIC+zjpUL4djFVm2ivOOw== - -source-list-map@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34" - integrity sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw== - -source-map-js@^0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-0.6.2.tgz#0bb5de631b41cfbda6cfba8bd05a80efdfd2385e" - integrity sha512-/3GptzWzu0+0MBQFrDKzw/DvvMTUORvgY6k6jd/VS6iCR4RDTKWH6v6WPwQoUO8667uQEf9Oe38DxAYWY5F/Ug== - -source-map-js@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.1.tgz#a1741c131e3c77d048252adfa24e23b908670caf" - integrity sha512-4+TN2b3tqOCd/kaGRJ/sTYA0tR0mdXx26ipdolxcwtJVqEnqNYvlCAt1q3ypy4QMlYus+Zh34RNtYLoq2oQ4IA== - -source-map-resolve@^0.5.0: - version "0.5.3" - resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz#190866bece7553e1f8f267a2ee82c606b5509a1a" - integrity sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw== - dependencies: - atob "^2.1.2" - decode-uri-component "^0.2.0" - resolve-url "^0.2.1" - source-map-url "^0.4.0" - urix "^0.1.0" - -source-map-support@~0.5.12: - version "0.5.21" - resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" - integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== - dependencies: - buffer-from "^1.0.0" - source-map "^0.6.0" - -source-map-support@~0.5.20: - version "0.5.20" - resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.20.tgz#12166089f8f5e5e8c56926b377633392dd2cb6c9" - integrity sha512-n1lZZ8Ve4ksRqizaBQgxXDgKwttHDhyfQjA6YZZn8+AroHbsIz+JjwxQDxbp+7y5OYCI8t1Yk7etjD9CRd2hIw== - dependencies: - buffer-from "^1.0.0" - source-map "^0.6.0" - -source-map-url@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.1.tgz#0af66605a745a5a2f91cf1bbf8a7afbc283dec56" - integrity sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw== - -source-map@^0.5.0, source-map@^0.5.6: - version "0.5.7" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" - integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w= - -source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.0, source-map@~0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" - integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== - -source-map@~0.7.2: - version "0.7.3" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.3.tgz#5302f8169031735226544092e64981f751750383" - integrity sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ== - -space-separated-tokens@^1.0.0: - version "1.1.5" - resolved "https://registry.yarnpkg.com/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz#85f32c3d10d9682007e917414ddc5c26d1aa6899" - integrity sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA== - -spdy-transport@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/spdy-transport/-/spdy-transport-3.0.0.tgz#00d4863a6400ad75df93361a1608605e5dcdcf31" - integrity sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw== - dependencies: - debug "^4.1.0" - detect-node "^2.0.4" - hpack.js "^2.1.6" - obuf "^1.1.2" - readable-stream "^3.0.6" - wbuf "^1.7.3" - -spdy@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/spdy/-/spdy-4.0.2.tgz#b74f466203a3eda452c02492b91fb9e84a27677b" - integrity sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA== - dependencies: - debug "^4.1.0" - handle-thing "^2.0.0" - http-deceiver "^1.2.7" - select-hose "^2.0.0" - spdy-transport "^3.0.0" - -split-string@^3.0.1, split-string@^3.0.2: - version "3.1.0" - resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2" - integrity sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw== - dependencies: - extend-shallow "^3.0.0" - -sprintf-js@~1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" - integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= - -stable@^0.1.8: - version "0.1.8" - resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf" - integrity sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w== - -state-toggle@^1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/state-toggle/-/state-toggle-1.0.3.tgz#e123b16a88e143139b09c6852221bc9815917dfe" - integrity sha512-d/5Z4/2iiCnHw6Xzghyhb+GcmF89bxwgXG60wjIiZaxnymbyOmI8Hk4VqHXiVVp6u2ysaskFfXg3ekCj4WNftQ== - -static-extend@^0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/static-extend/-/static-extend-0.1.2.tgz#60809c39cbff55337226fd5e0b520f341f1fb5c6" - integrity sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY= - dependencies: - define-property "^0.2.5" - object-copy "^0.1.0" - -"statuses@>= 1.4.0 < 2", "statuses@>= 1.5.0 < 2", statuses@~1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" - integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= - -std-env@^2.2.1: - version "2.3.0" - resolved "https://registry.yarnpkg.com/std-env/-/std-env-2.3.0.tgz#66d4a4a4d5224242ed8e43f5d65cfa9095216eee" - integrity sha512-4qT5B45+Kjef2Z6pE0BkskzsH0GO7GrND0wGlTM1ioUe3v0dGYx9ZJH0Aro/YyA8fqQ5EyIKDRjZojJYMFTflw== - dependencies: - ci-info "^3.0.0" - -string-width@^3.0.0, string-width@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-3.1.0.tgz#22767be21b62af1081574306f69ac51b62203961" - integrity sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w== - dependencies: - emoji-regex "^7.0.1" - is-fullwidth-code-point "^2.0.0" - strip-ansi "^5.1.0" - -string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0: - version "4.2.2" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5" - integrity sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.0" - -string.prototype.trimend@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz#e75ae90c2942c63504686c18b287b4a0b1a45f80" - integrity sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" - -string.prototype.trimstart@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz#b36399af4ab2999b4c9c648bd7a3fb2bb26feeed" - integrity sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" - -string_decoder@^1.1.1: - version "1.3.0" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" - integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== - dependencies: - safe-buffer "~5.2.0" - -string_decoder@~1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" - integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== - dependencies: - safe-buffer "~5.1.0" - -stringify-object@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/stringify-object/-/stringify-object-3.3.0.tgz#703065aefca19300d3ce88af4f5b3956d7556629" - integrity sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw== - dependencies: - get-own-enumerable-property-symbols "^3.0.0" - is-obj "^1.0.1" - is-regexp "^1.0.0" - -strip-ansi@6.0.0, strip-ansi@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" - integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== - dependencies: - ansi-regex "^5.0.0" - -strip-ansi@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" - integrity sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8= - dependencies: - ansi-regex "^2.0.0" - -strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae" - integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA== - dependencies: - ansi-regex "^4.1.0" - -strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-bom-string@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/strip-bom-string/-/strip-bom-string-1.0.0.tgz#e5211e9224369fbb81d633a2f00044dc8cedad92" - integrity sha1-5SEekiQ2n7uB1jOi8ABE3IztrZI= - -strip-eof@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf" - integrity sha1-u0P/VZim6wXYm1n80SnJgzE2Br8= - -strip-final-newline@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" - integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== - -strip-json-comments@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" - integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== - -strip-json-comments@~2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" - integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo= - -style-to-object@0.3.0, style-to-object@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/style-to-object/-/style-to-object-0.3.0.tgz#b1b790d205991cc783801967214979ee19a76e46" - integrity sha512-CzFnRRXhzWIdItT3OmF8SQfWyahHhjq3HwcMNCNLn+N7klOOqPjMeG/4JSu77D7ypZdGvSzvkrbyeTMizz2VrA== - dependencies: - inline-style-parser "0.1.1" - -stylehacks@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/stylehacks/-/stylehacks-5.0.1.tgz#323ec554198520986806388c7fdaebc38d2c06fb" - integrity sha512-Es0rVnHIqbWzveU1b24kbw92HsebBepxfcqe5iix7t9j0PQqhs0IxXVXv0pY2Bxa08CgMkzD6OWql7kbGOuEdA== - dependencies: - browserslist "^4.16.0" - postcss-selector-parser "^6.0.4" - -supports-color@^5.3.0: - version "5.5.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" - integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== - dependencies: - has-flag "^3.0.0" - -supports-color@^6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3" - integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ== - dependencies: - has-flag "^3.0.0" - -supports-color@^7.1.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" - integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== - dependencies: - has-flag "^4.0.0" - -supports-color@^8.0.0: - version "8.1.1" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" - integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== - dependencies: - has-flag "^4.0.0" - -supports-preserve-symlinks-flag@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" - integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== - -svg-parser@^2.0.2: - version "2.0.4" - resolved "https://registry.yarnpkg.com/svg-parser/-/svg-parser-2.0.4.tgz#fdc2e29e13951736140b76cb122c8ee6630eb6b5" - integrity sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ== - -svgo@^1.2.2: - version "1.3.2" - resolved "https://registry.yarnpkg.com/svgo/-/svgo-1.3.2.tgz#b6dc511c063346c9e415b81e43401145b96d4167" - integrity sha512-yhy/sQYxR5BkC98CY7o31VGsg014AKLEPxdfhora76l36hD9Rdy5NZA/Ocn6yayNPgSamYdtX2rFJdcv07AYVw== - dependencies: - chalk "^2.4.1" - coa "^2.0.2" - css-select "^2.0.0" - css-select-base-adapter "^0.1.1" - css-tree "1.0.0-alpha.37" - csso "^4.0.2" - js-yaml "^3.13.1" - mkdirp "~0.5.1" - object.values "^1.1.0" - sax "~1.2.4" - stable "^0.1.8" - unquote "~1.1.1" - util.promisify "~1.0.0" - -svgo@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/svgo/-/svgo-2.3.0.tgz#6b3af81d0cbd1e19c83f5f63cec2cb98c70b5373" - integrity sha512-fz4IKjNO6HDPgIQxu4IxwtubtbSfGEAJUq/IXyTPIkGhWck/faiiwfkvsB8LnBkKLvSoyNNIY6d13lZprJMc9Q== - dependencies: - "@trysound/sax" "0.1.1" - chalk "^4.1.0" - commander "^7.1.0" - css-select "^3.1.2" - css-tree "^1.1.2" - csso "^4.2.0" - stable "^0.1.8" - -svgo@^2.7.0: - version "2.8.0" - resolved "https://registry.yarnpkg.com/svgo/-/svgo-2.8.0.tgz#4ff80cce6710dc2795f0c7c74101e6764cfccd24" - integrity sha512-+N/Q9kV1+F+UeWYoSiULYo4xYSDQlTgb+ayMobAXPwMnLvop7oxKMo9OzIrX5x3eS4L4f2UHhc9axXwY8DpChg== - dependencies: - "@trysound/sax" "0.2.0" - commander "^7.2.0" - css-select "^4.1.3" - css-tree "^1.1.3" - csso "^4.2.0" - picocolors "^1.0.0" - stable "^0.1.8" - -tapable@^1.0.0: - version "1.1.3" - resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" - integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== - -tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.0.tgz#5c373d281d9c672848213d0e037d1c4165ab426b" - integrity sha512-FBk4IesMV1rBxX2tfiK8RAmogtWn53puLOQlvO8XuwlgxcYbP4mVPS9Ph4aeamSyyVjOl24aYWAuc8U5kCVwMw== - -terser-webpack-plugin@^5.1.3: - version "5.2.4" - resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.2.4.tgz#ad1be7639b1cbe3ea49fab995cbe7224b31747a1" - integrity sha512-E2CkNMN+1cho04YpdANyRrn8CyN4yMy+WdFKZIySFZrGXZxJwJP6PMNGGc/Mcr6qygQHUUqRxnAPmi0M9f00XA== - dependencies: - jest-worker "^27.0.6" - p-limit "^3.1.0" - schema-utils "^3.1.1" - serialize-javascript "^6.0.0" - source-map "^0.6.1" - terser "^5.7.2" - -terser@^4.6.3: - version "4.8.0" - resolved "https://registry.yarnpkg.com/terser/-/terser-4.8.0.tgz#63056343d7c70bb29f3af665865a46fe03a0df17" - integrity sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw== - dependencies: - commander "^2.20.0" - source-map "~0.6.1" - source-map-support "~0.5.12" - -terser@^5.7.2: - version "5.9.0" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.9.0.tgz#47d6e629a522963240f2b55fcaa3c99083d2c351" - integrity sha512-h5hxa23sCdpzcye/7b8YqbE5OwKca/ni0RQz1uRX3tGh8haaGHqcuSqbGRybuAKNdntZ0mDgFNXPJ48xQ2RXKQ== - dependencies: - commander "^2.20.0" - source-map "~0.7.2" - source-map-support "~0.5.20" - -text-table@0.2.0, text-table@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" - integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= - -thunky@^1.0.2: - version "1.1.0" - resolved "https://registry.yarnpkg.com/thunky/-/thunky-1.1.0.tgz#5abaf714a9405db0504732bbccd2cedd9ef9537d" - integrity sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA== - -timsort@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/timsort/-/timsort-0.3.0.tgz#405411a8e7e6339fe64db9a234de11dc31e02bd4" - integrity sha1-QFQRqOfmM5/mTbmiNN4R3DHgK9Q= - -tiny-invariant@^1.0.2: - version "1.1.0" - resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.1.0.tgz#634c5f8efdc27714b7f386c35e6760991d230875" - integrity sha512-ytxQvrb1cPc9WBEI/HSeYYoGD0kWnGEOR8RY6KomWLBVhqz0RgTwVO9dLrGz7dC+nN9llyI7OKAgRq8Vq4ZBSw== - -tiny-warning@^1.0.0, tiny-warning@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/tiny-warning/-/tiny-warning-1.0.3.tgz#94a30db453df4c643d0fd566060d60a875d84754" - integrity sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA== - -to-fast-properties@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" - integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4= - -to-object-path@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/to-object-path/-/to-object-path-0.3.0.tgz#297588b7b0e7e0ac08e04e672f85c1f4999e17af" - integrity sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68= - dependencies: - kind-of "^3.0.2" - -to-readable-stream@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/to-readable-stream/-/to-readable-stream-1.0.0.tgz#ce0aa0c2f3df6adf852efb404a783e77c0475771" - integrity sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q== - -to-regex-range@^2.1.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-2.1.1.tgz#7c80c17b9dfebe599e27367e0d4dd5590141db38" - integrity sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg= - dependencies: - is-number "^3.0.0" - repeat-string "^1.6.1" - -to-regex-range@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" - integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ== - dependencies: - is-number "^7.0.0" - -to-regex@^3.0.1, to-regex@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/to-regex/-/to-regex-3.0.2.tgz#13cfdd9b336552f30b51f33a8ae1b42a7a7599ce" - integrity sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw== - dependencies: - define-property "^2.0.2" - extend-shallow "^3.0.2" - regex-not "^1.0.2" - safe-regex "^1.1.0" - -toidentifier@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" - integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== - -totalist@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/totalist/-/totalist-1.1.0.tgz#a4d65a3e546517701e3e5c37a47a70ac97fe56df" - integrity sha512-gduQwd1rOdDMGxFG1gEvhV88Oirdo2p+KjoYFU7k2g+i7n6AFFbDQ5kMPUsW0pNbfQsB/cwXvT1i4Bue0s9g5g== - -trim-trailing-lines@^1.0.0: - version "1.1.4" - resolved "https://registry.yarnpkg.com/trim-trailing-lines/-/trim-trailing-lines-1.1.4.tgz#bd4abbec7cc880462f10b2c8b5ce1d8d1ec7c2c0" - integrity sha512-rjUWSqnfTNrjbB9NQWfPMH/xRK1deHeGsHoVfpxJ++XeYXE0d6B1En37AHfw3jtfTU7dzMzZL2jjpe8Qb5gLIQ== - -trim@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/trim/-/trim-0.0.1.tgz#5858547f6b290757ee95cccc666fb50084c460dd" - integrity sha1-WFhUf2spB1fulczMZm+1AITEYN0= - -trough@^1.0.0: - version "1.0.5" - resolved "https://registry.yarnpkg.com/trough/-/trough-1.0.5.tgz#b8b639cefad7d0bb2abd37d433ff8293efa5f406" - integrity sha512-rvuRbTarPXmMb79SmzEp8aqXNKcK+y0XaB298IXueQ8I2PsrATcPBCSPyK/dDNa2iWOhKlfNnOjdAOTBU/nkFA== - -ts-essentials@^2.0.3: - version "2.0.12" - resolved "https://registry.yarnpkg.com/ts-essentials/-/ts-essentials-2.0.12.tgz#c9303f3d74f75fa7528c3d49b80e089ab09d8745" - integrity sha512-3IVX4nI6B5cc31/GFFE+i8ey/N2eA0CZDbo6n0yrz0zDX8ZJ8djmU1p+XRz7G3is0F3bB3pu2pAroFdAWQKU3w== - -tslib@^1.9.0: - version "1.14.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" - integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== - -tslib@^2.0.3, tslib@^2.1.0, tslib@^2.2.0: - version "2.3.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01" - integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw== - -type-fest@^0.20.2: - version "0.20.2" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" - integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== - -type-fest@^0.21.3: - version "0.21.3" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" - integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== - -type-is@~1.6.17, type-is@~1.6.18: - version "1.6.18" - resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" - integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== - dependencies: - media-typer "0.3.0" - mime-types "~2.1.24" - -typedarray-to-buffer@^3.1.5: - version "3.1.5" - resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080" - integrity sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q== - dependencies: - is-typedarray "^1.0.0" - -ua-parser-js@^0.7.18: - version "0.7.28" - resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.28.tgz#8ba04e653f35ce210239c64661685bf9121dec31" - integrity sha512-6Gurc1n//gjp9eQNXjD9O3M/sMwVtN5S8Lv9bvOYBfKfDNiIIhqiyi01vMBO45u4zkDE420w/e0se7Vs+sIg+g== - -unbox-primitive@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471" - integrity sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw== - dependencies: - function-bind "^1.1.1" - has-bigints "^1.0.1" - has-symbols "^1.0.2" - which-boxed-primitive "^1.0.2" - -unherit@^1.0.4: - version "1.1.3" - resolved "https://registry.yarnpkg.com/unherit/-/unherit-1.1.3.tgz#6c9b503f2b41b262330c80e91c8614abdaa69c22" - integrity sha512-Ft16BJcnapDKp0+J/rqFC3Rrk6Y/Ng4nzsC028k2jdDII/rdZ7Wd3pPT/6+vIIxRagwRc9K0IUX0Ra4fKvw+WQ== - dependencies: - inherits "^2.0.0" - xtend "^4.0.0" - -unicode-canonical-property-names-ecmascript@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" - integrity sha512-jDrNnXWHd4oHiTZnx/ZG7gtUTVp+gCcTTKr8L0HjlwphROEW3+Him+IpvC+xcJEFegapiMZyZe02CyuOnRmbnQ== - -unicode-match-property-ecmascript@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-1.0.4.tgz#8ed2a32569961bce9227d09cd3ffbb8fed5f020c" - integrity sha512-L4Qoh15vTfntsn4P1zqnHulG0LdXgjSO035fEpdtp6YxXhMT51Q6vgM5lYdG/5X3MjS+k/Y9Xw4SFCY9IkR0rg== - dependencies: - unicode-canonical-property-names-ecmascript "^1.0.4" - unicode-property-aliases-ecmascript "^1.0.4" - -unicode-match-property-value-ecmascript@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-1.2.0.tgz#0d91f600eeeb3096aa962b1d6fc88876e64ea531" - integrity sha512-wjuQHGQVofmSJv1uVISKLE5zO2rNGzM/KCYZch/QQvez7C1hUhBIuZ701fYXExuufJFMPhv2SyL8CyoIfMLbIQ== - -unicode-property-aliases-ecmascript@^1.0.4: - version "1.1.0" - resolved "https://registry.yarnpkg.com/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.1.0.tgz#dd57a99f6207bedff4628abefb94c50db941c8f4" - integrity sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg== - -unified@9.2.0: - version "9.2.0" - resolved "https://registry.yarnpkg.com/unified/-/unified-9.2.0.tgz#67a62c627c40589edebbf60f53edfd4d822027f8" - integrity sha512-vx2Z0vY+a3YoTj8+pttM3tiJHCwY5UFbYdiWrwBEbHmK8pvsPj2rtAX2BFfgXen8T39CJWblWRDT4L5WGXtDdg== - dependencies: - bail "^1.0.0" - extend "^3.0.0" - is-buffer "^2.0.0" - is-plain-obj "^2.0.0" - trough "^1.0.0" - vfile "^4.0.0" - -unified@^8.4.2: - version "8.4.2" - resolved "https://registry.yarnpkg.com/unified/-/unified-8.4.2.tgz#13ad58b4a437faa2751a4a4c6a16f680c500fff1" - integrity sha512-JCrmN13jI4+h9UAyKEoGcDZV+i1E7BLFuG7OsaDvTXI5P0qhHX+vZO/kOhz9jn8HGENDKbwSeB0nVOg4gVStGA== - dependencies: - bail "^1.0.0" - extend "^3.0.0" - is-plain-obj "^2.0.0" - trough "^1.0.0" - vfile "^4.0.0" - -union-value@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.1.tgz#0b6fe7b835aecda61c6ea4d4f02c14221e109847" - integrity sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg== - dependencies: - arr-union "^3.1.0" - get-value "^2.0.6" - is-extendable "^0.1.1" - set-value "^2.0.1" - -uniqs@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/uniqs/-/uniqs-2.0.0.tgz#ffede4b36b25290696e6e165d4a59edb998e6b02" - integrity sha1-/+3ks2slKQaW5uFl1KWe25mOawI= - -unique-string@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/unique-string/-/unique-string-2.0.0.tgz#39c6451f81afb2749de2b233e3f7c5e8843bd89d" - integrity sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg== - dependencies: - crypto-random-string "^2.0.0" - -unist-builder@2.0.3, unist-builder@^2.0.0: - version "2.0.3" - resolved "https://registry.yarnpkg.com/unist-builder/-/unist-builder-2.0.3.tgz#77648711b5d86af0942f334397a33c5e91516436" - integrity sha512-f98yt5pnlMWlzP539tPc4grGMsFaQQlP/vM396b00jngsiINumNmsY8rkXjfoi1c6QaM8nQ3vaGDuoKWbe/1Uw== - -unist-util-generated@^1.0.0: - version "1.1.6" - resolved "https://registry.yarnpkg.com/unist-util-generated/-/unist-util-generated-1.1.6.tgz#5ab51f689e2992a472beb1b35f2ce7ff2f324d4b" - integrity sha512-cln2Mm1/CZzN5ttGK7vkoGw+RZ8VcUH6BtGbq98DDtRGquAAOXig1mrBQYelOwMXYS8rK+vZDyyojSjp7JX+Lg== - -unist-util-is@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-4.1.0.tgz#976e5f462a7a5de73d94b706bac1b90671b57797" - integrity sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg== - -unist-util-position@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/unist-util-position/-/unist-util-position-3.1.0.tgz#1c42ee6301f8d52f47d14f62bbdb796571fa2d47" - integrity sha512-w+PkwCbYSFw8vpgWD0v7zRCl1FpY3fjDSQ3/N/wNd9Ffa4gPi8+4keqt99N3XW6F99t/mUzp2xAhNmfKWp95QA== - -unist-util-remove-position@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unist-util-remove-position/-/unist-util-remove-position-2.0.1.tgz#5d19ca79fdba712301999b2b73553ca8f3b352cc" - integrity sha512-fDZsLYIe2uT+oGFnuZmy73K6ZxOPG/Qcm+w7jbEjaFcJgbQ6cqjs/eSPzXhsmGpAsWPkqZM9pYjww5QTn3LHMA== - dependencies: - unist-util-visit "^2.0.0" - -unist-util-remove@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/unist-util-remove/-/unist-util-remove-2.1.0.tgz#b0b4738aa7ee445c402fda9328d604a02d010588" - integrity sha512-J8NYPyBm4baYLdCbjmf1bhPu45Cr1MWTm77qd9istEkzWpnN6O9tMsEbB2JhNnBCqGENRqEWomQ+He6au0B27Q== - dependencies: - unist-util-is "^4.0.0" - -unist-util-stringify-position@^2.0.0: - version "2.0.3" - resolved "https://registry.yarnpkg.com/unist-util-stringify-position/-/unist-util-stringify-position-2.0.3.tgz#cce3bfa1cdf85ba7375d1d5b17bdc4cada9bd9da" - integrity sha512-3faScn5I+hy9VleOq/qNbAd6pAx7iH5jYBMS9I1HgQVijz/4mv5Bvw5iw1sC/90CODiKo81G/ps8AJrISn687g== - dependencies: - "@types/unist" "^2.0.2" - -unist-util-visit-parents@^3.0.0: - version "3.1.1" - resolved "https://registry.yarnpkg.com/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz#65a6ce698f78a6b0f56aa0e88f13801886cdaef6" - integrity sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg== - dependencies: - "@types/unist" "^2.0.0" - unist-util-is "^4.0.0" - -unist-util-visit@2.0.3, unist-util-visit@^2.0.0, unist-util-visit@^2.0.1, unist-util-visit@^2.0.2, unist-util-visit@^2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-2.0.3.tgz#c3703893146df47203bb8a9795af47d7b971208c" - integrity sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q== - dependencies: - "@types/unist" "^2.0.0" - unist-util-is "^4.0.0" - unist-util-visit-parents "^3.0.0" - -universalify@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" - integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ== - -unpipe@1.0.0, unpipe@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" - integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= - -unquote@~1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/unquote/-/unquote-1.1.1.tgz#8fded7324ec6e88a0ff8b905e7c098cdc086d544" - integrity sha1-j97XMk7G6IoP+LkF58CYzcCG1UQ= - -unset-value@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-1.0.0.tgz#8376873f7d2335179ffb1e6fc3a8ed0dfc8ab559" - integrity sha1-g3aHP30jNRef+x5vw6jtDfyKtVk= - dependencies: - has-value "^0.3.1" - isobject "^3.0.0" - -upath@^1.1.1: - version "1.2.0" - resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894" - integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg== - -update-notifier@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-5.1.0.tgz#4ab0d7c7f36a231dd7316cf7729313f0214d9ad9" - integrity sha512-ItnICHbeMh9GqUy31hFPrD1kcuZ3rpxDZbf4KUDavXwS0bW5m7SLbDQpGX3UYr072cbrF5hFUs3r5tUsPwjfHw== - dependencies: - boxen "^5.0.0" - chalk "^4.1.0" - configstore "^5.0.1" - has-yarn "^2.1.0" - import-lazy "^2.1.0" - is-ci "^2.0.0" - is-installed-globally "^0.4.0" - is-npm "^5.0.0" - is-yarn-global "^0.3.0" - latest-version "^5.1.0" - pupa "^2.1.1" - semver "^7.3.4" - semver-diff "^3.1.1" - xdg-basedir "^4.0.0" - -uri-js@^4.2.2: - version "4.4.1" - resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" - integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== - dependencies: - punycode "^2.1.0" - -urix@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" - integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI= - -url-loader@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-4.1.1.tgz#28505e905cae158cf07c92ca622d7f237e70a4e2" - integrity sha512-3BTV812+AVHHOJQO8O5MkWgZ5aosP7GnROJwvzLS9hWDj00lZ6Z0wNak423Lp9PBZN05N+Jk/N5Si8jRAlGyWA== - dependencies: - loader-utils "^2.0.0" - mime-types "^2.1.27" - schema-utils "^3.0.0" - -url-parse-lax@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-3.0.0.tgz#16b5cafc07dbe3676c1b1999177823d6503acb0c" - integrity sha1-FrXK/Afb42dsGxmZF3gj1lA6yww= - dependencies: - prepend-http "^2.0.0" - -url-parse@^1.4.3, url-parse@^1.5.3: - version "1.5.3" - resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.3.tgz#71c1303d38fb6639ade183c2992c8cc0686df862" - integrity sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ== - dependencies: - querystringify "^2.1.1" - requires-port "^1.0.0" - -url@^0.11.0: - version "0.11.0" - resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" - integrity sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE= - dependencies: - punycode "1.3.2" - querystring "0.2.0" - -use-composed-ref@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/use-composed-ref/-/use-composed-ref-1.1.0.tgz#9220e4e94a97b7b02d7d27eaeab0b37034438bbc" - integrity sha512-my1lNHGWsSDAhhVAT4MKs6IjBUtG6ZG11uUqexPH9PptiIZDQOzaF4f5tEbJ2+7qvNbtXNBbU3SfmN+fXlWDhg== - dependencies: - ts-essentials "^2.0.3" - -use-isomorphic-layout-effect@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.1.1.tgz#7bb6589170cd2987a152042f9084f9effb75c225" - integrity sha512-L7Evj8FGcwo/wpbv/qvSfrkHFtOpCzvM5yl2KVyDJoylVuSvzphiiasmjgQPttIGBAy2WKiBNR98q8w7PiNgKQ== - -use-latest@^1.0.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/use-latest/-/use-latest-1.2.0.tgz#a44f6572b8288e0972ec411bdd0840ada366f232" - integrity sha512-d2TEuG6nSLKQLAfW3By8mKr8HurOlTkul0sOpxbClIv4SQ4iOd7BYr7VIzdbktUCnv7dua/60xzd8igMU6jmyw== - dependencies: - use-isomorphic-layout-effect "^1.0.0" - -use@^3.1.0: - version "3.1.1" - resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" - integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ== - -util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" - integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= - -util.promisify@~1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/util.promisify/-/util.promisify-1.0.1.tgz#6baf7774b80eeb0f7520d8b81d07982a59abbaee" - integrity sha512-g9JpC/3He3bm38zsLupWryXHoEcS22YHthuPQSJdMy6KNrzIRzWqcsHzD/WUnqe45whVou4VIsPew37DoXWNrA== - dependencies: - define-properties "^1.1.3" - es-abstract "^1.17.2" - has-symbols "^1.0.1" - object.getownpropertydescriptors "^2.1.0" - -utila@~0.4: - version "0.4.0" - resolved "https://registry.yarnpkg.com/utila/-/utila-0.4.0.tgz#8a16a05d445657a3aea5eecc5b12a4fa5379772c" - integrity sha1-ihagXURWV6Oupe7MWxKk+lN5dyw= - -utility-types@^3.10.0: - version "3.10.0" - resolved "https://registry.yarnpkg.com/utility-types/-/utility-types-3.10.0.tgz#ea4148f9a741015f05ed74fd615e1d20e6bed82b" - integrity sha512-O11mqxmi7wMKCo6HKFt5AhO4BwY3VV68YU07tgxfz8zJTIxr4BpsezN49Ffwy9j3ZpwwJp4fkRwjRzq3uWE6Rg== - -utils-merge@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" - integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= - -uuid@^3.3.2, uuid@^3.4.0: - version "3.4.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" - integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== - -value-equal@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/value-equal/-/value-equal-1.0.1.tgz#1e0b794c734c5c0cade179c437d356d931a34d6c" - integrity sha512-NOJ6JZCAWr0zlxZt+xqCHNTEKOsrks2HQd4MqhP1qy4z1SkbEP467eNx6TgDKXMvUOb+OENfJCZwM+16n7fRfw== - -vary@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" - integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= - -vendors@^1.0.3: - version "1.0.4" - resolved "https://registry.yarnpkg.com/vendors/-/vendors-1.0.4.tgz#e2b800a53e7a29b93506c3cf41100d16c4c4ad8e" - integrity sha512-/juG65kTL4Cy2su4P8HjtkTxk6VmJDiOPBufWniqQ6wknac6jNiXS9vU+hO3wgusiyqWlzTbVHi0dyJqRONg3w== - -vfile-location@^3.0.0, vfile-location@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/vfile-location/-/vfile-location-3.2.0.tgz#d8e41fbcbd406063669ebf6c33d56ae8721d0f3c" - integrity sha512-aLEIZKv/oxuCDZ8lkJGhuhztf/BW4M+iHdCwglA/eWc+vtuRFJj8EtgceYFX4LRjOhCAAiNHsKGssC6onJ+jbA== - -vfile-message@^2.0.0: - version "2.0.4" - resolved "https://registry.yarnpkg.com/vfile-message/-/vfile-message-2.0.4.tgz#5b43b88171d409eae58477d13f23dd41d52c371a" - integrity sha512-DjssxRGkMvifUOJre00juHoP9DPWuzjxKuMDrhNbk2TdaYYBNMStsNhEOt3idrtI12VQYM/1+iM0KOzXi4pxwQ== - dependencies: - "@types/unist" "^2.0.0" - unist-util-stringify-position "^2.0.0" - -vfile@^4.0.0: - version "4.2.1" - resolved "https://registry.yarnpkg.com/vfile/-/vfile-4.2.1.tgz#03f1dce28fc625c625bc6514350fbdb00fa9e624" - integrity sha512-O6AE4OskCG5S1emQ/4gl8zK586RqA3srz3nfK/Viy0UPToBc5Trp9BVFb1u0CjsKrAWwnpr4ifM/KBXPWwJbCA== - dependencies: - "@types/unist" "^2.0.0" - is-buffer "^2.0.0" - unist-util-stringify-position "^2.0.0" - vfile-message "^2.0.0" - -wait-on@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/wait-on/-/wait-on-5.3.0.tgz#584e17d4b3fe7b46ac2b9f8e5e102c005c2776c7" - integrity sha512-DwrHrnTK+/0QFaB9a8Ol5Lna3k7WvUR4jzSKmz0YaPBpuN2sACyiPVKVfj6ejnjcajAcvn3wlbTyMIn9AZouOg== - dependencies: - axios "^0.21.1" - joi "^17.3.0" - lodash "^4.17.21" - minimist "^1.2.5" - rxjs "^6.6.3" - -watchpack@^2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.3.1.tgz#4200d9447b401156eeca7767ee610f8809bc9d25" - integrity sha512-x0t0JuydIo8qCNctdDrn1OzH/qDzk2+rdCOC3YzumZ42fiMqmQ7T3xQurykYMhYfHaPHTp4ZxAx2NfUo1K6QaA== - dependencies: - glob-to-regexp "^0.4.1" - graceful-fs "^4.1.2" - -wbuf@^1.1.0, wbuf@^1.7.3: - version "1.7.3" - resolved "https://registry.yarnpkg.com/wbuf/-/wbuf-1.7.3.tgz#c1d8d149316d3ea852848895cb6a0bfe887b87df" - integrity sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA== - dependencies: - minimalistic-assert "^1.0.0" - -web-namespaces@^1.0.0, web-namespaces@^1.1.2: - version "1.1.4" - resolved "https://registry.yarnpkg.com/web-namespaces/-/web-namespaces-1.1.4.tgz#bc98a3de60dadd7faefc403d1076d529f5e030ec" - integrity sha512-wYxSGajtmoP4WxfejAPIr4l0fVh+jeMXZb08wNc0tMg6xsfZXj3cECqIK0G7ZAqUq0PP8WlMDtaOGVBTAWztNw== - -webpack-bundle-analyzer@^4.4.2: - version "4.4.2" - resolved "https://registry.yarnpkg.com/webpack-bundle-analyzer/-/webpack-bundle-analyzer-4.4.2.tgz#39898cf6200178240910d629705f0f3493f7d666" - integrity sha512-PIagMYhlEzFfhMYOzs5gFT55DkUdkyrJi/SxJp8EF3YMWhS+T9vvs2EoTetpk5qb6VsCq02eXTlRDOydRhDFAQ== - dependencies: - acorn "^8.0.4" - acorn-walk "^8.0.0" - chalk "^4.1.0" - commander "^6.2.0" - gzip-size "^6.0.0" - lodash "^4.17.20" - opener "^1.5.2" - sirv "^1.0.7" - ws "^7.3.1" - -webpack-dev-middleware@^3.7.2: - version "3.7.3" - resolved "https://registry.yarnpkg.com/webpack-dev-middleware/-/webpack-dev-middleware-3.7.3.tgz#0639372b143262e2b84ab95d3b91a7597061c2c5" - integrity sha512-djelc/zGiz9nZj/U7PTBi2ViorGJXEWo/3ltkPbDyxCXhhEXkW0ce99falaok4TPj+AsxLiXJR0EBOb0zh9fKQ== - dependencies: - memory-fs "^0.4.1" - mime "^2.4.4" - mkdirp "^0.5.1" - range-parser "^1.2.1" - webpack-log "^2.0.0" - -webpack-dev-server@^3.11.2: - version "3.11.3" - resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-3.11.3.tgz#8c86b9d2812bf135d3c9bce6f07b718e30f7c3d3" - integrity sha512-3x31rjbEQWKMNzacUZRE6wXvUFuGpH7vr0lIEbYpMAG9BOxi0928QU1BBswOAP3kg3H1O4hiS+sq4YyAn6ANnA== - dependencies: - ansi-html-community "0.0.8" - bonjour "^3.5.0" - chokidar "^2.1.8" - compression "^1.7.4" - connect-history-api-fallback "^1.6.0" - debug "^4.1.1" - del "^4.1.1" - express "^4.17.1" - html-entities "^1.3.1" - http-proxy-middleware "0.19.1" - import-local "^2.0.0" - internal-ip "^4.3.0" - ip "^1.1.5" - is-absolute-url "^3.0.3" - killable "^1.0.1" - loglevel "^1.6.8" - opn "^5.5.0" - p-retry "^3.0.1" - portfinder "^1.0.26" - schema-utils "^1.0.0" - selfsigned "^1.10.8" - semver "^6.3.0" - serve-index "^1.9.1" - sockjs "^0.3.21" - sockjs-client "^1.5.0" - spdy "^4.0.2" - strip-ansi "^3.0.1" - supports-color "^6.1.0" - url "^0.11.0" - webpack-dev-middleware "^3.7.2" - webpack-log "^2.0.0" - ws "^6.2.1" - yargs "^13.3.2" - -webpack-log@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/webpack-log/-/webpack-log-2.0.0.tgz#5b7928e0637593f119d32f6227c1e0ac31e1b47f" - integrity sha512-cX8G2vR/85UYG59FgkoMamwHUIkSSlV3bBMRsbxVXVUk2j6NleCKjQ/WE9eYg9WY4w25O9w8wKP4rzNZFmUcUg== - dependencies: - ansi-colors "^3.0.0" - uuid "^3.3.2" - -webpack-merge@^5.8.0: - version "5.8.0" - resolved "https://registry.yarnpkg.com/webpack-merge/-/webpack-merge-5.8.0.tgz#2b39dbf22af87776ad744c390223731d30a68f61" - integrity sha512-/SaI7xY0831XwP6kzuwhKWVKDP9t1QY1h65lAFLbZqMPIuYcD9QAW4u9STIbU9kaJbPBB/geU/gLr1wDjOhQ+Q== - dependencies: - clone-deep "^4.0.1" - wildcard "^2.0.0" - -webpack-sources@^1.1.0, webpack-sources@^1.4.3: - version "1.4.3" - resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-1.4.3.tgz#eedd8ec0b928fbf1cbfe994e22d2d890f330a933" - integrity sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ== - dependencies: - source-list-map "^2.0.0" - source-map "~0.6.1" - -webpack-sources@^3.2.2: - version "3.2.2" - resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-3.2.2.tgz#d88e3741833efec57c4c789b6010db9977545260" - integrity sha512-cp5qdmHnu5T8wRg2G3vZZHoJPN14aqQ89SyQ11NpGH5zEMDCclt49rzo+MaRazk7/UeILhAI+/sEtcM+7Fr0nw== - -webpack@^5.40.0: - version "5.65.0" - resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.65.0.tgz#ed2891d9145ba1f0d318e4ea4f89c3fa18e6f9be" - integrity sha512-Q5or2o6EKs7+oKmJo7LaqZaMOlDWQse9Tm5l1WAfU/ujLGN5Pb0SqGeVkN/4bpPmEqEP5RnVhiqsOtWtUVwGRw== - dependencies: - "@types/eslint-scope" "^3.7.0" - "@types/estree" "^0.0.50" - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/wasm-edit" "1.11.1" - "@webassemblyjs/wasm-parser" "1.11.1" - acorn "^8.4.1" - acorn-import-assertions "^1.7.6" - browserslist "^4.14.5" - chrome-trace-event "^1.0.2" - enhanced-resolve "^5.8.3" - es-module-lexer "^0.9.0" - eslint-scope "5.1.1" - events "^3.2.0" - glob-to-regexp "^0.4.1" - graceful-fs "^4.2.4" - json-parse-better-errors "^1.0.2" - loader-runner "^4.2.0" - mime-types "^2.1.27" - neo-async "^2.6.2" - schema-utils "^3.1.0" - tapable "^2.1.1" - terser-webpack-plugin "^5.1.3" - watchpack "^2.3.1" - webpack-sources "^3.2.2" - -webpackbar@^5.0.0-3: - version "5.0.0-3" - resolved "https://registry.yarnpkg.com/webpackbar/-/webpackbar-5.0.0-3.tgz#f4f96c8fb13001b2bb1348252db4c980ab93aaac" - integrity sha512-viW6KCYjMb0NPoDrw2jAmLXU2dEOhRrtku28KmOfeE1vxbfwCYuTbTaMhnkrCZLFAFyY9Q49Z/jzYO80Dw5b8g== - dependencies: - ansi-escapes "^4.3.1" - chalk "^4.1.0" - consola "^2.15.0" - figures "^3.2.0" - pretty-time "^1.1.0" - std-env "^2.2.1" - text-table "^0.2.0" - wrap-ansi "^7.0.0" - -websocket-driver@>=0.5.1, websocket-driver@^0.7.4: - version "0.7.4" - resolved "https://registry.yarnpkg.com/websocket-driver/-/websocket-driver-0.7.4.tgz#89ad5295bbf64b480abcba31e4953aca706f5760" - integrity sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg== - dependencies: - http-parser-js ">=0.5.1" - safe-buffer ">=5.1.0" - websocket-extensions ">=0.1.1" - -websocket-extensions@>=0.1.1: - version "0.1.4" - resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42" - integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg== - -which-boxed-primitive@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" - integrity sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg== - dependencies: - is-bigint "^1.0.1" - is-boolean-object "^1.1.0" - is-number-object "^1.0.4" - is-string "^1.0.5" - is-symbol "^1.0.3" - -which-module@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" - integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho= - -which@^1.2.9, which@^1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" - integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== - dependencies: - isexe "^2.0.0" - -which@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" - integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== - dependencies: - isexe "^2.0.0" - -widest-line@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/widest-line/-/widest-line-3.1.0.tgz#8292333bbf66cb45ff0de1603b136b7ae1496eca" - integrity sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg== - dependencies: - string-width "^4.0.0" - -wildcard@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/wildcard/-/wildcard-2.0.0.tgz#a77d20e5200c6faaac979e4b3aadc7b3dd7f8fec" - integrity sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw== - -worker-rpc@^0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/worker-rpc/-/worker-rpc-0.1.1.tgz#cb565bd6d7071a8f16660686051e969ad32f54d5" - integrity sha512-P1WjMrUB3qgJNI9jfmpZ/htmBEjFh//6l/5y8SD9hg1Ef5zTTVVoRjTrTEzPrNBQvmhMxkoTsjOXN10GWU7aCg== - dependencies: - microevent.ts "~0.1.1" - -wrap-ansi@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-5.1.0.tgz#1fd1f67235d5b6d0fee781056001bfb694c03b09" - integrity sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q== - dependencies: - ansi-styles "^3.2.0" - string-width "^3.0.0" - strip-ansi "^5.0.0" - -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - -wrappy@1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" - integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= - -write-file-atomic@^3.0.0: - version "3.0.3" - resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8" - integrity sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q== - dependencies: - imurmurhash "^0.1.4" - is-typedarray "^1.0.0" - signal-exit "^3.0.2" - typedarray-to-buffer "^3.1.5" - -ws@^6.2.1: - version "6.2.2" - resolved "https://registry.yarnpkg.com/ws/-/ws-6.2.2.tgz#dd5cdbd57a9979916097652d78f1cc5faea0c32e" - integrity sha512-zmhltoSR8u1cnDsD43TX59mzoMZsLKqUweyYBAIvTngR3shc0W6aOZylZmq/7hqyVxPdi+5Ud2QInblgyE72fw== - dependencies: - async-limiter "~1.0.0" - -ws@^7.3.1: - version "7.4.5" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.5.tgz#a484dd851e9beb6fdb420027e3885e8ce48986c1" - integrity sha512-xzyu3hFvomRfXKH8vOFMU3OguG6oOvhXMo3xsGy3xWExqaM2dxBbVxuD99O7m3ZUFMvvscsZDqxfgMaRr/Nr1g== - -xdg-basedir@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13" - integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q== - -xml-js@^1.6.11: - version "1.6.11" - resolved "https://registry.yarnpkg.com/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9" - integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g== - dependencies: - sax "^1.2.4" - -xtend@^4.0.0, xtend@^4.0.1: - version "4.0.2" - resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" - integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== - -y18n@^4.0.0: - version "4.0.3" - resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.3.tgz#b5f259c82cd6e336921efd7bfd8bf560de9eeedf" - integrity sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ== - -yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - -yaml@^1.10.0, yaml@^1.10.2: - version "1.10.2" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" - integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== - -yargs-parser@^13.1.2: - version "13.1.2" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-13.1.2.tgz#130f09702ebaeef2650d54ce6e3e5706f7a4fb38" - integrity sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg== - dependencies: - camelcase "^5.0.0" - decamelize "^1.2.0" - -yargs@^13.3.2: - version "13.3.2" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-13.3.2.tgz#ad7ffefec1aa59565ac915f82dccb38a9c31a2dd" - integrity sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw== - dependencies: - cliui "^5.0.0" - find-up "^3.0.0" - get-caller-file "^2.0.1" - require-directory "^2.1.1" - require-main-filename "^2.0.0" - set-blocking "^2.0.0" - string-width "^3.0.0" - which-module "^2.0.0" - y18n "^4.0.0" - yargs-parser "^13.1.2" - -yocto-queue@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" - integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== - -zwitch@^1.0.0: - version "1.0.5" - resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-1.0.5.tgz#d11d7381ffed16b742f6af7b3f223d5cd9fe9920" - integrity sha512-V50KMwwzqJV0NpZIZFwfOD5/lyny3WlSzRiXgA0G7VUnRlqttta1L6UQIHzd6EuBY/cHGfwTIck7w1yH6Q5zUw== diff --git a/OpenRefine/extensions/database/.eclipse-pmd b/OpenRefine/extensions/database/.eclipse-pmd deleted file mode 100644 index 4a705f215..000000000 --- a/OpenRefine/extensions/database/.eclipse-pmd +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/OpenRefine/extensions/database/.eslintrc.json b/OpenRefine/extensions/database/.eslintrc.json deleted file mode 100644 index d15f78754..000000000 --- a/OpenRefine/extensions/database/.eslintrc.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "env": { - "browser": true, - "es6": true - }, - "extends": "eslint:recommended", - "parserOptions": { - "sourceType": "module" - }, - "rules": { - "indent": [ - "error", - 4 - ], - "linebreak-style": [ - "error", - "unix" - ], - "quotes": [ - "error", - "double" - ], - "semi": [ - "error", - "always" - ] - } -} \ No newline at end of file diff --git a/OpenRefine/extensions/database/.travis.yml b/OpenRefine/extensions/database/.travis.yml deleted file mode 100644 index 800ed8b08..000000000 --- a/OpenRefine/extensions/database/.travis.yml +++ /dev/null @@ -1,24 +0,0 @@ -language: java - -jdk: - - oraclejdk8 - -addons: - mariadb: '10.0' - -services: - - mysql - - postgresql - -before_install: - - - mysql -u root -e 'CREATE DATABASE test_db;' - - mysql -u root test_db < test/conf/travis-mysql.sql - - - psql -c 'CREATE DATABASE test_db;' -U postgres - - psql -U postgres test_db < test/conf/travis-pgsql.sql - - -script: - - ant test - diff --git a/OpenRefine/extensions/database/README.md b/OpenRefine/extensions/database/README.md deleted file mode 100644 index 0dd07734c..000000000 --- a/OpenRefine/extensions/database/README.md +++ /dev/null @@ -1,9 +0,0 @@ -This project is an OpenRefine extension for importing database data using JDBC. -For exporting to a database, other code can be found under folder `OpenRefine/main/src/com/google/refine/exporters/sql` - -## Adding support for other database vendors - -1. You'll want to register an additional Database Service: https://github.com/OpenRefine/OpenRefine/blob/master/extensions/database/src/com/google/refine/extension/database/DatabaseService.java -2. Provide the connection and service classes, look at the PostgreSQL one or MySQL one as examples: https://github.com/OpenRefine/OpenRefine/tree/master/extensions/database/src/com/google/refine/extension/database -3. Then wire up the interface with defaults as desired: https://github.com/OpenRefine/OpenRefine/blob/master/extensions/database/module/scripts/index/database-source-ui.js#L93 -4. Add drivers manually to the classpath, or update the pom file to provide them as dependencies as other DB libraries are done: https://github.com/OpenRefine/OpenRefine/blob/master/extensions/database/pom.xml diff --git a/OpenRefine/extensions/database/licenses/jdbc-client.LICENSE.txt b/OpenRefine/extensions/database/licenses/jdbc-client.LICENSE.txt deleted file mode 100644 index a14b06b91..000000000 --- a/OpenRefine/extensions/database/licenses/jdbc-client.LICENSE.txt +++ /dev/null @@ -1,188 +0,0 @@ -Copyright 2006 Google - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. \ No newline at end of file diff --git a/OpenRefine/extensions/database/module/MOD-INF/.gitignore b/OpenRefine/extensions/database/module/MOD-INF/.gitignore deleted file mode 100644 index 840e7d312..000000000 --- a/OpenRefine/extensions/database/module/MOD-INF/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/classes/ diff --git a/OpenRefine/extensions/database/module/MOD-INF/controller.js b/OpenRefine/extensions/database/module/MOD-INF/controller.js deleted file mode 100644 index 959745b5a..000000000 --- a/OpenRefine/extensions/database/module/MOD-INF/controller.js +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * Controller for JDBC Database extension. - * - * This is run in the Butterfly (ie Refine) server context using the Rhino - * Javascript interpreter. - */ - -var html = "text/html"; -var encoding = "UTF-8"; -var version = "0.1"; -var ClientSideResourceManager = Packages.com.google.refine.ClientSideResourceManager; - -var logger = Packages.org.slf4j.LoggerFactory.getLogger("database-extension"), -File = Packages.java.io.File, -refineServlet = Packages.com.google.refine.RefineServlet; - -/* - * Register our custom commands. - */ -function registerCommands() { - - logger.trace("Registering Database Extension Commands......"); - var RS = Packages.com.google.refine.RefineServlet; - RS.registerCommand(module, "test-connect", Packages.com.google.refine.extension.database.cmd.TestConnectCommand()); - RS.registerCommand(module, "connect", Packages.com.google.refine.extension.database.cmd.ConnectCommand()); - RS.registerCommand(module, "saved-connection", Packages.com.google.refine.extension.database.cmd.SavedConnectionCommand()); - RS.registerCommand(module, "execute-query", Packages.com.google.refine.extension.database.cmd.ExecuteQueryCommand()); - RS.registerCommand(module, "test-query", Packages.com.google.refine.extension.database.cmd.TestQueryCommand()); - logger.trace("Database Extension Command Registration done!!"); -} - -function registerOperations() { -} - -function registerFunctions() { -} - - -/* - * Function invoked to initialize the extension. - */ -function init() { - - logger.trace("Initializing OpenRefine Database Extension..."); - logger.trace("Database Extension Mount point " + module.getMountPoint()); - - registerCommands(); - registerOperations(); - registerFunctions(); - - - // Register importer and exporter - var IM = Packages.com.google.refine.importing.ImportingManager; - - IM.registerController( - module, - "database-import-controller", - new Packages.com.google.refine.extension.database.DatabaseImportController() - ); - - - // Script files to inject into /index page - ClientSideResourceManager.addPaths( - "index/scripts", - module, - [ - "scripts/index/jquery.contextMenu.min.js", - "scripts/index/jquery.ui.position.min.js", - "scripts/database-extension.js", - "scripts/index/database-import-controller.js", - "scripts/index/database-source-ui.js" - ] - ); - // Style files to inject into /index page - ClientSideResourceManager.addPaths( - "index/styles", - module, - [ - "styles/jquery.contextMenu.css", - "styles/pure.css", - "styles/bootstrap.css", - "styles/database-import.less" - - ] - ); - - // Script files to inject into /project page - ClientSideResourceManager.addPaths( - "project/scripts", - module, - [ - "scripts/database-extension.js", - "scripts/project/database-exporters.js" - ] - ); -} - -/* - * Function invoked to handle each request in a custom way. - */ -function process(path, request, response) { - - - var method = request.getMethod(); - - logger.trace('receiving request for ' + path); - - if (path == "/" || path == "") { - var context = {}; - context.version = version; - send(request, response, "index.vt", context); - } -} - -function send(request, response, template, context) { - butterfly.sendTextFromTemplate(request, response, context, template, encoding, html); -} diff --git a/OpenRefine/extensions/database/module/MOD-INF/dbextension.properties b/OpenRefine/extensions/database/module/MOD-INF/dbextension.properties deleted file mode 100644 index db482f687..000000000 --- a/OpenRefine/extensions/database/module/MOD-INF/dbextension.properties +++ /dev/null @@ -1,3 +0,0 @@ -# Batch size for import data -preview.batchSize = 100 -create.batchSize = 1000 diff --git a/OpenRefine/extensions/database/module/MOD-INF/module.properties b/OpenRefine/extensions/database/module/MOD-INF/module.properties deleted file mode 100644 index 2b0cef961..000000000 --- a/OpenRefine/extensions/database/module/MOD-INF/module.properties +++ /dev/null @@ -1,7 +0,0 @@ -name = database -description = Database importer/exporter for OpenRefine -templating.macros = macros.vm -requires = core - -# Use our custom class for a module implementation. -module-impl = com.google.refine.extension.database.DatabaseModuleImpl diff --git a/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.eot b/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.eot deleted file mode 100644 index b93a4953f..000000000 Binary files a/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.eot and /dev/null differ diff --git a/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.svg b/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.svg deleted file mode 100644 index 94fb5490a..000000000 --- a/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.svg +++ /dev/null @@ -1,288 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.ttf b/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.ttf deleted file mode 100644 index 1413fc609..000000000 Binary files a/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.ttf and /dev/null differ diff --git a/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.woff b/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.woff deleted file mode 100644 index 9e612858f..000000000 Binary files a/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.woff and /dev/null differ diff --git a/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.woff2 b/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.woff2 deleted file mode 100644 index 64539b54c..000000000 Binary files a/OpenRefine/extensions/database/module/images/fonts/glyphicons-halflings-regular.woff2 and /dev/null differ diff --git a/OpenRefine/extensions/database/module/images/more-option-horiz-16.png b/OpenRefine/extensions/database/module/images/more-option-horiz-16.png deleted file mode 100644 index 0620d0a57..000000000 Binary files a/OpenRefine/extensions/database/module/images/more-option-horiz-16.png and /dev/null differ diff --git a/OpenRefine/extensions/database/module/images/more_option-vert-16.png b/OpenRefine/extensions/database/module/images/more_option-vert-16.png deleted file mode 100644 index fa41b4f3b..000000000 Binary files a/OpenRefine/extensions/database/module/images/more_option-vert-16.png and /dev/null differ diff --git a/OpenRefine/extensions/database/module/index.vt b/OpenRefine/extensions/database/module/index.vt deleted file mode 100644 index 147cb507b..000000000 --- a/OpenRefine/extensions/database/module/index.vt +++ /dev/null @@ -1,20 +0,0 @@ -#* - * Access this page at the URL /extension/database/ - *# - - - OpenRefine Database Extension v$version - - -

OpenRefine DATABASE Extension v$version

-

by Tony Opara

-

The JDBC extension allows OpenRefine to read - (and write, soon) data from JDBC compatible databases(MySQL, PostgreSQL, MariaDB). -

-

Known Limitations

-
    -
  • No write support
  • - -
- - \ No newline at end of file diff --git a/OpenRefine/extensions/database/module/langs/translation-bn.json b/OpenRefine/extensions/database/module/langs/translation-bn.json deleted file mode 100644 index 97fc6d466..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-bn.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "database-import/preparing": "ফলাফল তৈরি করা হচ্ছে…", - "database-import/creating": "প্রকল্প তৈরি হচ্ছে…" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-cs.json b/OpenRefine/extensions/database/module/langs/translation-cs.json deleted file mode 100644 index 11f3ec52b..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-cs.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "database-source/databasePasswordPlaceholder": "Zadejte heslo do databáze", - "database-parsing/ignore": "řádek/řádky/řádků na začátku souboru", - "database-parsing/ignore-first": "Ignorovat první(ch)", - "database-source/alert-db-port-invalid-character": "Chyba v označení portu databáze: neplatný znak na vstupu. Povolené znaky: číslice.", - "database-source/alert-db-user-invalid-character": "Chyba v názvu uživatele databáze: neplatný znak na vstupu. Povolené znaky: písmena a číslice", - "database-source/alert-db-host-invalid-character": "Chyba v názvu hostitele databáze: neplatný znak na vstupu. Povolené znaky: písmena a číslice", - "database-source/alert-conn-name-invalid-character": "Chyba v názvu připojení: neplatný znak na vstupu. Povolené znaky: [a-zA-Z0-9._-]", - "database-parsing/store-row": "Uložit prázdné řádky", - "database-parsing/store-cell": "Uložit prádné buňky jako null", - "database-parsing/limit": "řádek/řádky/řádků dat", - "database-parsing/limit-next": "Nahrát nejvýše", - "database-parsing/discard": "řádek/řádky/řádků dat", - "database-parsing/discard-next": "Vynechat počáteční(ch)", - "database-parsing/parse": "řádek/řádky jako hlavičky sloupců", - "database-parsing/parse-next": "Analyzuj další(ch)", - "database-parsing/preview-button": "Aktualizuj náhled", - "database-parsing/option": "Možnosti", - "database-parsing/worksheet": "Pracovní listy", - "database-parsing/updating-preview": "Aktualizace náhledu…", - "database-parsing/create-proj": "Vytvoři projekt »", - "database-parsing/proj-name": "Názevt projektu", - "database-parsing/conf-pars": "Nastavit možnosti parsování", - "database-parsing/start-over": "« Začít znovu", - "database-source/savedConnectionSpan": "Uložená připojení", - "database-source/new-connection-legend": "Editor nového připojení", - "database-source/newConnectionButtonDiv": "Nové připojení", - "database-source/databaseConnectButton": "Připojit", - "database-source/databaseSaveButton": "Uložit", - "database-source/databaseTestButton": "Otestovat", - "database-source/connectionNameDefaultValue": "Nazev_noveho_pripojeni", - "database-source/databaseSchemaPlaceholder": "Zadejte výchozí schéma databáze", - "database-source/databaseFileNamePlaceholder": "Zadejte plnou cestu k databázovému souboru", - "database-source/databaseNamePlaceholder": "Zadejte název databáze", - "database-source/databaseUserPlaceholder": "Zadejte uživatele databáze", - "database-source/databasePortPlaceholder": "Zadejte port databáze, např. 3306", - "database-source/databaseHostPlaceholder": "Zadejte doménu nebo IP adresu databázového serveru", - "database-source/connectionNamePlaceholder": "Zadejte nový název připojení", - "database-source/databaseSchemaLabel": "Schéma:", - "database-source/databaseFileNameLabel": "Databázový soubor:", - "database-source/databaseNameLabel": "Název databáze:", - "database-source/databasePasswordLabel": "Heslo:", - "database-source/databaseUserLabel": "Uživatel:", - "database-source/databasePortLabel": "Port:", - "database-source/databaseHostLabel": "Hostitel:", - "database-source/databaseTypeLabel": "Typ:", - "database-source/connectionNameLabel": "Název:", - "database-source/alert-connection-edit": "Připojení bylo upraveno", - "database-source/form-validation-failure": "Nová podpoba připojení není platná!", - "database-source/alert-invalid-query-select": "Dotaz musí začínat klíčovým slovem SELECT", - "database-source/alert-invalid-query-keyword": "Dotaz nemůže obsahovat klíčové slovo pro manipulaci s daty:", - "database-source/alert-query": "Musíte zadat validní dotaz", - "database-source/alert-initial-database": "Musíte určit výchozí databázi", - "database-source/alert-connection-name": "Musíte určit název připojení", - "database-source/alert-password": "Musíte určit heslo pro databázi", - "database-source/alert-user": "Musíte určit uživatele databáze", - "database-source/alert-port": "Musíte určit port u databáze", - "database-source/alert-host": "Musíte určit hostitele databáze", - "database-import/creating": "Vytváření projektu…", - "database-import/checking": "Kontrola dotazu…", - "database-import/preparing": "Příprava výsledků…", - "database-import/title": "Databázové servery", - "database-import/importer-name": "Databáze" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-en.json b/OpenRefine/extensions/database/module/langs/translation-en.json deleted file mode 100644 index d72ffb805..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-en.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "database-import/importer-name": "Database", - "database-import/title": "Database Servers", - "database-import/preparing": "Preparing result …", - "database-import/checking": "Validating query …", - "database-import/creating": "Creating project …", - "database-source/alert-host": "You must specify a database host", - "database-source/alert-port": "You must specify a database port", - "database-source/alert-user": "You must specify a database user", - "database-source/alert-password": "You must specify a database password", - "database-source/alert-connection-name": "You must specify a connection name", - "database-source/alert-initial-database": "You must specify an initial database", - "database-source/alert-query": "You must specify a valid query", - "database-source/alert-invalid-query-keyword": "Query cannot contain data manipulation keyword:", - "database-source/alert-invalid-query-select": "Query must start with SELECT Keyword", - "database-source/form-validation-failure": "New connection form invalid!", - "database-source/alert-connection-edit": "Connection edited", - "database-source/connectionNameLabel": "Name:", - "database-source/databaseTypeLabel": "Type:", - "database-source/databaseHostLabel": "Host:", - "database-source/databasePortLabel": "Port:", - "database-source/databaseUserLabel": "User:", - "database-source/databasePasswordLabel": "Password:", - "database-source/databaseNameLabel": "Database name:", - "database-source/databaseFileNameLabel": "Database file:", - "database-source/databaseSchemaLabel": "Schema:", - "database-source/connectionNamePlaceholder": "Enter a new Connection Name", - "database-source/databaseHostPlaceholder": "Enter the database server domain or IP", - "database-source/databasePortPlaceholder": "Enter the database port e.g. 3306", - "database-source/databaseUserPlaceholder": "Enter the database user", - "database-source/databasePasswordPlaceholder": "Enter the database password", - "database-source/databaseNamePlaceholder": "Enter the name of the database", - "database-source/databaseFileNamePlaceholder": "Enter the full path to the database file", - "database-source/databaseSchemaPlaceholder": "Enter the initial schema of the database", - "database-source/connectionNameDefaultValue": "New_Connection_Name", - "database-source/databaseTestButton": "Test", - "database-source/databaseSaveButton": "Save", - "database-source/databaseConnectButton": "Connect", - "database-source/newConnectionButtonDiv": "New connection", - "database-source/new-connection-legend":"New Connection Editor", - "database-source/savedConnectionSpan": "Saved connections", - "database-parsing/start-over": "« Start over", - "database-parsing/conf-pars": "Configure parsing options", - "database-parsing/proj-name": "Project name", - "database-parsing/create-proj": "Create Project »", - "database-parsing/updating-preview": "Updating preview …", - "database-parsing/worksheet": "Worksheets", - "database-parsing/option": "Options", - "database-parsing/preview-button": "Update preview", - "database-parsing/disable-auto-preview": "Disable auto preview", - "database-parsing/ignore-first": "Ignore first", - "database-parsing/ignore": "line(s) at beginning of file", - "database-parsing/parse-next": "Parse next", - "database-parsing/parse": "line(s) as column headers", - "database-parsing/discard-next": "Discard initial", - "database-parsing/discard": "row(s) of data", - "database-parsing/limit-next": "Load at most", - "database-parsing/limit": "row(s) of data", - "database-parsing/store-row": "Store blank rows", - "database-parsing/store-cell": "Store blank cells as nulls", - "database-source/alert-conn-name-invalid-character": "Connection Name Input Error: Illegal Character in Input. Only [a-zA-Z0-9._-] Allowed", - "database-source/alert-db-host-invalid-character": "Database Host Error: Illegal Character in Input. Only Alphanumeric characters allowed", - "database-source/alert-db-user-invalid-character": "Database User Error: Illegal Character in Input. Only Alphanumeric characters allowed", - "database-source/alert-db-port-invalid-character": "Database Port Error: Illegal Character in Input. Only Numeric values allowed." - -} diff --git a/OpenRefine/extensions/database/module/langs/translation-en_GB.json b/OpenRefine/extensions/database/module/langs/translation-en_GB.json deleted file mode 100644 index bd0cfebbb..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-en_GB.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "database-source/alert-db-port-invalid-character": "Database Port Error: Illegal Character in Input. Only Numeric values allowed.", - "database-source/alert-db-user-invalid-character": "Database User Error: Illegal Character in Input. Only Alphanumeric characters allowed", - "database-source/alert-db-host-invalid-character": "Database Host Error: Illegal Character in Input. Only Alphanumeric characters allowed", - "database-source/alert-conn-name-invalid-character": "Connection Name Input Error: Illegal Character in Input. Only [a-zA-Z0-9._-] Allowed", - "database-parsing/store-cell": "Store blank cells as nulls", - "database-parsing/store-row": "Store blank rows", - "database-parsing/limit": "row(s) of data", - "database-parsing/limit-next": "Load at most", - "database-parsing/discard": "row(s) of data", - "database-parsing/discard-next": "Discard initial", - "database-parsing/parse": "line(s) as column headers", - "database-parsing/parse-next": "Parse next", - "database-parsing/ignore": "line(s) at beginning of file", - "database-parsing/ignore-first": "Ignore first", - "database-parsing/preview-button": "Update preview", - "database-parsing/option": "Options", - "database-parsing/worksheet": "Worksheets", - "database-parsing/updating-preview": "Updating preview …", - "database-parsing/create-proj": "Create Project »", - "database-parsing/proj-name": "Project name", - "database-parsing/conf-pars": "Configure parsing options", - "database-parsing/start-over": "« Start over", - "database-source/savedConnectionSpan": "Saved connections", - "database-source/newConnectionButtonDiv": "New connection", - "database-source/databaseConnectButton": "Connect", - "database-source/databaseSaveButton": "Save", - "database-source/databaseTestButton": "Test", - "database-source/databaseSchemaLabel": "Schema:", - "database-source/databaseNameLabel": "Database:", - "database-source/databaseFileNameLabel": "Database file:", - "database-source/databasePasswordLabel": "Password:", - "database-source/databaseUserLabel": "User:", - "database-source/databasePortLabel": "Port:", - "database-source/databaseHostLabel": "Host:", - "database-source/databaseTypeLabel": "Type:", - "database-source/connectionNameLabel": "Name:", - "database-source/alert-connection-edit": "Connection edited", - "database-source/form-validation-failure": "New connection form invalid!", - "database-source/alert-invalid-query-select": "Query must start with SELECT Keyword", - "database-source/alert-invalid-query-keyword": "Query cannot contain data manipulation keyword:", - "database-source/alert-connection-name": "You must specify a connection name", - "database-source/alert-query": "You must specify a valid query", - "database-source/alert-initial-database": "You must specify an initial database", - "database-source/alert-password": "You must specify a database password", - "database-source/alert-user": "You must specify a database user", - "database-source/alert-port": "You must specify a database port", - "database-source/alert-host": "You must specify a database host", - "database-import/creating": "Creating project …", - "database-import/checking": "Validating query …", - "database-import/preparing": "Preparing result …", - "database-import/title": "Database Servers" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-eu.json b/OpenRefine/extensions/database/module/langs/translation-eu.json deleted file mode 100644 index 0967ef424..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-eu.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/OpenRefine/extensions/database/module/langs/translation-fr.json b/OpenRefine/extensions/database/module/langs/translation-fr.json deleted file mode 100644 index e64f8eb1e..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-fr.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "database-import/importer-name": "Base de données", - "database-import/title": "Serveurs de base de données", - "database-import/preparing": "Préparation du résultat…", - "database-import/checking": "Validation de la requête…", - "database-import/creating": "Création du projet…", - "database-source/alert-host": "Vous devez spécifier un hôte de base de données", - "database-source/alert-port": "Vous devez spécifier un port de base de données", - "database-source/alert-user": "Vous devez spécifier un utilisateur de base de données", - "database-source/alert-password": "Vous devez spécifier un mot de passe de base de données", - "database-source/alert-connection-name": "Vous devez spécifier un nom de connexion", - "database-source/alert-initial-database": "Vous devez spécifier une base de données initiale", - "database-source/alert-query": "Vous devez spécifier une requête valide", - "database-source/alert-invalid-query-keyword": "La requête ne peut pas contenir ce mot clé destiné à manipuler la base de données :", - "database-source/alert-invalid-query-select": "La requête doit débuter par le mot-clé SELECT", - "database-source/form-validation-failure": "Le nouveau formulaire de connexion est invalide !", - "database-source/alert-connection-edit": "La connexion a été éditée avec succès", - "database-source/connectionNameLabel": "Nom :", - "database-source/databaseTypeLabel": "Type :", - "database-source/databaseHostLabel": "Hôte :", - "database-source/databasePortLabel": "Port :", - "database-source/databaseUserLabel": "Utilisateur :", - "database-source/databasePasswordLabel": "Mot de passe :", - "database-source/databaseNameLabel": "Base de données :", - "database-source/databaseFileNameLabel": "Fichier de données :", - "database-source/databaseSchemaLabel": "Schéma :", - "database-source/connectionNamePlaceholder": "Entrez le nom de la connexion", - "database-source/databaseHostPlaceholder": "Entrez l’adresse de l’hôte de la base de données", - "database-source/databasePortPlaceholder": "Entrez le port de la base de données par exemple 3306", - "database-source/databaseUserPlaceholder": "Entrez l’utilisateur de la base de données", - "database-source/databasePasswordPlaceholder": "Entrez le mot de passe d’utilisateur", - "database-source/databaseNamePlaceholder": "Entrez le nom de la base de données", - "database-source/databaseFileNamePlaceholder": "Entrez le chemin du fichier base de données", - "database-source/databaseSchemaPlaceholder": "Entrez le schéma initial de la base de données", - "database-source/connectionNameDefaultValue": "Nom_nouvelle_connexion", - "database-source/databaseTestButton": "Test", - "database-source/databaseSaveButton": "Enregistrer", - "database-source/databaseConnectButton": "Connexion", - "database-source/newConnectionButtonDiv": "Nouvelle connexion", - "database-source/new-connection-legend": "Éditeur de nouvelle connexion", - "database-source/savedConnectionSpan": "Connexions enregistrées", - "database-parsing/start-over": "« Recommencer", - "database-parsing/conf-pars": "Configurer les options d'analyse syntaxique (parsing)", - "database-parsing/proj-name": "Nom du projet", - "database-parsing/create-proj": "Créer un projet »", - "database-parsing/updating-preview": "Mise à jour de l'aperçu…", - "database-parsing/worksheet": "Feuilles de calcul", - "database-parsing/option": "Options", - "database-parsing/preview-button": "Mise à jour de l'aperçu", - "database-parsing/ignore-first": "Ignorer la/les", - "database-parsing/ignore": "première(s) ligne(s) au début du fichier", - "database-parsing/parse-next": "Analyser (parse) la/les", - "database-parsing/parse": "ligne(s) suivante(s) comme des entêtes de colonnes", - "database-parsing/discard-next": "Omettre la/les", - "database-parsing/discard": "première(s) ligne(s) de données", - "database-parsing/limit-next": "Charger au plus", - "database-parsing/limit": "ligne(s) de données", - "database-parsing/store-row": "Stocker les lignes vides", - "database-parsing/store-cell": "Enregistrer les cellules vides comme des valeurs nulles", - "database-source/alert-db-port-invalid-character": "Erreur de port de base de données : caractère illégal en entrée. Seules les valeurs numériques sont autorisées.", - "database-source/alert-db-user-invalid-character": "Erreur de l'utilisateur de la base de données : caractère illégal en entrée. Seuls les caractères alphanumériques sont autorisés", - "database-source/alert-db-host-invalid-character": "Erreur de l'hôte de la base de données : caractère non autorisé en entrée. Seuls les caractères alphanumériques sont autorisés", - "database-source/alert-conn-name-invalid-character": "Erreur de saisie du nom de connexion : caractère non autorisé en entrée. Uniquement [a-zA-Z0-9 ._-] autorisé", - "database-parsing/disable-auto-preview": "Désactiver l'aperçu automatique" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-he.json b/OpenRefine/extensions/database/module/langs/translation-he.json deleted file mode 100644 index 76f5ff07f..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-he.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "database-parsing/limit": "שורות נתונים לכל היותר", - "database-parsing/limit-next": "לטעון", - "database-parsing/discard": "שורות הנתונים הראשונות", - "database-parsing/discard-next": "התעלמות מ־", - "database-parsing/parse": "השורות הבאות ככותרות העמודות", - "database-parsing/parse-next": "לעבד את", - "database-parsing/ignore": "השורות הראשונות בתחילת הקובץ", - "database-parsing/ignore-first": "התעלמות מ־", - "database-source/alert-db-port-invalid-character": "שגיאת פתחת מסד נתונים: תו שגוי בקלט. מותר רק ספרות.", - "database-source/alert-db-user-invalid-character": "שגיאת משתמש מסד נתונים: תו שגוי בקלט. מותר רק תווים אלפאנומריים לטיניים", - "database-source/alert-db-host-invalid-character": "שגיאת מארח מסד נתונים: תו שגוי בקלט. מותר רק תווים אלפאנומריים לטיניים", - "database-source/alert-conn-name-invalid-character": "שגיאת קלט שם חיבור: תו שגוי בקלט. מותר רק [a-zA-Z0-9._-‎]", - "database-parsing/store-cell": "לאחסן תאים ריקים כ־null (ריק)", - "database-parsing/store-row": "לאחסן שורות ריקות", - "database-parsing/preview-button": "עדכון תצוגה מקדימה", - "database-parsing/option": "אפשרויות", - "database-parsing/worksheet": "גיליונות עבודה", - "database-parsing/updating-preview": "התצוגה המקדימה מעודכנת…", - "database-parsing/create-proj": "יצירת מיזם «", - "database-parsing/proj-name": "שם המיזם", - "database-parsing/conf-pars": "הגדרת אפשרויות פענוח", - "database-parsing/start-over": "» להתחיל מחדש", - "database-source/savedConnectionSpan": "חיבורים שנשמרו", - "database-source/new-connection-legend": "עורך חיבורים חדשים", - "database-source/newConnectionButtonDiv": "חיבור חדש", - "database-source/databaseConnectButton": "התחברות", - "database-source/databaseSaveButton": "שמירה", - "database-source/databaseTestButton": "בדיקה", - "database-source/connectionNameDefaultValue": "שם_חיבור_חדש", - "database-source/databaseSchemaPlaceholder": "נא למלא את הסכמה הראשונית של מסד הנתונים", - "database-source/databaseFileNamePlaceholder": "נא למלא את הנתיב המלא לקובץ מסד הנתונים", - "database-source/databaseNamePlaceholder": "נא למלא את שם מסד הנתונים", - "database-source/databasePasswordPlaceholder": "נא למלא את ססמת מסד הנתונים", - "database-source/databaseUserPlaceholder": "נא למלא את משתמש מסד הנתונים", - "database-source/databasePortPlaceholder": "נא למלא את פתחת מסד הנתונים, למשל: 3306", - "database-source/databaseHostPlaceholder": "נא למלא את השם או ה־IP של מסד הנתונים", - "database-source/connectionNamePlaceholder": "נא למלא שם חדש לחיבור", - "database-source/databaseSchemaLabel": "סכמה:", - "database-source/databaseFileNameLabel": "קובץ מסד נתונים:", - "database-source/databaseNameLabel": "שם מסד הנתונים:", - "database-source/databasePasswordLabel": "ססמה:", - "database-source/databaseUserLabel": "משתמש:", - "database-source/databasePortLabel": "פתחה:", - "database-source/databaseHostLabel": "מארח:", - "database-source/databaseTypeLabel": "סוג:", - "database-source/connectionNameLabel": "שם:", - "database-source/alert-connection-edit": "החיבור נערך", - "database-source/form-validation-failure": "טופס החיבור החדש שגוי!", - "database-source/alert-invalid-query-select": "שאילתה חייבת להתחיל במילת המפתח SELECT (בחירה)", - "database-source/alert-invalid-query-keyword": "השאילתה לא יכולה להכיל מילת מפתח שמשנה נתונים:", - "database-source/alert-query": "עליך לציין שאילתה תקפה", - "database-source/alert-initial-database": "עליך לציין מסד נתונים ראשוני", - "database-source/alert-connection-name": "עליך לציין שם לחיבור", - "database-source/alert-password": "עליך לציין ססמה למסד הנתונים", - "database-source/alert-user": "עליך לציין משתמש למסד הנתונים", - "database-source/alert-port": "עליך לציין פתחה למסד הנתונים", - "database-source/alert-host": "עליך לציין מארח למסד הנתונים", - "database-import/creating": "המיזם נוצר…", - "database-import/checking": "השאילתה מתוקפת…", - "database-import/preparing": "התוצאות בהכנות…", - "database-import/title": "שרתי מסדי נתונים", - "database-import/importer-name": "מסד נתונים", - "database-parsing/disable-auto-preview": "השבתת תצוגה מקדימה אוטומטית" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-id.json b/OpenRefine/extensions/database/module/langs/translation-id.json deleted file mode 100644 index e77b35231..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-id.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "database-source/alert-db-port-invalid-character": "Error Port Basis Data: Karakter Ilegal pada input. Hanya boleh nilai numerik.", - "database-source/alert-db-user-invalid-character": "Error Pengguna Basis Data: Karakter Ilegal pada input. Hanya boleh karakter alfanumerik", - "database-source/alert-db-host-invalid-character": "Error Host Basis Data: Karakter Ilegal pada input. Hanya boleh karakter alfanumerik", - "database-source/alert-conn-name-invalid-character": "Error Masukkan Nama Koneksi: Karakter Ilegal pada input. Hanya Boleh [a-zA-Z0-9._-]", - "database-parsing/store-cell": "Simpan cell kosong sebagai null", - "database-parsing/store-row": "Simpan baris kosong", - "database-parsing/limit": "baris data", - "database-parsing/limit-next": "Muat paling banyak", - "database-parsing/discard": "baris data", - "database-parsing/discard-next": "Buang awalan", - "database-parsing/parse": "baris sebagai header kolom", - "database-parsing/parse-next": "Uraikan berikutnya", - "database-parsing/ignore": "baris di awal berkas", - "database-parsing/ignore-first": "Abaikan dahulu", - "database-parsing/preview-button": "Pratinjau Pembaruan", - "database-parsing/option": "Pilihan", - "database-parsing/worksheet": "Lembar kerja", - "database-parsing/updating-preview": "Pratinjau pembaruan …", - "database-parsing/create-proj": "Buat Proyek»", - "database-parsing/proj-name": "Nama proyek", - "database-parsing/conf-pars": "Mengkonfigurasi pilihan parsing", - "database-parsing/start-over": "Mulai lagi", - "database-source/savedConnectionSpan": "Koneksi tersimpan", - "database-source/new-connection-legend": "Editor Koneksi Baru", - "database-source/newConnectionButtonDiv": "Koneksi baru", - "database-source/databaseConnectButton": "Hubungkan", - "database-source/databaseSaveButton": "Simpan", - "database-source/databaseTestButton": "Tes", - "database-source/connectionNameDefaultValue": "Nama_Konseksi_Baru", - "database-source/databaseSchemaPlaceholder": "Masukkan skema awal basis data", - "database-source/databaseFileNamePlaceholder": "Masukkan path lengkap berkas basis data", - "database-source/databaseNamePlaceholder": "Masukkan nama basis data", - "database-source/databasePasswordPlaceholder": "Masukkan password basis data", - "database-source/databaseUserPlaceholder": "Masukkan pengguna basis data", - "database-source/databasePortPlaceholder": "Masukkan port basis data, misal: 3306", - "database-source/databaseHostPlaceholder": "Masukkan domain server basis data atau IP", - "database-source/connectionNamePlaceholder": "Masukkan sebuah Nama Koneksi baru", - "database-source/databaseSchemaLabel": "Skema:", - "database-source/databaseFileNameLabel": "Berkas basis data:", - "database-source/databaseNameLabel": "Nama basis data:", - "database-source/databasePasswordLabel": "Password:", - "database-source/databaseUserLabel": "Pengguna:", - "database-source/databasePortLabel": "Port:", - "database-source/databaseHostLabel": "Host:", - "database-source/databaseTypeLabel": "Tipe:", - "database-source/connectionNameLabel": "Nama:", - "database-source/alert-connection-edit": "Koneksi diubah", - "database-source/form-validation-failure": "Form koneksi baru tidak valid!", - "database-source/alert-invalid-query-select": "Kueri harus dimulai dengan kata kunci SELECT", - "database-source/alert-invalid-query-keyword": "Kueri tidak bisa mengandung kata kunci manipulasi data:", - "database-source/alert-query": "Anda harus menentukan sebuah kueri yang valid", - "database-source/alert-initial-database": "Anda harus menentukan basis data awal", - "database-source/alert-connection-name": "Anda harus membuat nama koneksi", - "database-source/alert-password": "Anda harus menetukan password basis data", - "database-source/alert-user": "Anda harus menentukan pengguna basis data", - "database-source/alert-port": "Anda harus menentukan sebuah port basis data", - "database-source/alert-host": "Anda harus menentukan sebuah host basis data", - "database-import/preparing": "Menyiapkan hasil …", - "database-import/creating": "Membuat proyek …", - "database-import/checking": "Memvalidasi kueri …", - "database-import/title": "Server Basis Data", - "database-import/importer-name": "Basis Data" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-it.json b/OpenRefine/extensions/database/module/langs/translation-it.json deleted file mode 100644 index f327b98de..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-it.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "database-import/title": "Server del database", - "database-import/preparing": "Risultato in preparazione…", - "database-import/checking": "Validazione query…", - "database-import/creating": "Creazione progetto…", - "database-source/alert-host": "È necessario specificare un host per il database", - "database-source/alert-port": "È necessario specificare una porta di connessione al database", - "database-source/alert-user": "È necessario specificare un utente per la connessione al database", - "database-source/alert-password": "È necessario specificare una password per la connessione al database", - "database-source/alert-connection-name": "È necessario impostare un nome alla connessione", - "database-source/alert-initial-database": "È necessario specificare un database iniziale", - "database-source/alert-query": "È necessario impostare una query valida", - "database-source/alert-invalid-query-keyword": "La query non può contenere keyword di manipolazione dati:", - "database-source/alert-invalid-query-select": "La query deve iniziare con la keyword SELECT", - "database-source/alert-connection-edit": "Connessione modificata", - "database-source/connectionNameLabel": "Nome:", - "database-source/databaseTypeLabel": "Tipo:", - "database-source/databaseHostLabel": "Host:", - "database-source/databasePortLabel": "Porta:", - "database-source/databaseUserLabel": "Utente:", - "database-source/databasePasswordLabel": "Password:", - "database-source/databaseNameLabel": "Database:", - "database-source/databaseFileNameLabel": "Database file:", - "database-source/databaseSchemaLabel": "Schema:", - "database-source/databaseTestButton": "Test", - "database-source/databaseSaveButton": "Salva", - "database-source/databaseConnectButton": "Connetti", - "database-source/newConnectionButtonDiv": "Nuova connessione", - "database-source/savedConnectionSpan": "Connessioni salvate", - "database-parsing/start-over": "« Riparti da capo", - "database-parsing/conf-pars": "Configura impostazioni di parsing", - "database-parsing/proj-name": "Nome progetto", - "database-parsing/create-proj": "Crea progetto »", - "database-parsing/updating-preview": "Aggiornamento anteprima…", - "database-parsing/worksheet": "Fogli di lavoro", - "database-parsing/option": "Opzioni", - "database-parsing/preview-button": "Aggiorna anteprima", - "database-parsing/ignore-first": "Ignora le prime", - "database-parsing/ignore": "righe dall'inizio del file", - "database-parsing/parse-next": "Usa le successive", - "database-parsing/parse": "linee come nomi delle colonne", - "database-parsing/discard-next": "Scarta le prime", - "database-parsing/discard": "righe di dati", - "database-parsing/limit-next": "Carica al massimo", - "database-parsing/limit": "righe di dati", - "database-parsing/store-row": "Salva righe vuote", - "database-parsing/store-cell": "Salva celle vuote come null", - "database-source/form-validation-failure": "Nuova forma di connessione non valida!", - "database-source/alert-conn-name-invalid-character": "Errore di input del nome di connessione: carattere non valido nell'input. Sono consentiti solo i caratteri [a-zA-z-0-9._-]", - "database-source/alert-db-host-invalid-character": "Errore dell'host del database: carattere non valido nell'input. Sono consentiti solo caratteri alfanumerici", - "database-source/alert-db-user-invalid-character": "Errore utente del database: carattere non valido nell'input. Sono consentiti solo caratteri alfanumerici", - "database-source/alert-db-port-invalid-character": "Errore porta database: carattere non valido nell'input. Sono consentiti solo valori numerici.", - "database-source/new-connection-legend": "Editor della nuova connessione", - "database-source/connectionNameDefaultValue": "Nome_nuova_connessione", - "database-source/databaseSchemaPlaceholder": "Indica lo schema iniziale del database", - "database-source/databaseFileNamePlaceholder": "Indica il percorso completo del file del database", - "database-source/databaseNamePlaceholder": "Indica il nome del database", - "database-source/databasePasswordPlaceholder": "Indica la password del database", - "database-source/databaseUserPlaceholder": "Indica il nome dell'utente del database", - "database-source/databasePortPlaceholder": "Indica il nome della porta del database, es. 3306", - "database-source/databaseHostPlaceholder": "Indica il nome del dominio o l'IP del server del database", - "database-source/connectionNamePlaceholder": "Indica il nome della nuova connessione", - "database-import/importer-name": "Database" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-jp.json b/OpenRefine/extensions/database/module/langs/translation-jp.json deleted file mode 100644 index 41d3e910b..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-jp.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "database-import/title": "データベースサーバー", - "database-import/preparing": "準備しています…", - "database-import/checking": "クエリーをチェックしています…", - "database-import/creating": "プロジェクトを作成中…", - "database-source/alert-host": "database hostを指定してください", - "database-source/alert-port": "database portを指定してください", - "database-source/alert-user": "database userを指定してください", - "database-source/alert-password": "database passwordを指定してください", - "database-source/alert-connection-name": "connection nameを指定してください", - "database-source/alert-initial-database": "最初のデータベースを指定してください", - "database-source/alert-query": "有効なクエリーを設定してください", - "database-source/alert-invalid-query-keyword": "クエリーにデータ操作語があります:", - "database-source/alert-invalid-query-select": "クエリーはSELECTで始まります", - "database-source/form-validation-failure": "接続フォームが無効です!", - "database-source/alert-connection-edit": "接続の設定に成功しました", - "database-source/connectionNameLabel": "接続先:", - "database-source/databaseTypeLabel": "タイプ:", - "database-source/databaseHostLabel": "ホスト:", - "database-source/databasePortLabel": "ポート番号:", - "database-source/databaseUserLabel": "ユーザー名:", - "database-source/databasePasswordLabel": "パスワード:", - "database-source/databaseNameLabel": "データベース名:", - "database-source/databaseFileNameLabel": "データベースファイル:", - "database-source/databaseSchemaLabel": "スキーマ :", - "database-source/databaseTestButton": "テスト", - "database-source/databaseSaveButton": "保存", - "database-source/databaseConnectButton": "接続", - "database-source/newConnectionButtonDiv": "新しい接続", - "database-source/savedConnectionSpan": "保存済み接続", - "database-parsing/start-over": "« やり直し", - "database-parsing/conf-pars": "パースオプションの設定", - "database-parsing/proj-name": "プロジェクト 名", - "database-parsing/create-proj": "プロジェクトを作成 »", - "database-parsing/updating-preview": "プレビューを更新中…", - "database-parsing/worksheet": "シート名", - "database-parsing/option": "オプション", - "database-parsing/preview-button": "プレビュー 更新", - "database-parsing/ignore-first": "最初の", - "database-parsing/ignore": "行を無視", - "database-parsing/parse-next": "その次の", - "database-parsing/parse": "行をカラム名として解釈", - "database-parsing/discard-next": "最初の", - "database-parsing/discard": "行分を破棄", - "database-parsing/limit-next": "最大で", - "database-parsing/limit": "行を読み込む", - "database-parsing/store-row": "空白行を保存", - "database-parsing/store-cell": "空白行をnullとして保存", - "database-source/alert-conn-name-invalid-character": "接続名エラー( [a-zA-Z0-9._-] のみ使用できます)", - "database-source/alert-db-host-invalid-character": "データベースホスト名エラー(記号は使用できません)", - "database-source/alert-db-user-invalid-character": "ユーザー名エラー(記号は使用できません)", - "database-source/alert-db-port-invalid-character": "ポート番号エラー(半角数字のみ可能)", - "database-source/connectionNameDefaultValue": "新規接続名", - "database-source/databaseSchemaPlaceholder": "データベースの初期スキーマを入力してください", - "database-source/databaseFileNamePlaceholder": "データベースファイルへのフルパスを入力してください", - "database-source/databaseNamePlaceholder": "データベース名を入力してください", - "database-source/databasePasswordPlaceholder": "パスワードを入力してください", - "database-source/databaseUserPlaceholder": "ユーザー名を入力してください", - "database-source/databasePortPlaceholder": "ポート番号を入力してください(例: 3306)", - "database-source/databaseHostPlaceholder": "データベースのドメイン名かIPアドレスを入力してください", - "database-source/connectionNamePlaceholder": "新規接続名を入力してください", - "database-source/new-connection-legend": "新規接続エディタ", - "database-import/importer-name": "データベース" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-ko.json b/OpenRefine/extensions/database/module/langs/translation-ko.json deleted file mode 100644 index 52b227021..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-ko.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "database-import/title": "데이터베이스 서버들", - "database-import/preparing": "결과를 준비중 …", - "database-import/checking": "쿼리를 검토중 …", - "database-import/creating": "프로젝트 생성중 …", - "database-source/alert-host": "하나의 데이터베이스 호스트를 지정해야합니다", - "database-source/alert-port": "하나의 데이터베이스 포트를 지정해야합니다", - "database-source/alert-user": "데이터베이스 사용자를 지정해야합니다", - "database-source/alert-password": "데이터베이스 비밀번호를 지정해야합니다", - "database-source/alert-connection-name": "접속이름을 지정해야 합니다", - "database-source/alert-initial-database": "초기 데이터베이스를 지정해야 합니다", - "database-source/alert-query": "유효한 쿼리를 지정해야 합니다", - "database-source/alert-invalid-query-keyword": "쿼리는 데이터 조작 키워드를 포함할 수 없습니다:", - "database-source/alert-invalid-query-select": "쿼리는 SELECT 키워드로 시작해야 합니다", - "database-source/form-validation-failure": "신규 접속이 유효하지 않습니다!", - "database-source/alert-connection-edit": "접속이 성공적으로 편집되었습니다", - "database-source/connectionNameLabel": "이름:", - "database-source/databaseTypeLabel": "타입:", - "database-source/databaseHostLabel": "호스트:", - "database-source/databasePortLabel": "포트:", - "database-source/databaseUserLabel": "사용자:", - "database-source/databasePasswordLabel": "비밀번호:", - "database-source/databaseNameLabel": "데이터베이스:", - "database-source/databaseFileNameLabel": "데이터베이스:", - "database-source/databaseSchemaLabel": "스키마:", - "database-source/databaseTestButton": "테스트", - "database-source/databaseSaveButton": "저장", - "database-source/databaseConnectButton": "접속", - "database-source/newConnectionButtonDiv": "신규 접속", - "database-source/savedConnectionSpan": "저장된 접속", - "database-parsing/start-over": "« 다시 시작", - "database-parsing/conf-pars": "파싱 옵션 설정", - "database-parsing/proj-name": "프로젝트 이름", - "database-parsing/create-proj": "프로젝트 생성 »", - "database-parsing/updating-preview": "미리보기를 업데이트중 …", - "database-parsing/worksheet": "워크시트", - "database-parsing/option": "옵션", - "database-parsing/preview-button": "미리보기 업데이트", - "database-parsing/ignore-first": "처음을 무시", - "database-parsing/ignore": "파일 시작시 라인(들)", - "database-parsing/parse-next": "다음을 파싱", - "database-parsing/parse": "컬럼 헤더 라인(들)", - "database-parsing/discard-next": "초기값을 버리세요", - "database-parsing/discard": "데이터 줄수", - "database-parsing/limit-next": "최대한 로드하세요", - "database-parsing/limit": "데이터 줄수", - "database-parsing/store-row": "빈 줄을 저장하세요", - "database-parsing/store-cell": "빈 셀을 null 로 저장하세요" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-ml.json b/OpenRefine/extensions/database/module/langs/translation-ml.json deleted file mode 100644 index 570ec34d8..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-ml.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "database-source/databaseSaveButton": "സേവ് ചെയ്യുക", - "database-source/databaseTestButton": "പരീക്ഷണം", - "database-source/databaseNamePlaceholder": "ഡാറ്റാബേസിന്റെ പേര് ചേർക്കുക", - "database-source/databaseNameLabel": "ഡാറ്റാബേസിന്റെ പേര്:", - "database-source/databasePasswordLabel": "രഹസ്യവാക്ക്:", - "database-source/databasePortLabel": "പോർട്ട്:", - "database-source/databaseTypeLabel": "തരം:", - "database-source/connectionNameLabel": "പേര്:", - "database-import/title": "ഡാറ്റാബേസ് സെർവറുകൾ", - "database-import/importer-name": "ഡാറ്റാബേസ്", - "database-parsing/start-over": "« ആദ്യം മുതൽ തുടങ്ങുക", - "database-source/databaseHostLabel": "ഹോസ്റ്റ്:", - "database-import/preparing": "ഫലം തയ്യാറാക്കുന്നു…" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-nb_NO.json b/OpenRefine/extensions/database/module/langs/translation-nb_NO.json deleted file mode 100644 index 70ebdfce4..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-nb_NO.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "database-import/title": "Databasetjenere", - "database-import/preparing": "Forbereder resultat …", - "database-import/checking": "Validerer spørring …", - "database-import/creating": "Oppretter prosjekt …", - "database-source/alert-host": "Du må angi en databasevert", - "database-source/alert-port": "Du må angi en databaseport", - "database-source/alert-user": "Du må angi en databasebruker", - "database-source/alert-password": "Du må angi et databasepassord", - "database-source/alert-connection-name": "Du må angi et tilkoblingsnavn", - "database-source/alert-initial-database": "Du må angi en initiell database", - "database-source/alert-query": "Du må angi en gyldig spørring", - "database-source/alert-invalid-query-select": "Spørringen må starte med nøkkelordet SELECT", - "database-source/alert-connection-edit": "Tilkobling redigert", - "database-source/connectionNameLabel": "Navn:", - "database-source/databaseTypeLabel": "Type:", - "database-source/databaseHostLabel": "Vert:", - "database-source/databasePortLabel": "Port:", - "database-source/databaseUserLabel": "Bruker:", - "database-source/databasePasswordLabel": "Passord:", - "database-source/databaseNameLabel": "Database:", - "database-source/databaseFileNameLabel": "Database file:", - "database-source/databaseSchemaLabel": "Skjema:", - "database-source/databaseTestButton": "Test", - "database-source/databaseSaveButton": "Lagre", - "database-source/databaseConnectButton": "Koble til", - "database-source/newConnectionButtonDiv": "Ny tilkobling", - "database-source/savedConnectionSpan": "Lagrede tilkoblinger", - "database-parsing/start-over": "« Begynn på nytt", - "database-parsing/conf-pars": "Sett opp tolkningsinnstillinger", - "database-parsing/proj-name": "Prosjektnavn", - "database-parsing/create-proj": "Opprett prosjekt »", - "database-parsing/updating-preview": "Oppdaterer forhåndsvisning …", - "database-parsing/option": "Innstillinger", - "database-parsing/preview-button": "Oppdater forhåndsvisning", - "database-parsing/ignore-first": "Ignorer første", - "database-parsing/ignore": "linje(r) i begynnelsen av fila", - "database-parsing/parse-next": "Tolk neste", - "database-parsing/parse": "linje(r) som kolonneoverskrifter", - "database-parsing/discard-next": "Forkast de første", - "database-parsing/discard": "raden(e) med data", - "database-parsing/limit-next": "Last maksimalt", - "database-parsing/limit": "rad(er) med data", - "database-parsing/store-row": "Lagre blanke rader", - "database-parsing/store-cell": "Lagre blanke celler som null", - "database-source/alert-invalid-query-keyword": "Spørring kan ikke inneholde datamanipuleringsnøkkelord:", - "database-source/form-validation-failure": "Nytt tilkoblingsskjema ugyldig!", - "database-parsing/worksheet": "Regneark", - "database-source/databaseHostPlaceholder": "Skriv inn databasetjener eller IP", - "database-source/connectionNamePlaceholder": "Skriv inn nytt tilkoblingsnavn" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-pl.json b/OpenRefine/extensions/database/module/langs/translation-pl.json deleted file mode 100644 index 51e6a21a3..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-pl.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "database-source/newConnectionButtonDiv": "Nowe połączenie", - "database-source/databaseConnectButton": "Połącz", - "database-source/databaseSaveButton": "Zapisz", - "database-source/databaseNameLabel": "Nazwa Bazy danych:", - "database-source/databasePasswordLabel": "Hasło:", - "database-source/databaseUserLabel": "Użytkownik:", - "database-source/databaseTypeLabel": "Typ:", - "database-source/connectionNameLabel": "Nazwa:", - "database-import/creating": "Tworzenie projektu …", - "database-import/preparing": "Przygotowanie wyników …", - "database-import/title": "Serwis bazy danych", - "database-import/importer-name": "Baza danych", - "database-parsing/store-cell": "Zachowaj puste komórki jako null", - "database-parsing/store-row": "Zachowaj puste wiersze", - "database-parsing/limit": "Wiersz(e) danych", - "database-parsing/discard": "Wiersz(e) danych", - "database-parsing/parse-next": "Analizuj następne", - "database-parsing/ignore": "linia(e) na początku pliku", - "database-parsing/preview-button": "Aktualizuj podgląd", - "database-parsing/create-proj": "Utwórz projekt »", - "database-parsing/proj-name": "Nazwa projektu", - "database-parsing/conf-pars": "Skonfiguruj opcje analizowania", - "database-parsing/start-over": "« Zacznij od nowa", - "database-parsing/discard-next": "Odrzuć początkowe", - "database-parsing/parse": "Wiersz(e) jako nagłówki kolumn", - "database-parsing/limit-next": "Załaduj co najwyżej", - "database-source/alert-db-user-invalid-character": "Błąd użytkownika bazy danych. Wprowadzono zabronione znaki. Dozwolone jedynie znaki alfanumeryczne", - "database-source/new-connection-legend": "Edytor nowego połączenia", - "database-source/databaseSchemaPlaceholder": "Wprowadź początkowy schemat bazy danych", - "database-source/alert-db-port-invalid-character": "Błąd portu bazy danych: Wprowadzono zabronione znaki. Dozwolone jedynie cyfry.", - "database-source/alert-db-host-invalid-character": "Błąd hosta bazy danych: Wprowadzono zabronione znaki. Dozwolone jedynie znaki alfanumeryczne", - "database-source/alert-conn-name-invalid-character": "Błąd wprowadzenia nazwy połączenia: Wprowadzono zabronione znaki. Dozwolone tylko [a-zA-Z0-9._-]", - "database-source/alert-invalid-query-keyword": "Kwerenda nie może zawierać słowa kluczowego manipulacji danymi:", - "database-source/alert-query": "Musisz określić prawidłowe zapytanie", - "database-source/alert-initial-database": "Musisz określić początkową bazę danych", - "database-source/form-validation-failure": "Błędna nowa forma połączenia!", - "database-source/alert-host": "Musisz określić host bazy danych", - "database-source/alert-port": "Musisz określić numer portu bazy danych", - "database-source/alert-password": "Musisz określić hasło bazy danych", - "database-source/alert-invalid-query-select": "Zapytanie musi zaczynać się od słowa SELECT", - "database-source/alert-user": "Musisz określić użytkownika bazy danych", - "database-source/databaseTestButton": "Testuj", - "database-source/savedConnectionSpan": "Zapisane połączenia", - "database-parsing/updating-preview": "Aktualizacja podglądu…", - "database-source/connectionNameDefaultValue": "Nowa_nazwa_połączenia", - "database-source/databasePasswordPlaceholder": "Wprowadź hasło bazy danych", - "database-source/databaseUserPlaceholder": "Wprowadź nazwę użytkownika bazy danych", - "database-source/databaseFileNameLabel": "Plik bazy danych:", - "database-source/alert-connection-edit": "Połączenie zedytowane", - "database-source/databasePortPlaceholder": "Wprowadź numer portu bazy danych np. 3306", - "database-source/databaseHostPlaceholder": "Wprowadź domenę serwera bazy danych lub IP", - "database-source/connectionNamePlaceholder": "Wprowadź nową nazwę połączenia", - "database-source/databaseSchemaLabel": "Schemat:", - "database-source/databaseFileNamePlaceholder": "Wprowadź pełną ścieżkę do pliku bazy danych", - "database-source/databaseNamePlaceholder": "Wprowadź nazwę bazy danych", - "database-source/databasePortLabel": "Port:", - "database-source/databaseHostLabel": "Host:", - "database-import/checking": "Walidacja zapytania …", - "database-parsing/option": "Opcje", - "database-parsing/ignore-first": "Ignoruj pierwsze", - "database-parsing/worksheet": "Arkusze", - "database-source/alert-connection-name": "Musisz określić nazwę połączenia" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-pt.json b/OpenRefine/extensions/database/module/langs/translation-pt.json deleted file mode 100644 index b8a115473..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-pt.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "database-source/alert-initial-database": "Deve definir um banco de dados inicial", - "database-source/databaseTypeLabel": "Tipo:", - "database-source/alert-host": "Deve definir um host de banco de dados", - "database-source/databasePortLabel": "Porta:", - "database-source/databasePasswordLabel": "Palavra-passe:", - "database-source/databaseSchemaLabel": "Esquema:", - "database-source/databaseTestButton": "Teste", - "database-source/databaseNamePlaceholder": "Digite o nome do banco de dados", - "database-parsing/updating-preview": "Atualizando pré-visualização …", - "database-source/newConnectionButtonDiv": "Nova conexão", - "database-parsing/store-row": "Carregar linhas em branco", - "database-parsing/start-over": "« Recomeçar", - "database-parsing/discard": "linhas(s) de dados", - "database-source/databaseUserPlaceholder": "Digite o utilizador de banco de dados", - "database-parsing/limit": "linhas(s) de dados", - "database-source/alert-db-port-invalid-character": "Erro de porta de banco de dados: caractere ilegal na entrada. Apenas valores numéricos são permitidos.", - "database-parsing/proj-name": "Nome do projeto", - "database-source/alert-query": "Deve definir uma consulta válida", - "database-source/databaseConnectButton": "Conectar", - "database-parsing/limit-next": "Carregar ao menos", - "database-source/connectionNameDefaultValue": "Novo_nome_de_conexão", - "database-parsing/conf-pars": "Configurar opções de análise", - "database-source/databasePortPlaceholder": "Digite a porta do banco de dados, p.e.x 3306", - "database-source/savedConnectionSpan": "Conexões salvas", - "database-source/alert-invalid-query-select": "A consulta deve começar com a palavra-chave SELECT", - "database-source/alert-conn-name-invalid-character": "Erro de entrada de nome de conexão: caractere ilegal na entrada. Apenas [a-zA-Z0-9._-] são permitidos", - "database-import/preparing": "Preparando resultado …", - "database-source/new-connection-legend": "Novo editor de conexão", - "database-source/databaseUserLabel": "Utilizador:", - "database-source/alert-db-host-invalid-character": "Erro de host de banco de dados: caractere ilegal na entrada. Apenas caracteres alfanuméricos são permitidos", - "database-source/databaseSchemaPlaceholder": "Digite o esquema inicial do banco de dados", - "database-parsing/discard-next": "Descartar primeira(s)", - "database-source/databaseNameLabel": "Nome do banco de dados:", - "database-source/alert-port": "Deve definir uma porta de banco de dados", - "database-source/alert-invalid-query-keyword": "A consulta não pode conter palavra-chave de manipulação de dados:", - "database-source/databasePasswordPlaceholder": "Digite a palavra-passe do banco de dados", - "database-source/connectionNamePlaceholder": "Digite um novo nome de conexão", - "database-source/databaseFileNamePlaceholder": "Digite o caminho completo do ficheiro de banco de dados", - "database-parsing/preview-button": "Atualizar Pré-visualização", - "database-source/databaseFileNameLabel": "Ficheiro do banco de dados:", - "database-source/alert-connection-name": "Deve definir um nome de conexão", - "database-import/checking": "Validando consulta …", - "database-source/alert-db-user-invalid-character": "Erro de utilizador de banco de dados: caractere ilegal na entrada. Apenas caracteres alfanuméricos são permitidos", - "database-source/form-validation-failure": "Formulário de nova conexão inválido!", - "database-import/creating": "Criando projeto …", - "database-source/alert-connection-edit": "Conexão editada", - "database-source/databaseHostPlaceholder": "Digite o domínio ou IP do servidor de banco de dados", - "database-parsing/create-proj": "Criar Projeto »", - "database-source/alert-password": "Deve definir uma palavra-passe de banco de dados", - "database-parsing/ignore-first": "Ignorar primeira(s)", - "database-parsing/parse-next": "Analisar próximo", - "database-source/connectionNameLabel": "Nome:", - "database-parsing/option": "Opções", - "database-parsing/worksheet": "Planilhas", - "database-parsing/parse": "linha(s) como nomes das colunas", - "database-parsing/store-cell": "Carregar células em branco como nulas", - "database-source/alert-user": "Deve definir um utilizador de banco de dados", - "database-source/databaseHostLabel": "Host:", - "database-import/title": "Servidores de banco de dados", - "database-source/databaseSaveButton": "Gravar", - "database-parsing/ignore": "linhas(s) no começo do ficheiro", - "database-import/importer-name": "Banco de dados" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-pt_BR.json b/OpenRefine/extensions/database/module/langs/translation-pt_BR.json deleted file mode 100644 index b11fd33ac..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-pt_BR.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "database-source/databaseNamePlaceholder": "Digite o nome do banco de dados", - "database-source/alert-db-port-invalid-character": "Erro de porta de banco de dados: caractere ilegal na entrada. Apenas valores numéricos são permitidos.", - "database-source/alert-db-user-invalid-character": "Erro de usuário de banco de dados: caractere ilegal na entrada. Apenas caracteres alfanuméricos são permitidos", - "database-source/alert-db-host-invalid-character": "Erro de host de banco de dados: caractere ilegal na entrada. Apenas caracteres alfanuméricos são permitidos", - "database-source/alert-conn-name-invalid-character": "Erro de entrada de nome de conexão: caractere ilegal na entrada. Apenas [a-zA-Z0-9._-] são permitidos", - "database-parsing/store-cell": "Carregar células em branco como nulas", - "database-parsing/store-row": "Carregar linhas em branco", - "database-parsing/limit": "linhas(s) de dados", - "database-parsing/limit-next": "Carregar ao menos", - "database-parsing/discard": "linhas(s) de dados", - "database-parsing/discard-next": "Descartar primeira(s)", - "database-parsing/parse": "linha(s) como nomes das colunas", - "database-parsing/parse-next": "Analisar próximo", - "database-parsing/ignore": "linhas(s) no começo do arquivo", - "database-parsing/create-proj": "Criar Projeto »", - "database-parsing/ignore-first": "Ignorar primeira(s)", - "database-parsing/preview-button": "Atualizar Pré-visualização", - "database-parsing/option": "Opções", - "database-parsing/worksheet": "Planilhas", - "database-parsing/updating-preview": "Atualizando pré-visualização …", - "database-parsing/proj-name": "Nome do projeto", - "database-parsing/conf-pars": "Configurar opções de análise", - "database-parsing/start-over": "« Recomeçar", - "database-source/savedConnectionSpan": "Conexões salvas", - "database-source/newConnectionButtonDiv": "Nova conexão", - "database-source/databaseConnectButton": "Conectar", - "database-source/databaseSaveButton": "Salvar", - "database-source/databaseTestButton": "Teste", - "database-source/connectionNameDefaultValue": "Novo_nome_de_conexão", - "database-source/databaseSchemaPlaceholder": "Digite o esquema inicial do banco de dados", - "database-source/databaseFileNamePlaceholder": "Digite o caminho completo do arquivo de banco de dados", - "database-source/databasePasswordPlaceholder": "Digite a senha do banco de dados", - "database-source/databaseUserPlaceholder": "Digite o usuário de banco de dados", - "database-source/databasePortPlaceholder": "Digite a porta do banco de dados, p.e.x 3306", - "database-source/databaseHostPlaceholder": "Digite o domínio ou IP do servidor de banco de dados", - "database-source/connectionNamePlaceholder": "Digite um novo nome de conexão", - "database-source/databaseSchemaLabel": "Esquema:", - "database-source/databaseFileNameLabel": "Arquivo do banco de dados:", - "database-source/databaseNameLabel": "Nome do banco de dados:", - "database-source/databasePasswordLabel": "Senha:", - "database-source/databaseUserLabel": "Usuário:", - "database-source/databasePortLabel": "Porta:", - "database-source/databaseHostLabel": "Host:", - "database-source/databaseTypeLabel": "Tipo:", - "database-source/connectionNameLabel": "Nome:", - "database-source/alert-connection-edit": "Conexão editada", - "database-source/form-validation-failure": "Formulário de nova conexão inválido!", - "database-source/alert-invalid-query-select": "A consulta deve começar com a palavra-chave SELECT", - "database-source/alert-invalid-query-keyword": "A consulta não pode conter palavra-chave de manipulação de dados:", - "database-source/alert-query": "Você deve definir uma consulta válida", - "database-source/alert-initial-database": "Você deve definir um banco de dados inicial", - "database-source/alert-connection-name": "Você deve definir um nome de conexão", - "database-source/alert-password": "Você deve definir uma senha de banco de dados", - "database-source/alert-port": "Você deve definir uma porta de banco de dados", - "database-source/alert-host": "Você deve definir um host de banco de dados", - "database-source/alert-user": "Você deve definir um usuário de banco de dados", - "database-import/creating": "Criando projeto …", - "database-import/checking": "Validando consulta …", - "database-import/preparing": "Preparando resultado …", - "database-import/title": "Servidores de banco de dados", - "database-source/new-connection-legend": "Novo editor de conexão" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-sv.json b/OpenRefine/extensions/database/module/langs/translation-sv.json deleted file mode 100644 index 7f2f3738c..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-sv.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "database-import/title": "Databasservrar", - "database-import/preparing": "Förbereder resultat …", - "database-import/checking": "Validerar fråga …", - "database-import/creating": "Skapar projekt …", - "database-source/alert-host": "Du måste ange en värd för databasen", - "database-source/alert-port": "Du måste ange en port för databasen", - "database-source/alert-user": "Du måste ange en användare för databasen", - "database-source/alert-password": "Du måste ange ett lösenord för databasen", - "database-source/alert-connection-name": "Du måste ange ett anslutningsnamn", - "database-source/alert-initial-database": "Du måste ange en första databas", - "database-source/alert-query": "Du måste ange en giltig fråga", - "database-source/alert-invalid-query-keyword": "Fråga kan inte innehålla nyckelord för datamanipulering:", - "database-source/alert-invalid-query-select": "Frågan måste börja med nyckelordet SELECT", - "database-source/form-validation-failure": "Ny anslutningsform är ogiltig!", - "database-source/alert-connection-edit": "Anslutning redigerad", - "database-source/connectionNameLabel": "Namn:", - "database-source/databaseTypeLabel": "Typ:", - "database-source/databaseHostLabel": "Värd:", - "database-source/databasePortLabel": "Port:", - "database-source/databaseUserLabel": "Användare:", - "database-source/databasePasswordLabel": "Lösenord:", - "database-source/databaseNameLabel": "Databas:", - "database-source/databaseFileNameLabel": "Databas file:", - "database-source/databaseSchemaLabel": "Schema:", - "database-source/databaseTestButton": "Test", - "database-source/databaseSaveButton": "Spara", - "database-source/databaseConnectButton": "Anslut", - "database-source/newConnectionButtonDiv": "Ny anslutning", - "database-source/savedConnectionSpan": "Sparade anslutningar", - "database-parsing/start-over": "« Börja om", - "database-parsing/conf-pars": "Konfigurera parsningsalternativ", - "database-parsing/proj-name": "Projektnamn", - "database-parsing/create-proj": "Skapa projekt »", - "database-parsing/updating-preview": "Uppdaterar förhandsgranskning …", - "database-parsing/worksheet": "kalkylblad", - "database-parsing/option": "Inställningar", - "database-parsing/preview-button": "Uppdatera förhandsgranskning", - "database-parsing/ignore-first": "Ignorera första", - "database-parsing/ignore": "rad(er) i början av filen", - "database-parsing/parse-next": "Tolka nästa", - "database-parsing/parse": "rad(er) som kolumnrubriker", - "database-parsing/discard-next": "Släng första", - "database-parsing/discard": "rad(er) av data", - "database-parsing/limit-next": "Ladda max", - "database-parsing/limit": "rad(er) med data", - "database-parsing/store-row": "Spara tomma rader", - "database-parsing/store-cell": "Spara tomma celler som nullvärden", - "database-source/connectionNameDefaultValue": "Nytt namn på kopplingen", - "database-source/databaseSchemaPlaceholder": "Mata in det initiala databas-schemat", - "database-source/databaseFileNamePlaceholder": "Mata in databasfilens fullständiga adress", - "database-source/databaseNamePlaceholder": "Mata in databasens namn", - "database-source/databasePasswordPlaceholder": "Mata in databas lösenord", - "database-source/databaseUserPlaceholder": "Mata in databas användarnamn", - "database-source/databasePortPlaceholder": "Mata in databas-port, ex.vis 3306", - "database-source/databaseHostPlaceholder": "Mata in databas-domän eller IP-adress", - "database-source/connectionNamePlaceholder": "Mata in ett nytt namn på kopplingen", - "database-import/importer-name": "Databas" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-tr.json b/OpenRefine/extensions/database/module/langs/translation-tr.json deleted file mode 100644 index f17804c1c..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-tr.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "database-import/title": "Veri Tabanı Sunucuları", - "database-import/preparing": "Sonuç hazırlanıyor…", - "database-import/checking": "Sorgu doğrulanıyor…", - "database-import/creating": "Proje oluşturuluyor…", - "database-source/alert-db-port-invalid-character": "Veri Tabanı Bağlantı Noktası Hatası: Girişte Geçersiz Karakter. Yalnızca rakamlara izin verilir.", - "database-source/alert-db-user-invalid-character": "Veri Tabanı Kullanıcısı Hatası: Girişte Geçersiz Karakter. Yalnızca harflere ve rakamlara izin verilir", - "database-source/connectionNameDefaultValue": "Yeni_Baglanti_Adi", - "database-source/alert-db-host-invalid-character": "Veri Tabanı Sunucusu Hatası: Girişte Geçersiz Karakter. Yalnızca harflere ve rakamlara izin verilir", - "database-source/alert-conn-name-invalid-character": "Bağlantı Adı Giriş Hatası: Girişte Geçersiz Karakter. Yalnızca [a-zA-Z0-9._-] Karakterlerine İzin Verilir", - "database-parsing/store-cell": "Boş hücreleri boş (null) olarak kaydet", - "database-parsing/store-row": "Boş satırları kaydet", - "database-parsing/limit": "veri satırı", - "database-parsing/limit-next": "Yükle, en fazla", - "database-parsing/discard": "veri satırını", - "database-parsing/parse": "satırı, sütun başlığı olarak", - "database-parsing/ignore": "satırı, dosyanın başındaki", - "database-parsing/discard-next": "At, ilk", - "database-parsing/parse-next": "Ayrıştır, sonraki", - "database-parsing/ignore-first": "Yok say, ilk", - "database-parsing/preview-button": "Ön izlemeyi güncelle", - "database-parsing/option": "Seçenekler", - "database-parsing/worksheet": "Çalışma sayfaları", - "database-parsing/updating-preview": "Ön izleme güncelleniyor…", - "database-parsing/create-proj": "Proje Oluştur »", - "database-parsing/proj-name": "Proje adı", - "database-parsing/conf-pars": "Ayrıştırma seçeneklerini yapılandır", - "database-parsing/start-over": "« Baştan başla", - "database-source/savedConnectionSpan": "Kaydedilen bağlantılar", - "database-source/new-connection-legend": "Yeni Bağlantı Düzenleyici", - "database-source/newConnectionButtonDiv": "Yeni bağlantı", - "database-source/databaseConnectButton": "Bağlan", - "database-source/databaseSaveButton": "Kaydet", - "database-source/databaseTestButton": "Test et", - "database-source/databaseSchemaPlaceholder": "Veri tabanının başlangıç şemasını girin", - "database-source/databaseFileNamePlaceholder": "Veri tabanı dosyasının tam yolunu girin", - "database-source/databaseNamePlaceholder": "Veri tabanının adını girin", - "database-source/databasePasswordPlaceholder": "Veri tabanı parolasını girin", - "database-source/databaseUserPlaceholder": "Veri tabanı kullanıcısını girin", - "database-source/databasePortPlaceholder": "Veri tabanı bağlantı noktasını girin, örn. 3306", - "database-source/databaseHostPlaceholder": "Veri tabanı sunucusu etki alanını veya IP adresini girin", - "database-source/connectionNamePlaceholder": "Yeni bir bağlantı adı girin", - "database-source/databaseSchemaLabel": "Şema:", - "database-source/databaseFileNameLabel": "Veri tabanı dosyası:", - "database-source/databaseNameLabel": "Veri tabanı adı:", - "database-source/databasePasswordLabel": "Parola:", - "database-source/databaseUserLabel": "Kullanıcı:", - "database-source/databasePortLabel": "Bağlantı noktası:", - "database-source/databaseHostLabel": "Sunucu:", - "database-source/databaseTypeLabel": "Tür:", - "database-source/connectionNameLabel": "Ad:", - "database-source/alert-connection-edit": "Bağlantı düzenlendi", - "database-source/form-validation-failure": "Yeni bağlantı formu geçersiz!", - "database-source/alert-invalid-query-select": "Sorgu, SELECT anahtar sözcüğüyle başlamalıdır", - "database-source/alert-invalid-query-keyword": "Sorgu, veri işleme anahtar sözcüğü içeremez:", - "database-source/alert-query": "Geçerli bir sorgu belirtmelisiniz", - "database-source/alert-initial-database": "Bir başlangıç veri tabanı belirtmelisiniz", - "database-source/alert-connection-name": "Bir bağlantı adı belirtmelisiniz", - "database-source/alert-password": "Bir veri tabanı parolası belirtmelisiniz", - "database-source/alert-user": "Bir veri tabanı kullanıcısı belirtmelisiniz", - "database-source/alert-port": "Bir veri tabanı bağlantı noktası belirtmelisiniz", - "database-source/alert-host": "Bir veri tabanı sunucusu belirtmelisiniz", - "database-import/importer-name": "Veri tabanı", - "database-parsing/disable-auto-preview": "Otomatik ön izlemeyi devre dışı bırak" -} diff --git a/OpenRefine/extensions/database/module/langs/translation-zh_Hans.json b/OpenRefine/extensions/database/module/langs/translation-zh_Hans.json deleted file mode 100644 index 887e2f9fe..000000000 --- a/OpenRefine/extensions/database/module/langs/translation-zh_Hans.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "database-import/title": "数据库服务器", - "database-import/preparing": "正在准备返回结果…", - "database-import/checking": "正在验证查询…", - "database-import/creating": "创建项目…", - "database-source/alert-host": "你必须指定一个数据库主机", - "database-source/alert-port": "你必须指定一个数据库端口", - "database-source/alert-user": "你必须指定一个数据库用户", - "database-source/alert-password": "你必须设定一个数据库密码", - "database-source/alert-connection-name": "你必须指定一个连接名称", - "database-source/alert-initial-database": "你必须指定一个初始数据库", - "database-source/alert-query": "你必须指定一个有效的查询", - "database-source/alert-invalid-query-keyword": "任何数据库查询语句均不能够包含数据操作关键字:", - "database-source/alert-invalid-query-select": "查询语句必须从 SELECT 关键字开始", - "database-source/form-validation-failure": "无效的新建连接表!", - "database-source/alert-connection-edit": "连接已编辑", - "database-source/connectionNameLabel": "名称:", - "database-source/databaseTypeLabel": "类型:", - "database-source/databaseHostLabel": "主机:", - "database-source/databasePortLabel": "端口:" -} diff --git a/OpenRefine/extensions/database/module/macros.vm b/OpenRefine/extensions/database/module/macros.vm deleted file mode 100644 index 18a83bfce..000000000 --- a/OpenRefine/extensions/database/module/macros.vm +++ /dev/null @@ -1,14 +0,0 @@ -#* - This file contains common velocity macros used in all .vt files. - For Velocity documentation, see: - - http://velocity.apache.org/engine/releases/velocity-1.5/user-guide.html -*# - -#macro( makeAList $list ) -
    - #foreach($item in $list) -
  • $item
  • - #end -
-#end \ No newline at end of file diff --git a/OpenRefine/extensions/database/module/scripts/database-extension.js b/OpenRefine/extensions/database/module/scripts/database-extension.js deleted file mode 100644 index db871124e..000000000 --- a/OpenRefine/extensions/database/module/scripts/database-extension.js +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -var DatabaseExtension = {}; - -DatabaseExtension.currentConnection = {}; - -DatabaseExtension.handleSavedConnectionClicked = function(menuKey, connectionName) { - var jdbcConnectionInfo = {}; - jdbcConnectionInfo.connectionName = connectionName; - - if(menuKey === "edit"){ - DatabaseExtension.handleEditConnectionClicked(connectionName); - - }else if(menuKey === "delete"){ - DatabaseExtension.handleDeleteConnectionClicked(connectionName); - - }else if(menuKey === "connect"){ - DatabaseExtension.handleConnectClicked(connectionName); - } - -}; - - -DatabaseExtension.handleConnectClicked = function(connectionName) { - - $.get( - "command/database/saved-connection" + '?' + $.param({"connectionName": connectionName}), - null, - - function(savedDatabaseConfig) { - - if(savedDatabaseConfig){ - - var savedConfig = savedDatabaseConfig.savedConnections[0]; - var databaseConfig = {}; - databaseConfig.connectionName = savedConfig.connectionName; - databaseConfig.databaseType = savedConfig.databaseType; - databaseConfig.databaseServer = savedConfig.databaseHost; - databaseConfig.databasePort = savedConfig.databasePort; - databaseConfig.databaseUser = savedConfig.databaseUser; - databaseConfig.databasePassword = savedConfig.databasePassword; - databaseConfig.initialDatabase = savedConfig.databaseName; - databaseConfig.initialSchema = savedConfig.databaseSchema; - - Refine.postCSRF( - "command/database/connect", - databaseConfig, - - function(databaseInfo) { - - if(databaseInfo){ - DatabaseExtension.currentConnection.databaseInfo; - $( "#currentConnectionNameInput" ).val(databaseConfig.connectionName); - $( "#currentDatabaseTypeInput" ).val(databaseConfig.databaseType); - $( "#currentDatabaseUserInput" ).val(databaseConfig.databaseUser); - $( "#currentDatabasePasswordInput" ).val(databaseConfig.databasePassword); - $( "#currentDatabaseHostInput" ).val(databaseConfig.databaseServer); - $( "#currentDatabasePortInput" ).val(databaseConfig.databasePort); - $( "#currentInitialDatabaseInput" ).val(databaseConfig.initialDatabase); - - var connectionParam = "Connection[" + databaseConfig.connectionName + "] :: " - + "jdbc:" - + databaseConfig.databaseType + "://" - + databaseConfig.databaseServer + ":" - + databaseConfig.databasePort + "/" - + databaseConfig.initialDatabase; - - $( "#connectionParameterSpan" ).text(connectionParam); - $( "#newConnectionDiv" ).hide(); - $( "#sqlEditorDiv" ).show(); - - }else{ - window.alert("Unable to establish connection to database"); - } - - }, - "json", - function( jqXhr, textStatus, errorThrown ){ - alert( textStatus + ':' + errorThrown ); - }); - - } - - }, - "json" - ); -}; - -DatabaseExtension.handleDeleteConnectionClicked = function(connectionName) { - $.ajax({ - url: 'command/database/saved-connection' + '?' + $.param({"connectionName": connectionName}), - type: 'DELETE', - contentType:'application/json', - data: null, - success: function(settings) { - if(settings){ - - $( "#menuListUl" ).empty(); - var items = []; - $.each(settings.savedConnections,function(index,savedConnection){ - items.push('
  • ' - + '' + savedConnection.connectionName + '' - + '
  • '); - }) - - $( "#menuListUl" ).append(items.join('')); - } - } - }).fail(function( jqXhr, textStatus, errorThrown ){ - alert( textStatus + ':' + errorThrown ); - }); -} - -DatabaseExtension.handleEditConnectionClicked = function(connectionName) { - $.get( - "command/database/saved-connection" + '?' + $.param({ "connectionName": connectionName }), - null, - function(savedDatabaseConfig) { - if (savedDatabaseConfig) { - var savedConfig = savedDatabaseConfig.savedConnections[0]; - - $( "#connectionName" ).val(savedConfig.connectionName); - $( "select#databaseTypeSelect" ).val(savedConfig.databaseType); - Refine.DatabaseSourceUI.prototype._updateDatabaseType(savedConfig.databaseType); - - $( "#databaseHost" ).val(savedConfig.databaseHost); - $( "#databasePort" ).val(savedConfig.databasePort); - $( "#databaseUser" ).val(savedConfig.databaseUser); - $( "#databasePassword" ).val(savedConfig.databasePassword); - $( "#initialDatabase" ).val(savedConfig.databaseName); - $( "#initialSchema" ).val(savedConfig.databaseSchema); - $( "#newConnectionControlDiv" ).hide(); - $( "#editConnectionControlDiv" ).show(); - $( "#newConnectionDiv" ).show(); - $('#sqlEditorDiv').hide(); - $("#connectionName").attr('readonly', 'readonly'); - } - }, - "json" - ); -} diff --git a/OpenRefine/extensions/database/module/scripts/index/database-import-controller.js b/OpenRefine/extensions/database/module/scripts/index/database-import-controller.js deleted file mode 100644 index 42b2779db..000000000 --- a/OpenRefine/extensions/database/module/scripts/index/database-import-controller.js +++ /dev/null @@ -1,399 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -//Internationalization init -var lang = navigator.language.split("-")[0] - || navigator.userLanguage.split("-")[0]; -var dictionary = ""; -$.ajax({ - url : "command/core/load-language?", - type : "POST", - async : false, - data : { - module : "database", - }, - success : function(data) { - dictionary = data['dictionary']; - lang = data['lang']; - } -}); -$.i18n().load(dictionary, lang); -// End internationalization - -Refine.DatabaseImportController = function(createProjectUI) { - this._createProjectUI = createProjectUI; - - this._parsingPanel = createProjectUI.addCustomPanel(); - - createProjectUI.addSourceSelectionUI({ - label: $.i18n('database-import/importer-name'), - id: "database-source", - ui: new Refine.DatabaseSourceUI(this) - }); - -}; -Refine.CreateProjectUI.controllers.push(Refine.DatabaseImportController); - -Refine.DatabaseImportController.prototype.startImportingDocument = function(queryInfo) { - var dismiss = DialogSystem.showBusy($.i18n('database-import/preparing')); - //alert(queryInfo.query); - var self = this; - - Refine.postCSRF( - "command/core/create-importing-job", - null, - function(data) { - Refine.wrapCSRF(function(token) { - $.post( - "command/core/importing-controller?" + $.param({ - "controller": "database/database-import-controller", - "subCommand": "initialize-parser-ui", - "csrf_token": token - }), - queryInfo, - - function(data2) { - dismiss(); - - if (data2.status == 'ok') { - self._queryInfo = queryInfo; - self._jobID = data.jobID; - self._options = data2.options; - - self._showParsingPanel(); - - } else { - alert(data2.message); - } - }, - "json" - ); - }); - }, - "json" - ); -}; - -Refine.DatabaseImportController.prototype.getOptions = function() { - var options = { - - }; - - var parseIntDefault = function(s, def) { - try { - var n = parseInt(s); - if (!isNaN(n)) { - return n; - } - } catch (e) { - // Ignore - } - return def; - }; - - - if (this._parsingPanelElmts.skipCheckbox[0].checked) { - options.skipDataLines = parseIntDefault(this._parsingPanelElmts.skipInput[0].value, 0); - } else { - options.skipDataLines = 0; - } - if (this._parsingPanelElmts.limitCheckbox[0].checked) { - options.limit = parseIntDefault(this._parsingPanelElmts.limitInput[0].value, -1); - } else { - options.limit = -1; - } - options.storeBlankRows = this._parsingPanelElmts.storeBlankRowsCheckbox[0].checked; - options.storeBlankCellsAsNulls = this._parsingPanelElmts.storeBlankCellsAsNullsCheckbox[0].checked; - - options.disableAutoPreview = this._parsingPanelElmts.disableAutoPreviewCheckbox[0].checked; - - return options; -}; - -Refine.DatabaseImportController.prototype._showParsingPanel = function() { - var self = this; - - this._parsingPanel.unbind().empty().html( - DOM.loadHTML("database",'scripts/index/database-parsing-panel.html')); - - this._parsingPanelElmts = DOM.bind(this._parsingPanel); - - this._parsingPanelElmts.startOverButton.html($.i18n('database-parsing/start-over')); - this._parsingPanelElmts.database_conf_pars.html($.i18n('database-parsing/conf-pars')); - this._parsingPanelElmts.database_proj_name.html($.i18n('database-parsing/proj-name')); - this._parsingPanelElmts.createProjectButton.html($.i18n('database-parsing/create-proj')); - this._parsingPanelElmts.database_options.html($.i18n('database-parsing/option')); - this._parsingPanelElmts.previewButton.html($.i18n('database-parsing/preview-button')); - this._parsingPanelElmts.database_updating.html($.i18n('database-parsing/updating-preview')); - this._parsingPanelElmts.database_discard_next.html($.i18n('database-parsing/discard-next')); - this._parsingPanelElmts.database_discard.html($.i18n('database-parsing/discard')); - this._parsingPanelElmts.database_limit_next.html($.i18n('database-parsing/limit-next')); - this._parsingPanelElmts.database_limit.html($.i18n('database-parsing/limit')); - this._parsingPanelElmts.database_store_row.html($.i18n('database-parsing/store-row')); - this._parsingPanelElmts.database_store_cell.html($.i18n('database-parsing/store-cell')); - this._parsingPanelElmts.database_disable_auto_preview.text($.i18n('database-parsing/disable-auto-preview')); - - if (this._parsingPanelResizer) { - $(window).unbind('resize', this._parsingPanelResizer); - } - - this._parsingPanelResizer = function() { - var elmts = self._parsingPanelElmts; - var width = self._parsingPanel.width(); - var height = self._parsingPanel.height(); - var headerHeight = elmts.wizardHeader.outerHeight(true); - var controlPanelHeight = 250; - - elmts.dataPanel - .css("left", "0px") - .css("top", headerHeight + "px") - .css("width", (width - DOM.getHPaddings(elmts.dataPanel)) + "px") - .css("height", (height - headerHeight - controlPanelHeight - DOM.getVPaddings(elmts.dataPanel)) + "px"); - elmts.progressPanel - .css("left", "0px") - .css("top", headerHeight + "px") - .css("width", (width - DOM.getHPaddings(elmts.progressPanel)) + "px") - .css("height", (height - headerHeight - controlPanelHeight - DOM.getVPaddings(elmts.progressPanel)) + "px"); - - elmts.controlPanel - .css("left", "0px") - .css("top", (height - controlPanelHeight) + "px") - .css("width", (width - DOM.getHPaddings(elmts.controlPanel)) + "px") - .css("height", (controlPanelHeight - DOM.getVPaddings(elmts.controlPanel)) + "px"); - }; - - $(window).resize(this._parsingPanelResizer); - this._parsingPanelResizer(); - - this._parsingPanelElmts.startOverButton.click(function() { - // explicitly cancel the import job - Refine.CreateProjectUI.cancelImportingJob(self._jobID); - - delete self._jobID; - delete self._options; - - self._createProjectUI.showSourceSelectionPanel(); - }); - - this._parsingPanelElmts.createProjectButton.click(function() { self._createProject(); }); - this._parsingPanelElmts.previewButton.click(function() { self._updatePreview(); }); - //alert("datetime::" + $.now()); - //this._parsingPanelElmts.projectNameInput[0].value = this._queryInfo.connectionName + "_" + this._queryInfo.databaseUser + "_" + $.now(); - this._parsingPanelElmts.projectNameInput[0].value = this._queryInfo.databaseServer + "_" + this._queryInfo.initialDatabase + "_" + $.now(); - - - if (this._options.limit > 0) { - this._parsingPanelElmts.limitCheckbox.prop("checked", true); - this._parsingPanelElmts.limitInput[0].value = this._options.limit.toString(); - } - if (this._options.skipDataLines > 0) { - this._parsingPanelElmts.skipCheckbox.prop("checked", true); - this._parsingPanelElmts.skipInput.value[0].value = this._options.skipDataLines.toString(); - } - if (this._options.storeBlankRows) { - this._parsingPanelElmts.storeBlankRowsCheckbox.prop("checked", true); - } - if (this._options.storeBlankCellsAsNulls) { - this._parsingPanelElmts.storeBlankCellsAsNullsCheckbox.prop("checked", true); - } - - if (this._options.disableAutoPreview) { - this._parsingPanelElmts.disableAutoPreviewCheckbox.prop('checked', true); - } - - // If disableAutoPreviewCheckbox is not checked, we will schedule an automatic update - var onChange = function() { - if (!self._parsingPanelElmts.disableAutoPreviewCheckbox[0].checked) - { - self._scheduleUpdatePreview(); - } - }; - this._parsingPanel.find("input").bind("change", onChange); - this._parsingPanel.find("select").bind("change", onChange); - - this._createProjectUI.showCustomPanel(this._parsingPanel); - this._updatePreview(); -}; - -Refine.DatabaseImportController.prototype._scheduleUpdatePreview = function() { - if (this._timerID != null) { - window.clearTimeout(this._timerID); - this._timerID = null; - } - - var self = this; - this._timerID = window.setTimeout(function() { - self._timerID = null; - self._updatePreview(); - }, 500); // 0.5 second - }; - -Refine.DatabaseImportController.prototype._updatePreview = function() { - var self = this; - // alert("query::" + this._queryInfo.query); - this._parsingPanelElmts.dataPanel.hide(); - this._parsingPanelElmts.progressPanel.show(); - this._queryInfo.options = JSON.stringify(this.getOptions()); - //alert("options:" + this._queryInfo.options); - - Refine.wrapCSRF(function(token) { - $.post( - "command/core/importing-controller?" + $.param({ - "controller": "database/database-import-controller", - "jobID": self._jobID, - "subCommand": "parse-preview", - "csrf_token": token - }), - - self._queryInfo, - - function(result) { - if (result.status == "ok") { - self._getPreviewData(function(projectData) { - self._parsingPanelElmts.progressPanel.hide(); - self._parsingPanelElmts.dataPanel.show(); - - new Refine.PreviewTable(projectData, self._parsingPanelElmts.dataPanel.unbind().empty()); - }); - } else { - - alert('Errors:\n' + (result.message) ? result.message : Refine.CreateProjectUI.composeErrorMessage(job)); - self._parsingPanelElmts.progressPanel.hide(); - - Refine.CreateProjectUI.cancelImportingJob(self._jobID); - - delete self._jobID; - delete self._options; - - self._createProjectUI.showSourceSelectionPanel(); - - - } - }, - "json" - ); - }); - }; - -Refine.DatabaseImportController.prototype._getPreviewData = function(callback, numRows) { - var self = this; - var result = {}; - - $.post( - "command/core/get-models?" + $.param({ "importingJobID" : this._jobID }), - null, - function(data) { - for (var n in data) { - if (data.hasOwnProperty(n)) { - result[n] = data[n]; - } - } - - $.post( - "command/core/get-rows?" + $.param({ - "importingJobID" : self._jobID, - "start" : 0, - "limit" : numRows || 100 // More than we parse for preview anyway - }), - null, - function(data) { - result.rowModel = data; - callback(result); - }, - "jsonp" - ); - }, - "json" - ); -}; - -Refine.DatabaseImportController.prototype._createProject = function() { - var projectName = $.trim(this._parsingPanelElmts.projectNameInput[0].value); - if (projectName.length == 0) { - window.alert("Please name the project."); - this._parsingPanelElmts.projectNameInput.focus(); - return; - } - - var self = this; - var options = this.getOptions(); - options.projectName = projectName; - - this._queryInfo.options = JSON.stringify(options); - Refine.wrapCSRF(function(token) { - $.post( - "command/core/importing-controller?" + $.param({ - "controller": "database/database-import-controller", - "jobID": self._jobID, - "subCommand": "create-project", - "csrf_token": token - }), - self._queryInfo, - function(o) { - if (o.status == 'error') { - alert(o.message); - } else { - var start = new Date(); - var timerID = window.setInterval( - function() { - self._createProjectUI.pollImportJob( - start, - self._jobID, - timerID, - function(job) { - return "projectID" in job.config; - }, - function(jobID, job) { - //alert("jobID::" + jobID + " job :" + job); - window.clearInterval(timerID); - Refine.CreateProjectUI.cancelImportingJob(jobID); - document.location = "project?project=" + job.config.projectID; - }, - function(job) { - alert(Refine.CreateProjectUI.composeErrorMessage(job)); - } - ); - }, - 1000 - ); - self._createProjectUI.showImportProgressPanel($.i18n('database-import/creating'), function() { - // stop the timed polling - window.clearInterval(timerID); - - // explicitly cancel the import job - Refine.CreateProjectUI.cancelImportingJob(jobID); - - self._createProjectUI.showSourceSelectionPanel(); - }); - } - }, - "json" - ); - }); -}; \ No newline at end of file diff --git a/OpenRefine/extensions/database/module/scripts/index/database-import-form.html b/OpenRefine/extensions/database/module/scripts/index/database-import-form.html deleted file mode 100644 index cac9556e3..000000000 --- a/OpenRefine/extensions/database/module/scripts/index/database-import-form.html +++ /dev/null @@ -1,142 +0,0 @@ -
    - -
    - -
    -
    -
    -
    -
    - -
    - Saved Connections - -
    - -
    -
    - -
    - -
    - - -
    -
    - - -
    - - -
    - -
    - - -
    - -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    - -
    - - - -
    - - - -
    -
    - - - - - -
    - - - -
    - -
    - -
    - - -
    - \ No newline at end of file diff --git a/OpenRefine/extensions/database/module/scripts/index/database-parsing-panel.html b/OpenRefine/extensions/database/module/scripts/index/database-parsing-panel.html deleted file mode 100644 index b909a158e..000000000 --- a/OpenRefine/extensions/database/module/scripts/index/database-parsing-panel.html +++ /dev/null @@ -1,58 +0,0 @@ -
    -
    - - - - - - - - -
    -
    -
    -
    -
    - - -
    -
    -
    - - - - - - - - - - - - -
      -
    -
    - - - - - - - - - - - - - - - - -
    -
    -
    -
    -
    -
    -
    \ No newline at end of file diff --git a/OpenRefine/extensions/database/module/scripts/index/database-source-ui.js b/OpenRefine/extensions/database/module/scripts/index/database-source-ui.js deleted file mode 100644 index f34be3f2d..000000000 --- a/OpenRefine/extensions/database/module/scripts/index/database-source-ui.js +++ /dev/null @@ -1,528 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -$(function(){ - $.contextMenu({ - selector: '.context-menu-one', - trigger: 'left', - build: function($trigger, e) { - - return { - callback: function(key, options) { - var m = "clicked: " + key; - DatabaseExtension.handleSavedConnectionClicked(key, $(this).text()); - - }, - - items: { - "edit": {name: " Edit "}, - "sep0": "", - "delete": {name: " Delete "}, - "sep1": "---------", - "connect": {name: " Connect "}, - "dummy": {name: "", icon: ""} - } - }; - } - }); -}); - -Refine.DatabaseSourceUI = function(controller) { - this._controller = controller; -}; - -Refine.DatabaseSourceUI.prototype.attachUI = function(body) { - this._body = body; - - this._body.html(DOM.loadHTML("database", "scripts/index/database-import-form.html")); - this._elmts = DOM.bind(this._body); - var self = this; - - self._defaultDatabaseHost = "localhost"; - self._defaultDatabaseType = $( "select#databaseTypeSelect" ).val(); - if (self._defaultDatabaseType == "") { self._defaultDatabaseType = "mysql"; } - - $('#database-title').text($.i18n('database-import/title')); - $('#new-connection-legend').text($.i18n('database-source/new-connection-legend')); - $('#connectionNameLabel').html($.i18n('database-source/connectionNameLabel')); - $('#databaseTypeLabel').html($.i18n('database-source/databaseTypeLabel')); - $('#databaseHostLabel').text($.i18n('database-source/databaseHostLabel')); - $('#databasePortLabel').text($.i18n('database-source/databasePortLabel')); - $('#databaseUserLabel').text($.i18n('database-source/databaseUserLabel')); - $('#databasePasswordLabel').text($.i18n('database-source/databasePasswordLabel')); - $('#databaseNameLabel').text($.i18n('database-source/databaseNameLabel')); - $('#databaseSchemaLabel').text($.i18n('database-source/databaseSchemaLabel')); - $('#databaseTestButton').text($.i18n('database-source/databaseTestButton')); - $('#databaseSaveButton').text($.i18n('database-source/databaseSaveButton')); - $('#databaseConnectButton').text($.i18n('database-source/databaseConnectButton')); - $('#newConnectionButtonDiv').text($.i18n('database-source/newConnectionButtonDiv')); - $('#savedConnectionSpan').text($.i18n('database-source/savedConnectionSpan')); - - $('input#connectionName').attr('placeholder', $.i18n('database-source/connectionNamePlaceholder')); - $('input#databaseHost').attr('placeholder', $.i18n('database-source/databaseHostPlaceholder')); - $('input#databasePort').attr('placeholder', $.i18n('database-source/databasePortPlaceholder')); - $('input#databaseUser').attr('placeholder', $.i18n('database-source/databaseUserPlaceholder')); - $('input#databasePassword').attr('placeholder', $.i18n('database-source/databasePasswordPlaceholder')); - $('input#initialDatabase').attr('placeholder', $.i18n('database-source/databaseNamePlaceholder')); - $('input#initialSchema').attr('placeholder', $.i18n('database-source/databaseSchemaPlaceholder')); - - $('input#connectionName').val($.i18n('database-source/connectionNameDefaultValue')); - - this._elmts.newConnectionButton.click(function(evt) { - self._resetDatabaseImportForm(); - $( "#newConnectionDiv" ).show(); - $( "#sqlEditorDiv" ).hide(); - // self._body.find('.newConnectionDiv').show(); - // self._body.find('.sqlEditorDiv').hide(); - }); - - this._elmts.databaseTypeSelect.change(function(event) { - var type = $( "select#databaseTypeSelect" ).val(); - - self._updateDatabaseType(type); - }); - - var defaultDatabase = $( "select#databaseTypeSelect" ).val(); - self._updateDatabaseType(defaultDatabase); - - this._elmts.testDatabaseButton.click(function(evt) { - - if(self._validateNewConnectionForm() === true){ - self._testDatabaseConnect(self._getConnectionInfo()); - } - - }); - - this._elmts.databaseConnectButton.click(function(evt) { - - if(self._validateNewConnectionForm() === true){ - self._connect(self._getConnectionInfo()); - } - - - }); - - this._elmts.saveConnectionButton.click(function(evt) { - - if(self._validateNewConnectionForm() == true){ - var connectionNameInput = $.trim(self._elmts.connectionNameInput[0].value); - if (connectionNameInput.length === 0) { - window.alert($.i18n('database-source/alert-connection-name')); - } else{ - self._saveConnection(self._getConnectionInfo()); - } - - } - - }); - - this._elmts.executeQueryButton.click(function(evt) { - var jdbcQueryInfo = {}; - jdbcQueryInfo.connectionName = $( "#currentConnectionNameInput" ).val(); - jdbcQueryInfo.databaseType = $( "#currentDatabaseTypeInput" ).val(); - jdbcQueryInfo.databaseServer = $( "#currentDatabaseHostInput" ).val(); - jdbcQueryInfo.databasePort = $( "#currentDatabasePortInput" ).val(); - jdbcQueryInfo.databaseUser = $( "#currentDatabaseUserInput" ).val(); - jdbcQueryInfo.databasePassword = $( "#currentDatabasePasswordInput" ).val(); - jdbcQueryInfo.initialDatabase = $( "#currentInitialDatabaseInput" ).val(); - jdbcQueryInfo.query = $.trim($( "#queryTextArea" ).val()); - - if(self.validateQuery(jdbcQueryInfo.query)) { - self._executeQuery(jdbcQueryInfo); - } - - - }); - - - - this._elmts.editConnectionButton.click(function(evt) { - - if(self._validateNewConnectionForm() == true){ - var connectionNameInput = $.trim(self._elmts.connectionNameInput[0].value); - if (connectionNameInput.length === 0) { - window.alert($.i18n('database-source/alert-connection-name')); - } else{ - self._editConnection(self._getConnectionInfo()); - } - - } - - }); - - this._elmts.cancelEditConnectionButton.click(function(evt) { - self._resetDatabaseImportForm(); - - }); - - //load saved connections from settings file in user home directory - self._loadSavedConnections(); - -};//end Refine.createUI - - -Refine.DatabaseSourceUI.prototype.focus = function() { -}; - -Refine.DatabaseSourceUI.prototype.validateQuery = function(query) { - //alert("query::" + query); - if(!query || query.length <= 0 ) { - window.alert($.i18n('database-source/alert-query')); - return false; - } - - var allCapsQuery = query.toUpperCase(); - - if(allCapsQuery.indexOf('DROP') > -1){ - window.alert($.i18n('database-source/alert-invalid-query-keyword') + " DROP"); - return false; - }else if(allCapsQuery.indexOf('TRUNCATE') > -1){ - window.alert($.i18n('database-source/alert-invalid-query-keyword') + " TRUNCATE"); - return false; - }else if(allCapsQuery.indexOf('DELETE') > -1){ - window.alert($.i18n('database-source/alert-invalid-query-keyword') + " DELETE"); - return false; - }else if(allCapsQuery.indexOf('ROLLBACK') > -1){ - window.alert($.i18n('database-source/alert-invalid-query-keyword') + " ROLLBACK"); - return false; - }else if(allCapsQuery.indexOf('SHUTDOWN') > -1){ - window.alert($.i18n('database-source/alert-invalid-query-keyword') + " SHUTDOWN"); - return false; - }else if(allCapsQuery.indexOf('INSERT') > -1){ - window.alert($.i18n('database-source/alert-invalid-query-keyword') + " INSERT"); - return false; - }else if(allCapsQuery.indexOf('ALTER') > -1){ - window.alert($.i18n('database-source/alert-invalid-query-keyword') + " ALTER"); - return false; - }else if(allCapsQuery.indexOf('UPDATE') > -1){ - window.alert($.i18n('database-source/alert-invalid-query-keyword') + " UPDATE"); - return false; - } - - if(!allCapsQuery.startsWith('SELECT')) { - window.alert($.i18n('database-source/alert-invalid-query-select')); - return false; - } - - return true; -}; - -Refine.DatabaseSourceUI.prototype._updateDatabaseType = function(databaseType) { - if(databaseType === "postgresql") { - $( "#databaseUser" ).val("postgres"); - $( "#databasePort" ).val("5432"); - - } else if(databaseType === "mysql") { - $( "#databaseUser" ).val("root"); - $( "#databasePort" ).val("3306"); - - } else if(databaseType === "mariadb") { - $( "#databaseUser" ).val("root"); - $( "#databasePort" ).val("3306"); - - } else if(databaseType === "sqlite") { - $( "#databaseUser" ).val("na"); - $( "#databasePort" ).val("0"); - $( "#databaseHost" ).val("na"); - - } else { - $( "#databaseUser" ).val("root"); - $( "#databasePort" ).val("3306"); - databaseType = "mysql"; - } - - $("div.pure-control-group.dbtype-options").hide(); - $("div.pure-control-group.dbtype-options.dbt-"+databaseType).show(); - - if (databaseType == "sqlite") { - $('#databaseNameLabel').text($.i18n('database-source/databaseFileNameLabel')); - $('input#initialDatabase').attr('placeholder', $.i18n('database-source/databaseFileNamePlaceholder')); - - } else { - $('#databaseNameLabel').text($.i18n('database-source/databaseNameLabel')); - $('input#initialDatabase').attr('placeholder', $.i18n('database-source/databaseNamePlaceholder')); - } -}; - -Refine.DatabaseSourceUI.prototype._editConnection = function(connectionInfo) { - //alert("database user:" + connectionInfo.databaseUser); - var self = this; - $.ajax({ - url: 'command/database/saved-connection', - type: 'PUT', - contentType:'application/x-www-form-urlencoded', - data: connectionInfo, - success: function(settings) { - if(settings){ - $( "#menuListUl" ).empty(); - var menuList = $('#menuListUl'); - var items = []; - $.each(settings.savedConnections,function(index,savedConnection){ - - var li = $('
  • ').appendTo(menuList); - var a = $('').appendTo(li); - $('').text(savedConnection.connectionName) - .appendTo(a); - $(' ').appendTo(a); - }) - - $( "#menuListUl" ).append(items.join('')); - window.alert($.i18n('database-source/alert-connection-edit')); - } - } - }).fail(function( jqXhr, textStatus, errorThrown ){ - alert( textStatus + ':' + errorThrown ); - }); - -}; - -Refine.DatabaseSourceUI.prototype._executeQuery = function(jdbcQueryInfo) { - var self = this; - //remove start line - - var dismiss = DialogSystem.showBusy($.i18n('database-import/checking')); - - Refine.postCSRF( - "command/database/test-query", - jdbcQueryInfo, - function(jdbcConnectionResult) { - - dismiss(); - self._controller.startImportingDocument(jdbcQueryInfo); - - }, - "json", - function( jqXhr, textStatus, errorThrown ){ - - dismiss(); - alert( textStatus + ':' + errorThrown ); - }); - -} - -Refine.DatabaseSourceUI.prototype._saveConnection = function(jdbcConnectionInfo) { - var self = this; - Refine.postCSRF( - "command/database/saved-connection", - jdbcConnectionInfo, - function(settings) { - if(settings){ - - self._elmts.menuListUl.empty(); - var items = []; - $.each(settings.savedConnections,function(index,savedConnection){ - - items.push('
  • ' - + '' + savedConnection.connectionName + '' - + '
  • '); - }) - - self._elmts.menuListUl.append(items.join('')); - } - - }, - "json", - function( jqXhr, textStatus, errorThrown ){ - alert( textStatus + ':' + errorThrown ); - }); - -}; - -Refine.DatabaseSourceUI.prototype._loadSavedConnections = function() { - var self = this; - $.get( - "command/database/saved-connection", - null, - function(settings) { - if(settings){ - - self._elmts.menuListUl.empty(); - - var items = []; - $.each(settings.savedConnections,function(index,savedConnection){ - - items.push('
  • ' - + '' + savedConnection.connectionName + '' - + '
  • '); - - }) - - self._elmts.menuListUl.append(items.join('')); - - } - - }, - "json" - ); - -}; - -Refine.DatabaseSourceUI.prototype._testDatabaseConnect = function(jdbcConnectionInfo) { - - var self = this; - Refine.postCSRF( - "command/database/test-connect", - jdbcConnectionInfo, - function(jdbcConnectionResult) { - if(jdbcConnectionResult && jdbcConnectionResult.connectionResult == true){ - window.alert("Test Connection Succeeded!"); - }else{ - window.alert("Unable to establish connection to database"); - } - - }, - "json", - function( jqXhr, textStatus, errorThrown ){ - alert( textStatus + ':' + errorThrown ); - }); -}; - -Refine.DatabaseSourceUI.prototype._connect = function(jdbcConnectionInfo) { - - var self = this; - Refine.postCSRF( - "command/database/connect", - jdbcConnectionInfo, - function(databaseInfo) { - - if(databaseInfo){ - $( "#currentConnectionNameInput" ).val(jdbcConnectionInfo.connectionName); - $( "#currentDatabaseTypeInput" ).val(jdbcConnectionInfo.databaseType); - $( "#currentDatabaseUserInput" ).val(jdbcConnectionInfo.databaseUser); - $( "#currentDatabasePasswordInput" ).val(jdbcConnectionInfo.databasePassword); - $( "#currentDatabaseHostInput" ).val(jdbcConnectionInfo.databaseServer); - $( "#currentDatabasePortInput" ).val(jdbcConnectionInfo.databasePort); - $( "#currentInitialDatabaseInput" ).val(jdbcConnectionInfo.initialDatabase); - - var connectionParam = "Connection :: " - + "jdbc:" - + jdbcConnectionInfo.databaseType + "://" - + jdbcConnectionInfo.databaseServer + ":" - + jdbcConnectionInfo.databasePort + "/" - + jdbcConnectionInfo.initialDatabase; - - - $( "#connectionParameterSpan" ).text(connectionParam); - $( "#newConnectionDiv" ).hide(); - $( "#sqlEditorDiv" ).show(); - - }else{ - window.alert("Unable to establish connection to database"); - return; - } - - }, - "json", - function( jqXhr, textStatus, errorThrown ){ - alert( textStatus + ':' + errorThrown ); - }); - -}; - -Refine.DatabaseSourceUI.prototype._getConnectionInfo = function() { - var self = this; - var jdbcConnectionInfo = {}; - jdbcConnectionInfo.connectionName = $.trim(self._elmts.connectionNameInput[0].value); - jdbcConnectionInfo.databaseType = $.trim(self._elmts.databaseTypeSelect[0].value); - jdbcConnectionInfo.databaseServer = $.trim(self._elmts.databaseHostInput[0].value); - jdbcConnectionInfo.databasePort = $.trim(self._elmts.databasePortInput[0].value); - jdbcConnectionInfo.databaseUser = $.trim(self._elmts.databaseUserInput[0].value); - jdbcConnectionInfo.databasePassword = $.trim(self._elmts.databasePasswordInput[0].value); - jdbcConnectionInfo.initialDatabase = $.trim(self._elmts.initialDatabaseInput[0].value); - jdbcConnectionInfo.initialSchema = $.trim(self._elmts.initialSchemaInput[0].value); - return jdbcConnectionInfo; - -} - - -Refine.DatabaseSourceUI.prototype._validateNewConnectionForm = function() { - - var self = this; - var connectionNameInput = $.trim(self._elmts.connectionNameInput[0].value); - var databaseTypeSelect = $.trim(self._elmts.databaseTypeSelect[0].value); - var databaseHostInput = $.trim(self._elmts.databaseHostInput[0].value); - var databasePortInput = $.trim(self._elmts.databasePortInput[0].value); - var databaseUserInput = $.trim(self._elmts.databaseUserInput[0].value); - var databasePasswordInput = $.trim(self._elmts.databasePasswordInput[0].value); - var initialDatabaseInput = $.trim(self._elmts.initialDatabaseInput[0].value); - var initialSchemaInput = $.trim(self._elmts.initialSchemaInput[0].value); - - var alphaNumRE = /^[a-zA-Z0-9._-]*$/; - var numRE = /^[0-9]*$/; - - var alphaNumConnNameTestResult = alphaNumRE.test(connectionNameInput); - var databaseHostTestResult = alphaNumRE.test(databaseHostInput); - var databasePortTestResult = numRE.test(databasePortInput); - var databaseUserTestResult = alphaNumRE.test(databaseUserInput); - - if(alphaNumConnNameTestResult == false){ - window.alert($.i18n('database-source/alert-conn-name-invalid-character')); - return false; - }else if (databaseHostInput.length === 0) { - window.alert($.i18n('database-source/alert-server')); - return false; - }else if(databasePortInput.length === 0){ - window.alert($.i18n('database-source/alert-port')); - return false; - }else if(databaseUserInput.length === 0){ - window.alert($.i18n('database-source/alert-user')); - return false; - }else if(initialDatabaseInput.length === 0){ - window.alert($.i18n('database-source/alert-initial-database')); - return false; - }else if(databasePortTestResult == false){ - window.alert($.i18n('database-source/alert-db-port-invalid-character')); - return false; - - } - else{ - return true; - - } - - return true; -}; - -Refine.DatabaseSourceUI.prototype._resetDatabaseImportForm = function() { - var self = this; - - $( "#databaseHost" ).val(self._defaultDatabaseHost); - - $('input#connectionName').val($.i18n('database-source/connectionNameDefaultValue')); - $( "select#databaseTypeSelect" ).val(self._defaultDatabaseType); - self._updateDatabaseType(self._defaultDatabaseType); - - $( "#databasePassword" ).val(""); - $( "#initialDatabase" ).val(""); - $( "#initialSchema" ).val(""); - - $( "#editConnectionControlDiv" ).hide(); - $( "#newConnectionControlDiv" ).show(); - $('#connectionName').removeAttr('readonly'); -}; diff --git a/OpenRefine/extensions/database/module/scripts/index/jquery.contextMenu.min.js b/OpenRefine/extensions/database/module/scripts/index/jquery.contextMenu.min.js deleted file mode 100644 index bf6b6169f..000000000 --- a/OpenRefine/extensions/database/module/scripts/index/jquery.contextMenu.min.js +++ /dev/null @@ -1,2 +0,0 @@ -!function(e){"function"==typeof define&&define.amd?define(["jquery"],e):e("object"==typeof exports?require("jquery"):jQuery)}(function(e){"use strict";function t(e){for(var t,n=e.split(/\s+/),a=[],o=0;t=n[o];o++)t=t.charAt(0).toUpperCase(),a.push(t);return a}function n(t){return t.id&&e('label[for="'+t.id+'"]').val()||t.name}function a(t,o,s){return s||(s=0),o.each(function(){var o,i,c=e(this),l=this,r=this.nodeName.toLowerCase();switch("label"===r&&c.find("input, textarea, select").length&&(o=c.text(),r=(l=(c=c.children().first()).get(0)).nodeName.toLowerCase()),r){case"menu":i={name:c.attr("label"),items:{}},s=a(i.items,c.children(),s);break;case"a":case"button":i={name:c.text(),disabled:!!c.attr("disabled"),callback:function(){c.get(0).click()}};break;case"menuitem":case"command":switch(c.attr("type")){case void 0:case"command":case"menuitem":i={name:c.attr("label"),disabled:!!c.attr("disabled"),icon:c.attr("icon"),callback:function(){c.get(0).click()}};break;case"checkbox":i={type:"checkbox",disabled:!!c.attr("disabled"),name:c.attr("label"),selected:!!c.attr("checked")};break;case"radio":i={type:"radio",disabled:!!c.attr("disabled"),name:c.attr("label"),radio:c.attr("radiogroup"),value:c.attr("id"),selected:!!c.attr("checked")};break;default:i=void 0}break;case"hr":i="-------";break;case"input":switch(c.attr("type")){case"text":i={type:"text",name:o||n(l),disabled:!!c.attr("disabled"),value:c.val()};break;case"checkbox":i={type:"checkbox",name:o||n(l),disabled:!!c.attr("disabled"),selected:!!c.attr("checked")};break;case"radio":i={type:"radio",name:o||n(l),disabled:!!c.attr("disabled"),radio:!!c.attr("name"),value:c.val(),selected:!!c.attr("checked")};break;default:i=void 0}break;case"select":i={type:"select",name:o||n(l),disabled:!!c.attr("disabled"),selected:c.val(),options:{}},c.children().each(function(){i.options[this.value]=e(this).text()});break;case"textarea":i={type:"textarea",name:o||n(l),disabled:!!c.attr("disabled"),value:c.val()};break;case"label":break;default:i={type:"html",html:c.clone(!0)}}i&&(t["key"+ ++s]=i)}),s}e.support.htmlMenuitem="HTMLMenuItemElement"in window,e.support.htmlCommand="HTMLCommandElement"in window,e.support.eventSelectstart="onselectstart"in document.documentElement,e.ui&&e.widget||(e.cleanData=function(t){return function(n){var a,o,s;for(s=0;null!=n[s];s++){o=n[s];try{(a=e._data(o,"events"))&&a.remove&&e(o).triggerHandler("remove")}catch(e){}}t(n)}}(e.cleanData));var o=null,s=!1,i=e(window),c=0,l={},r={},u={},d={selector:null,appendTo:null,trigger:"right",autoHide:!1,delay:200,reposition:!0,hideOnSecondTrigger:!1,selectableSubMenu:!1,classNames:{hover:"context-menu-hover",disabled:"context-menu-disabled",visible:"context-menu-visible",notSelectable:"context-menu-not-selectable",icon:"context-menu-icon",iconEdit:"context-menu-icon-edit",iconCut:"context-menu-icon-cut",iconCopy:"context-menu-icon-copy",iconPaste:"context-menu-icon-paste",iconDelete:"context-menu-icon-delete",iconAdd:"context-menu-icon-add",iconQuit:"context-menu-icon-quit",iconLoadingClass:"context-menu-icon-loading"},determinePosition:function(t){if(e.ui&&e.ui.position)t.css("display","block").position({my:"center top",at:"center bottom",of:this,offset:"0 5",collision:"fit"}).css("display","none");else{var n=this.offset();n.top+=this.outerHeight(),n.left+=this.outerWidth()/2-t.outerWidth()/2,t.css(n)}},position:function(e,t,n){var a;if(t||n){if("maintain"===t&&"maintain"===n)a=e.$menu.position();else{var o=e.$menu.offsetParent().offset();a={top:n-o.top,left:t-o.left}}var s=i.scrollTop()+i.height(),c=i.scrollLeft()+i.width(),l=e.$menu.outerHeight(),r=e.$menu.outerWidth();a.top+l>s&&(a.top-=l),a.top<0&&(a.top=0),a.left+r>c&&(a.left-=r),a.left<0&&(a.left=0),e.$menu.css(a)}else e.determinePosition.call(this,e.$menu)},positionSubmenu:function(t){if(void 0!==t)if(e.ui&&e.ui.position)t.css("display","block").position({my:"left top-5",at:"right top",of:this,collision:"flipfit fit"}).css("display","");else{var n={top:-9,left:this.outerWidth()-5};t.css(n)}},zIndex:1,animation:{duration:50,show:"slideDown",hide:"slideUp"},events:{show:e.noop,hide:e.noop,activated:e.noop},callback:null,items:{}},m={timer:null,pageX:null,pageY:null},p=function(e){for(var t=0,n=e;;)if(t=Math.max(t,parseInt(n.css("z-index"),10)||0),!(n=n.parent())||!n.length||"html body".indexOf(n.prop("nodeName").toLowerCase())>-1)break;return t},f={abortevent:function(e){e.preventDefault(),e.stopImmediatePropagation()},contextmenu:function(t){var n=e(this);if("right"===t.data.trigger&&(t.preventDefault(),t.stopImmediatePropagation()),!("right"!==t.data.trigger&&"demand"!==t.data.trigger&&t.originalEvent||!(void 0===t.mouseButton||!t.data||"left"===t.data.trigger&&0===t.mouseButton||"right"===t.data.trigger&&2===t.mouseButton)||n.hasClass("context-menu-active")||n.hasClass("context-menu-disabled"))){if(o=n,t.data.build){var a=t.data.build(o,t);if(!1===a)return;if(t.data=e.extend(!0,{},d,t.data,a||{}),!t.data.items||e.isEmptyObject(t.data.items))throw window.console&&(console.error||console.log).call(console,"No items specified to show in contextMenu"),new Error("No Items specified");t.data.$trigger=o,h.create(t.data)}var s=!1;for(var i in t.data.items)if(t.data.items.hasOwnProperty(i)){(e.isFunction(t.data.items[i].visible)?t.data.items[i].visible.call(e(t.currentTarget),i,t.data):void 0===t.data.items[i]||!t.data.items[i].visible||!0===t.data.items[i].visible)&&(s=!0)}s&&h.show.call(n,t.data,t.pageX,t.pageY)}},click:function(t){t.preventDefault(),t.stopImmediatePropagation(),e(this).trigger(e.Event("contextmenu",{data:t.data,pageX:t.pageX,pageY:t.pageY}))},mousedown:function(t){var n=e(this);o&&o.length&&!o.is(n)&&o.data("contextMenu").$menu.trigger("contextmenu:hide"),2===t.button&&(o=n.data("contextMenuActive",!0))},mouseup:function(t){var n=e(this);n.data("contextMenuActive")&&o&&o.length&&o.is(n)&&!n.hasClass("context-menu-disabled")&&(t.preventDefault(),t.stopImmediatePropagation(),o=n,n.trigger(e.Event("contextmenu",{data:t.data,pageX:t.pageX,pageY:t.pageY}))),n.removeData("contextMenuActive")},mouseenter:function(t){var n=e(this),a=e(t.relatedTarget),s=e(document);a.is(".context-menu-list")||a.closest(".context-menu-list").length||o&&o.length||(m.pageX=t.pageX,m.pageY=t.pageY,m.data=t.data,s.on("mousemove.contextMenuShow",f.mousemove),m.timer=setTimeout(function(){m.timer=null,s.off("mousemove.contextMenuShow"),o=n,n.trigger(e.Event("contextmenu",{data:m.data,pageX:m.pageX,pageY:m.pageY}))},t.data.delay))},mousemove:function(e){m.pageX=e.pageX,m.pageY=e.pageY},mouseleave:function(t){var n=e(t.relatedTarget);if(!n.is(".context-menu-list")&&!n.closest(".context-menu-list").length){try{clearTimeout(m.timer)}catch(t){}m.timer=null}},layerClick:function(t){var n,a,o=e(this).data("contextMenuRoot"),s=t.button,c=t.pageX,l=t.pageY;t.preventDefault(),setTimeout(function(){var r,u="left"===o.trigger&&0===s||"right"===o.trigger&&2===s;if(document.elementFromPoint&&o.$layer){if(o.$layer.hide(),(n=document.elementFromPoint(c-i.scrollLeft(),l-i.scrollTop())).isContentEditable){var d=document.createRange(),m=window.getSelection();d.selectNode(n),d.collapse(!0),m.removeAllRanges(),m.addRange(d)}e(n).trigger(t),o.$layer.show()}if(o.hideOnSecondTrigger&&u&&null!==o.$menu&&void 0!==o.$menu)o.$menu.trigger("contextmenu:hide");else{if(o.reposition&&u)if(document.elementFromPoint){if(o.$trigger.is(n))return void o.position.call(o.$trigger,o,c,l)}else if(a=o.$trigger.offset(),r=e(window),a.top+=r.scrollTop(),a.top<=t.pageY&&(a.left+=r.scrollLeft(),a.left<=t.pageX&&(a.bottom=a.top+o.$trigger.outerHeight(),a.bottom>=t.pageY&&(a.right=a.left+o.$trigger.outerWidth(),a.right>=t.pageX))))return void o.position.call(o.$trigger,o,c,l);n&&u&&o.$trigger.one("contextmenu:hidden",function(){e(n).contextMenu({x:c,y:l,button:s})}),null!==o&&void 0!==o&&null!==o.$menu&&void 0!==o.$menu&&o.$menu.trigger("contextmenu:hide")}},50)},keyStop:function(e,t){t.isInput||e.preventDefault(),e.stopPropagation()},key:function(e){var t={};o&&(t=o.data("contextMenu")||{}),void 0===t.zIndex&&(t.zIndex=0);var n=0,a=function(e){""!==e.style.zIndex?n=e.style.zIndex:null!==e.offsetParent&&void 0!==e.offsetParent?a(e.offsetParent):null!==e.parentElement&&void 0!==e.parentElement&&a(e.parentElement)};if(a(e.target),!(t.$menu&&parseInt(n,10)>parseInt(t.$menu.css("zIndex"),10))){switch(e.keyCode){case 9:case 38:if(f.keyStop(e,t),t.isInput){if(9===e.keyCode&&e.shiftKey)return e.preventDefault(),t.$selected&&t.$selected.find("input, textarea, select").blur(),void(null!==t.$menu&&void 0!==t.$menu&&t.$menu.trigger("prevcommand"));if(38===e.keyCode&&"checkbox"===t.$selected.find("input, textarea, select").prop("type"))return void e.preventDefault()}else if(9!==e.keyCode||e.shiftKey)return void(null!==t.$menu&&void 0!==t.$menu&&t.$menu.trigger("prevcommand"));break;case 40:if(f.keyStop(e,t),!t.isInput)return void(null!==t.$menu&&void 0!==t.$menu&&t.$menu.trigger("nextcommand"));if(9===e.keyCode)return e.preventDefault(),t.$selected&&t.$selected.find("input, textarea, select").blur(),void(null!==t.$menu&&void 0!==t.$menu&&t.$menu.trigger("nextcommand"));if(40===e.keyCode&&"checkbox"===t.$selected.find("input, textarea, select").prop("type"))return void e.preventDefault();break;case 37:if(f.keyStop(e,t),t.isInput||!t.$selected||!t.$selected.length)break;if(!t.$selected.parent().hasClass("context-menu-root")){var s=t.$selected.parent().parent();return t.$selected.trigger("contextmenu:blur"),void(t.$selected=s)}break;case 39:if(f.keyStop(e,t),t.isInput||!t.$selected||!t.$selected.length)break;var i=t.$selected.data("contextMenu")||{};if(i.$menu&&t.$selected.hasClass("context-menu-submenu"))return t.$selected=null,i.$selected=null,void i.$menu.trigger("nextcommand");break;case 35:case 36:return t.$selected&&t.$selected.find("input, textarea, select").length?void 0:((t.$selected&&t.$selected.parent()||t.$menu).children(":not(."+t.classNames.disabled+", ."+t.classNames.notSelectable+")")[36===e.keyCode?"first":"last"]().trigger("contextmenu:focus"),void e.preventDefault());case 13:if(f.keyStop(e,t),t.isInput){if(t.$selected&&!t.$selected.is("textarea, select"))return void e.preventDefault();break}return void(void 0!==t.$selected&&null!==t.$selected&&t.$selected.trigger("mouseup"));case 32:case 33:case 34:return void f.keyStop(e,t);case 27:return f.keyStop(e,t),void(null!==t.$menu&&void 0!==t.$menu&&t.$menu.trigger("contextmenu:hide"));default:var c=String.fromCharCode(e.keyCode).toUpperCase();if(t.accesskeys&&t.accesskeys[c])return void t.accesskeys[c].$node.trigger(t.accesskeys[c].$menu?"contextmenu:focus":"mouseup")}e.stopPropagation(),void 0!==t.$selected&&null!==t.$selected&&t.$selected.trigger(e)}},prevItem:function(t){t.stopPropagation();var n=e(this).data("contextMenu")||{},a=e(this).data("contextMenuRoot")||{};if(n.$selected){var o=n.$selected;(n=n.$selected.parent().data("contextMenu")||{}).$selected=o}for(var s=n.$menu.children(),i=n.$selected&&n.$selected.prev().length?n.$selected.prev():s.last(),c=i;i.hasClass(a.classNames.disabled)||i.hasClass(a.classNames.notSelectable)||i.is(":hidden");)if((i=i.prev().length?i.prev():s.last()).is(c))return;n.$selected&&f.itemMouseleave.call(n.$selected.get(0),t),f.itemMouseenter.call(i.get(0),t);var l=i.find("input, textarea, select");l.length&&l.focus()},nextItem:function(t){t.stopPropagation();var n=e(this).data("contextMenu")||{},a=e(this).data("contextMenuRoot")||{};if(n.$selected){var o=n.$selected;(n=n.$selected.parent().data("contextMenu")||{}).$selected=o}for(var s=n.$menu.children(),i=n.$selected&&n.$selected.next().length?n.$selected.next():s.first(),c=i;i.hasClass(a.classNames.disabled)||i.hasClass(a.classNames.notSelectable)||i.is(":hidden");)if((i=i.next().length?i.next():s.first()).is(c))return;n.$selected&&f.itemMouseleave.call(n.$selected.get(0),t),f.itemMouseenter.call(i.get(0),t);var l=i.find("input, textarea, select");l.length&&l.focus()},focusInput:function(){var t=e(this).closest(".context-menu-item"),n=t.data(),a=n.contextMenu,o=n.contextMenuRoot;o.$selected=a.$selected=t,o.isInput=a.isInput=!0},blurInput:function(){var t=e(this).closest(".context-menu-item").data(),n=t.contextMenu;t.contextMenuRoot.isInput=n.isInput=!1},menuMouseenter:function(){e(this).data().contextMenuRoot.hovering=!0},menuMouseleave:function(t){var n=e(this).data().contextMenuRoot;n.$layer&&n.$layer.is(t.relatedTarget)&&(n.hovering=!1)},itemMouseenter:function(t){var n=e(this),a=n.data(),o=a.contextMenu,s=a.contextMenuRoot;s.hovering=!0,t&&s.$layer&&s.$layer.is(t.relatedTarget)&&(t.preventDefault(),t.stopImmediatePropagation()),(o.$menu?o:s).$menu.children("."+s.classNames.hover).trigger("contextmenu:blur").children(".hover").trigger("contextmenu:blur"),n.hasClass(s.classNames.disabled)||n.hasClass(s.classNames.notSelectable)?o.$selected=null:n.trigger("contextmenu:focus")},itemMouseleave:function(t){var n=e(this),a=n.data(),o=a.contextMenu,s=a.contextMenuRoot;if(s!==o&&s.$layer&&s.$layer.is(t.relatedTarget))return void 0!==s.$selected&&null!==s.$selected&&s.$selected.trigger("contextmenu:blur"),t.preventDefault(),t.stopImmediatePropagation(),void(s.$selected=o.$selected=o.$node);o&&o.$menu&&o.$menu.hasClass("context-menu-visible")||n.trigger("contextmenu:blur")},itemClick:function(t){var n,a=e(this),o=a.data(),s=o.contextMenu,i=o.contextMenuRoot,c=o.contextMenuKey;if(!(!s.items[c]||a.is("."+i.classNames.disabled+", .context-menu-separator, ."+i.classNames.notSelectable)||a.is(".context-menu-submenu")&&!1===i.selectableSubMenu)){if(t.preventDefault(),t.stopImmediatePropagation(),e.isFunction(s.callbacks[c])&&Object.prototype.hasOwnProperty.call(s.callbacks,c))n=s.callbacks[c];else{if(!e.isFunction(i.callback))return;n=i.callback}!1!==n.call(i.$trigger,c,i,t)?i.$menu.trigger("contextmenu:hide"):i.$menu.parent().length&&h.update.call(i.$trigger,i)}},inputClick:function(e){e.stopImmediatePropagation()},hideMenu:function(t,n){var a=e(this).data("contextMenuRoot");h.hide.call(a.$trigger,a,n&&n.force)},focusItem:function(t){t.stopPropagation();var n=e(this),a=n.data(),o=a.contextMenu,s=a.contextMenuRoot;n.hasClass(s.classNames.disabled)||n.hasClass(s.classNames.notSelectable)||(n.addClass([s.classNames.hover,s.classNames.visible].join(" ")).parent().find(".context-menu-item").not(n).removeClass(s.classNames.visible).filter("."+s.classNames.hover).trigger("contextmenu:blur"),o.$selected=s.$selected=n,o&&o.$node&&o.$node.hasClass("context-menu-submenu")&&o.$node.addClass(s.classNames.hover),o.$node&&s.positionSubmenu.call(o.$node,o.$menu))},blurItem:function(t){t.stopPropagation();var n=e(this),a=n.data(),o=a.contextMenu,s=a.contextMenuRoot;o.autoHide&&n.removeClass(s.classNames.visible),n.removeClass(s.classNames.hover),o.$selected=null}},h={show:function(t,n,a){var s=e(this),i={};if(e("#context-menu-layer").trigger("mousedown"),t.$trigger=s,!1!==t.events.show.call(s,t)){if(h.update.call(s,t),t.position.call(s,t,n,a),t.zIndex){var c=t.zIndex;"function"==typeof t.zIndex&&(c=t.zIndex.call(s,t)),i.zIndex=p(s)+c}h.layer.call(t.$menu,t,i.zIndex),t.$menu.find("ul").css("zIndex",i.zIndex+1),t.$menu.css(i)[t.animation.show](t.animation.duration,function(){s.trigger("contextmenu:visible"),h.activated(t),t.events.activated()}),s.data("contextMenu",t).addClass("context-menu-active"),e(document).off("keydown.contextMenu").on("keydown.contextMenu",f.key),t.autoHide&&e(document).on("mousemove.contextMenuAutoHide",function(e){var n=s.offset();n.right=n.left+s.outerWidth(),n.bottom=n.top+s.outerHeight(),!t.$layer||t.hovering||e.pageX>=n.left&&e.pageX<=n.right&&e.pageY>=n.top&&e.pageY<=n.bottom||setTimeout(function(){t.hovering||null===t.$menu||void 0===t.$menu||t.$menu.trigger("contextmenu:hide")},50)})}else o=null},hide:function(t,n){var a=e(this);if(t||(t=a.data("contextMenu")||{}),n||!t.events||!1!==t.events.hide.call(a,t)){if(a.removeData("contextMenu").removeClass("context-menu-active"),t.$layer){setTimeout(function(e){return function(){e.remove()}}(t.$layer),10);try{delete t.$layer}catch(e){t.$layer=null}}o=null,t.$menu.find("."+t.classNames.hover).trigger("contextmenu:blur"),t.$selected=null,t.$menu.find("."+t.classNames.visible).removeClass(t.classNames.visible),e(document).off(".contextMenuAutoHide").off("keydown.contextMenu"),t.$menu&&t.$menu[t.animation.hide](t.animation.duration,function(){t.build&&(t.$menu.remove(),e.each(t,function(e){switch(e){case"ns":case"selector":case"build":case"trigger":return!0;default:t[e]=void 0;try{delete t[e]}catch(e){}return!0}})),setTimeout(function(){a.trigger("contextmenu:hidden")},10)})}},create:function(n,a){function o(t){var n=e("");if(t._accesskey)t._beforeAccesskey&&n.append(document.createTextNode(t._beforeAccesskey)),e("").addClass("context-menu-accesskey").text(t._accesskey).appendTo(n),t._afterAccesskey&&n.append(document.createTextNode(t._afterAccesskey));else if(t.isHtmlName){if(void 0!==t.accesskey)throw new Error("accesskeys are not compatible with HTML names and cannot be used together in the same item");n.html(t.name)}else n.text(t.name);return n}void 0===a&&(a=n),n.$menu=e('
      ').addClass(n.className||"").data({contextMenu:n,contextMenuRoot:a}),e.each(["callbacks","commands","inputs"],function(e,t){n[t]={},a[t]||(a[t]={})}),a.accesskeys||(a.accesskeys={}),e.each(n.items,function(s,i){var c=e('
    • ').addClass(i.className||""),l=null,r=null;if(c.on("click",e.noop),"string"!=typeof i&&"cm_separator"!==i.type||(i={type:"cm_seperator"}),i.$node=c.data({contextMenu:n,contextMenuRoot:a,contextMenuKey:s}),void 0!==i.accesskey)for(var d,m=t(i.accesskey),p=0;d=m[p];p++)if(!a.accesskeys[d]){a.accesskeys[d]=i;var v=i.name.match(new RegExp("^(.*?)("+d+")(.*)$","i"));v&&(i._beforeAccesskey=v[1],i._accesskey=v[2],i._afterAccesskey=v[3]);break}if(i.type&&u[i.type])u[i.type].call(c,i,n,a),e.each([n,a],function(t,a){a.commands[s]=i,!e.isFunction(i.callback)||void 0!==a.callbacks[s]&&void 0!==n.type||(a.callbacks[s]=i.callback)});else{switch("cm_seperator"===i.type?c.addClass("context-menu-separator "+a.classNames.notSelectable):"html"===i.type?c.addClass("context-menu-html "+a.classNames.notSelectable):"sub"===i.type||(i.type?(l=e("").appendTo(c),o(i).appendTo(l),c.addClass("context-menu-input"),n.hasTypes=!0,e.each([n,a],function(e,t){t.commands[s]=i,t.inputs[s]=i})):i.items&&(i.type="sub")),i.type){case"cm_seperator":break;case"text":r=e('').attr("name","context-menu-input-"+s).val(i.value||"").appendTo(l);break;case"textarea":r=e('').attr("name","context-menu-input-"+s).val(i.value||"").appendTo(l),i.height&&r.height(i.height);break;case"checkbox":r=e('').attr("name","context-menu-input-"+s).val(i.value||"").prop("checked",!!i.selected).prependTo(l);break;case"radio":r=e('').attr("name","context-menu-input-"+i.radio).val(i.value||"").prop("checked",!!i.selected).prependTo(l);break;case"select":r=e('').attr("name","context-menu-input-"+s).appendTo(l),i.options&&(e.each(i.options,function(t,n){e("").val(t).text(n).appendTo(r)}),r.val(i.selected));break;case"sub":o(i).appendTo(c),i.appendTo=i.$node,c.data("contextMenu",i).addClass("context-menu-submenu"),i.callback=null,"function"==typeof i.items.then?h.processPromises(i,a,i.items):h.create(i,a);break;case"html":e(i.html).appendTo(c);break;default:e.each([n,a],function(t,a){a.commands[s]=i,!e.isFunction(i.callback)||void 0!==a.callbacks[s]&&void 0!==n.type||(a.callbacks[s]=i.callback)}),o(i).appendTo(c)}i.type&&"sub"!==i.type&&"html"!==i.type&&"cm_seperator"!==i.type&&(r.on("focus",f.focusInput).on("blur",f.blurInput),i.events&&r.on(i.events,n)),i.icon&&(e.isFunction(i.icon)?i._icon=i.icon.call(this,this,c,s,i):"string"==typeof i.icon&&"fa-"===i.icon.substring(0,3)?i._icon=a.classNames.icon+" "+a.classNames.icon+"--fa fa "+i.icon:i._icon=a.classNames.icon+" "+a.classNames.icon+"-"+i.icon,c.addClass(i._icon))}i.$input=r,i.$label=l,c.appendTo(n.$menu),!n.hasTypes&&e.support.eventSelectstart&&c.on("selectstart.disableTextSelect",f.abortevent)}),n.$node||n.$menu.css("display","none").addClass("context-menu-root"),n.$menu.appendTo(n.appendTo||document.body)},resize:function(t,n){var a;t.css({position:"absolute",display:"block"}),t.data("width",(a=t.get(0)).getBoundingClientRect?Math.ceil(a.getBoundingClientRect().width):t.outerWidth()+1),t.css({position:"static",minWidth:"0px",maxWidth:"100000px"}),t.find("> li > ul").each(function(){h.resize(e(this),!0)}),n||t.find("ul").addBack().css({position:"",display:"",minWidth:"",maxWidth:""}).outerWidth(function(){return e(this).data("width")})},update:function(t,n){var a=this;void 0===n&&(n=t,h.resize(t.$menu)),t.$menu.children().each(function(){var o,s=e(this),i=s.data("contextMenuKey"),c=t.items[i],l=e.isFunction(c.disabled)&&c.disabled.call(a,i,n)||!0===c.disabled;if(o=e.isFunction(c.visible)?c.visible.call(a,i,n):void 0===c.visible||!0===c.visible,s[o?"show":"hide"](),s[l?"addClass":"removeClass"](n.classNames.disabled),e.isFunction(c.icon)&&(s.removeClass(c._icon),c._icon=c.icon.call(this,a,s,i,c),s.addClass(c._icon)),c.type)switch(s.find("input, select, textarea").prop("disabled",l),c.type){case"text":case"textarea":c.$input.val(c.value||"");break;case"checkbox":case"radio":c.$input.val(c.value||"").prop("checked",!!c.selected);break;case"select":c.$input.val((0===c.selected?"0":c.selected)||"")}c.$menu&&h.update.call(a,c,n)})},layer:function(t,n){var a=t.$layer=e('
      ').css({height:i.height(),width:i.width(),display:"block",position:"fixed","z-index":n,top:0,left:0,opacity:0,filter:"alpha(opacity=0)","background-color":"#000"}).data("contextMenuRoot",t).insertBefore(this).on("contextmenu",f.abortevent).on("mousedown",f.layerClick);return void 0===document.body.style.maxWidth&&a.css({position:"absolute",height:e(document).height()}),a},processPromises:function(e,t,n){function a(e,t,n){void 0===n?(n={error:{name:"No items and no error item",icon:"context-menu-icon context-menu-icon-quit"}},window.console&&(console.error||console.log).call(console,'When you reject a promise, provide an "items" object, equal to normal sub-menu items')):"string"==typeof n&&(n={error:{name:n}}),o(e,t,n)}function o(e,t,n){void 0!==t.$menu&&t.$menu.is(":visible")&&(e.$node.removeClass(t.classNames.iconLoadingClass),e.items=n,h.create(e,t,!0),h.update(e,t),t.positionSubmenu.call(e.$node,e.$menu))}e.$node.addClass(t.classNames.iconLoadingClass),n.then(function(e,t,n){void 0===n&&a(void 0),o(e,t,n)}.bind(this,e,t),a.bind(this,e,t))},activated:function(t){var n=t.$menu,a=n.offset(),o=e(window).height(),s=e(window).scrollTop(),i=n.height();i>o?n.css({height:o+"px","overflow-x":"hidden","overflow-y":"auto",top:s+"px"}):(a.tops+o)&&n.css({top:"0px"})}};e.fn.contextMenu=function(t){var n=this,a=t;if(this.length>0)if(void 0===t)this.first().trigger("contextmenu");else if(void 0!==t.x&&void 0!==t.y)this.first().trigger(e.Event("contextmenu",{pageX:t.x,pageY:t.y,mouseButton:t.button}));else if("hide"===t){var o=this.first().data("contextMenu")?this.first().data("contextMenu").$menu:null;o&&o.trigger("contextmenu:hide")}else"destroy"===t?e.contextMenu("destroy",{context:this}):e.isPlainObject(t)?(t.context=this,e.contextMenu("create",t)):t?this.removeClass("context-menu-disabled"):t||this.addClass("context-menu-disabled");else e.each(r,function(){this.selector===n.selector&&(a.data=this,e.extend(a.data,{trigger:"demand"}))}),f.contextmenu.call(a.target,a);return this},e.contextMenu=function(t,n){"string"!=typeof t&&(n=t,t="create"),"string"==typeof n?n={selector:n}:void 0===n&&(n={});var a=e.extend(!0,{},d,n||{}),o=e(document),i=o,u=!1;switch(a.context&&a.context.length?(i=e(a.context).first(),a.context=i.get(0),u=!e(a.context).is(document)):a.context=document,t){case"update":if(u)h.update(i);else for(var m in r)r.hasOwnProperty(m)&&h.update(r[m]);break;case"create":if(!a.selector)throw new Error("No selector specified");if(a.selector.match(/.context-menu-(list|item|input)($|\s)/))throw new Error('Cannot bind to selector "'+a.selector+'" as it contains a reserved className');if(!a.build&&(!a.items||e.isEmptyObject(a.items)))throw new Error("No Items specified");if(c++,a.ns=".contextMenu"+c,u||(l[a.selector]=a.ns),r[a.ns]=a,a.trigger||(a.trigger="right"),!s){var p="click"===a.itemClickEvent?"click.contextMenu":"mouseup.contextMenu",v={"contextmenu:focus.contextMenu":f.focusItem,"contextmenu:blur.contextMenu":f.blurItem,"contextmenu.contextMenu":f.abortevent,"mouseenter.contextMenu":f.itemMouseenter,"mouseleave.contextMenu":f.itemMouseleave};v[p]=f.itemClick,o.on({"contextmenu:hide.contextMenu":f.hideMenu,"prevcommand.contextMenu":f.prevItem,"nextcommand.contextMenu":f.nextItem,"contextmenu.contextMenu":f.abortevent,"mouseenter.contextMenu":f.menuMouseenter,"mouseleave.contextMenu":f.menuMouseleave},".context-menu-list").on("mouseup.contextMenu",".context-menu-input",f.inputClick).on(v,".context-menu-item"),s=!0}switch(i.on("contextmenu"+a.ns,a.selector,a,f.contextmenu),u&&i.on("remove"+a.ns,function(){e(this).contextMenu("destroy")}),a.trigger){case"hover":i.on("mouseenter"+a.ns,a.selector,a,f.mouseenter).on("mouseleave"+a.ns,a.selector,a,f.mouseleave);break;case"left":i.on("click"+a.ns,a.selector,a,f.click);break;case"touchstart":i.on("touchstart"+a.ns,a.selector,a,f.click)}a.build||h.create(a);break;case"destroy":var x;if(u){var g=a.context;e.each(r,function(t,n){if(!n)return!0;if(!e(g).is(n.selector))return!0;(x=e(".context-menu-list").filter(":visible")).length&&x.data().contextMenuRoot.$trigger.is(e(n.context).find(n.selector))&&x.trigger("contextmenu:hide",{force:!0});try{r[n.ns].$menu&&r[n.ns].$menu.remove(),delete r[n.ns]}catch(e){r[n.ns]=null}return e(n.context).off(n.ns),!0})}else if(a.selector){if(l[a.selector]){(x=e(".context-menu-list").filter(":visible")).length&&x.data().contextMenuRoot.$trigger.is(a.selector)&&x.trigger("contextmenu:hide",{force:!0});try{r[l[a.selector]].$menu&&r[l[a.selector]].$menu.remove(),delete r[l[a.selector]]}catch(e){r[l[a.selector]]=null}o.off(l[a.selector])}}else o.off(".contextMenu .contextMenuAutoHide"),e.each(r,function(t,n){e(n.context).off(n.ns)}),l={},r={},c=0,s=!1,e("#context-menu-layer, .context-menu-list").remove();break;case"html5":(!e.support.htmlCommand&&!e.support.htmlMenuitem||"boolean"==typeof n&&n)&&e('menu[type="context"]').each(function(){this.id&&e.contextMenu({selector:"[contextmenu="+this.id+"]",items:e.contextMenu.fromMenu(this)})}).css("display","none");break;default:throw new Error('Unknown operation "'+t+'"')}return this},e.contextMenu.setInputValues=function(t,n){void 0===n&&(n={}),e.each(t.inputs,function(e,t){switch(t.type){case"text":case"textarea":t.value=n[e]||"";break;case"checkbox":t.selected=!!n[e];break;case"radio":t.selected=(n[t.radio]||"")===t.value;break;case"select":t.selected=n[e]||""}})},e.contextMenu.getInputValues=function(t,n){return void 0===n&&(n={}),e.each(t.inputs,function(e,t){switch(t.type){case"text":case"textarea":case"select":n[e]=t.$input.val();break;case"checkbox":n[e]=t.$input.prop("checked");break;case"radio":t.$input.prop("checked")&&(n[t.radio]=t.value)}}),n},e.contextMenu.fromMenu=function(t){var n={};return a(n,e(t).children()),n},e.contextMenu.defaults=d,e.contextMenu.types=u,e.contextMenu.handle=f,e.contextMenu.op=h,e.contextMenu.menus=r}); -//# sourceMappingURL=jquery.contextMenu.min.js.map diff --git a/OpenRefine/extensions/database/module/scripts/index/jquery.ui.position.min.js b/OpenRefine/extensions/database/module/scripts/index/jquery.ui.position.min.js deleted file mode 100644 index 48d6a99b4..000000000 --- a/OpenRefine/extensions/database/module/scripts/index/jquery.ui.position.min.js +++ /dev/null @@ -1,6 +0,0 @@ -/*! jQuery UI - v1.12.1 - 2016-09-16 - * http://jqueryui.com - * Includes: position.js - * Copyright jQuery Foundation and other contributors; Licensed MIT */ - -(function(t){"function"==typeof define&&define.amd?define(["jquery"],t):t(jQuery)})(function(t){t.ui=t.ui||{},t.ui.version="1.12.1",function(){function e(t,e,i){return[parseFloat(t[0])*(u.test(t[0])?e/100:1),parseFloat(t[1])*(u.test(t[1])?i/100:1)]}function i(e,i){return parseInt(t.css(e,i),10)||0}function s(e){var i=e[0];return 9===i.nodeType?{width:e.width(),height:e.height(),offset:{top:0,left:0}}:t.isWindow(i)?{width:e.width(),height:e.height(),offset:{top:e.scrollTop(),left:e.scrollLeft()}}:i.preventDefault?{width:0,height:0,offset:{top:i.pageY,left:i.pageX}}:{width:e.outerWidth(),height:e.outerHeight(),offset:e.offset()}}var n,o=Math.max,a=Math.abs,r=/left|center|right/,l=/top|center|bottom/,h=/[\+\-]\d+(\.[\d]+)?%?/,c=/^\w+/,u=/%$/,d=t.fn.position;t.position={scrollbarWidth:function(){if(void 0!==n)return n;var e,i,s=t("
      "),o=s.children()[0];return t("body").append(s),e=o.offsetWidth,s.css("overflow","scroll"),i=o.offsetWidth,e===i&&(i=s[0].clientWidth),s.remove(),n=e-i},getScrollInfo:function(e){var i=e.isWindow||e.isDocument?"":e.element.css("overflow-x"),s=e.isWindow||e.isDocument?"":e.element.css("overflow-y"),n="scroll"===i||"auto"===i&&e.widthi?"left":e>0?"right":"center",vertical:0>r?"top":s>0?"bottom":"middle"};h>p&&p>a(e+i)&&(u.horizontal="center"),c>f&&f>a(s+r)&&(u.vertical="middle"),u.important=o(a(e),a(i))>o(a(s),a(r))?"horizontal":"vertical",n.using.call(this,t,u)}),l.offset(t.extend(D,{using:r}))})},t.ui.position={fit:{left:function(t,e){var i,s=e.within,n=s.isWindow?s.scrollLeft:s.offset.left,a=s.width,r=t.left-e.collisionPosition.marginLeft,l=n-r,h=r+e.collisionWidth-a-n;e.collisionWidth>a?l>0&&0>=h?(i=t.left+l+e.collisionWidth-a-n,t.left+=l-i):t.left=h>0&&0>=l?n:l>h?n+a-e.collisionWidth:n:l>0?t.left+=l:h>0?t.left-=h:t.left=o(t.left-r,t.left)},top:function(t,e){var i,s=e.within,n=s.isWindow?s.scrollTop:s.offset.top,a=e.within.height,r=t.top-e.collisionPosition.marginTop,l=n-r,h=r+e.collisionHeight-a-n;e.collisionHeight>a?l>0&&0>=h?(i=t.top+l+e.collisionHeight-a-n,t.top+=l-i):t.top=h>0&&0>=l?n:l>h?n+a-e.collisionHeight:n:l>0?t.top+=l:h>0?t.top-=h:t.top=o(t.top-r,t.top)}},flip:{left:function(t,e){var i,s,n=e.within,o=n.offset.left+n.scrollLeft,r=n.width,l=n.isWindow?n.scrollLeft:n.offset.left,h=t.left-e.collisionPosition.marginLeft,c=h-l,u=h+e.collisionWidth-r-l,d="left"===e.my[0]?-e.elemWidth:"right"===e.my[0]?e.elemWidth:0,p="left"===e.at[0]?e.targetWidth:"right"===e.at[0]?-e.targetWidth:0,f=-2*e.offset[0];0>c?(i=t.left+d+p+f+e.collisionWidth-r-o,(0>i||a(c)>i)&&(t.left+=d+p+f)):u>0&&(s=t.left-e.collisionPosition.marginLeft+d+p+f-l,(s>0||u>a(s))&&(t.left+=d+p+f))},top:function(t,e){var i,s,n=e.within,o=n.offset.top+n.scrollTop,r=n.height,l=n.isWindow?n.scrollTop:n.offset.top,h=t.top-e.collisionPosition.marginTop,c=h-l,u=h+e.collisionHeight-r-l,d="top"===e.my[1],p=d?-e.elemHeight:"bottom"===e.my[1]?e.elemHeight:0,f="top"===e.at[1]?e.targetHeight:"bottom"===e.at[1]?-e.targetHeight:0,g=-2*e.offset[1];0>c?(s=t.top+p+f+g+e.collisionHeight-r-o,(0>s||a(c)>s)&&(t.top+=p+f+g)):u>0&&(i=t.top-e.collisionPosition.marginTop+p+f+g-l,(i>0||u>a(i))&&(t.top+=p+f+g))}},flipfit:{left:function(){t.ui.position.flip.left.apply(this,arguments),t.ui.position.fit.left.apply(this,arguments)},top:function(){t.ui.position.flip.top.apply(this,arguments),t.ui.position.fit.top.apply(this,arguments)}}}}(),t.ui.position}); \ No newline at end of file diff --git a/OpenRefine/extensions/database/module/scripts/project/database-exporters.js b/OpenRefine/extensions/database/module/scripts/project/database-exporters.js deleted file mode 100644 index 5f11b3e5b..000000000 --- a/OpenRefine/extensions/database/module/scripts/project/database-exporters.js +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -var dictionary = ""; -$.ajax({ - url : "command/core/load-language?", - type : "POST", - async : false, - data : { - module : "database", - - }, - success : function(data) { - dictionary = data['dictionary']; - lang = data['lang']; - } -}); -$.i18n().load(dictionary, lang); -// End internationalization - -(function() { - -})(); diff --git a/OpenRefine/extensions/database/module/styles/bootstrap.css b/OpenRefine/extensions/database/module/styles/bootstrap.css deleted file mode 100644 index 39e2a1597..000000000 --- a/OpenRefine/extensions/database/module/styles/bootstrap.css +++ /dev/null @@ -1,1701 +0,0 @@ -/*! - * Bootstrap v3.3.7 (http://getbootstrap.com) - * Copyright 2011-2017 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - */ - -/*! - * Generated using the Bootstrap Customizer (http://getbootstrap.com/docs/3.3/customize/?id=3426e25f732fc7882717e871fd08de17) - * Config saved to config.json and https://gist.github.com/3426e25f732fc7882717e871fd08de17 - */ -/*! - * Bootstrap v3.3.7 (http://getbootstrap.com) - * Copyright 2011-2016 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - */ -/*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */ - -.panel { - margin-bottom: 20px; - background-color: #ffffff; - border: 1px solid transparent; - border-radius: 4px; - -webkit-box-shadow: 0 1px 1px rgba(0, 0, 0, 0.05); - box-shadow: 0 1px 1px rgba(0, 0, 0, 0.05); -} -.panel-body { - padding: 15px; -} -.panel-heading { - padding: 10px 15px; - border-bottom: 1px solid transparent; - border-top-right-radius: 3px; - border-top-left-radius: 3px; -} -.panel-heading > .dropdown .dropdown-toggle { - color: inherit; -} -.panel-title { - margin-top: 0; - margin-bottom: 0; - font-size: 16px; - color: inherit; -} -.panel-title > a, -.panel-title > small, -.panel-title > .small, -.panel-title > small > a, -.panel-title > .small > a { - color: inherit; -} -.panel-footer { - padding: 10px 15px; - background-color: #f5f5f5; - border-top: 1px solid #dddddd; - border-bottom-right-radius: 3px; - border-bottom-left-radius: 3px; -} -.panel > .list-group, -.panel > .panel-collapse > .list-group { - margin-bottom: 0; -} -.panel > .list-group .list-group-item, -.panel > .panel-collapse > .list-group .list-group-item { - border-width: 1px 0; - border-radius: 0; -} -.panel > .list-group:first-child .list-group-item:first-child, -.panel > .panel-collapse > .list-group:first-child .list-group-item:first-child { - border-top: 0; - border-top-right-radius: 3px; - border-top-left-radius: 3px; -} -.panel > .list-group:last-child .list-group-item:last-child, -.panel > .panel-collapse > .list-group:last-child .list-group-item:last-child { - border-bottom: 0; - border-bottom-right-radius: 3px; - border-bottom-left-radius: 3px; -} -.panel > .panel-heading + .panel-collapse > .list-group .list-group-item:first-child { - border-top-right-radius: 0; - border-top-left-radius: 0; -} -.panel-heading + .list-group .list-group-item:first-child { - border-top-width: 0; -} -.list-group + .panel-footer { - border-top-width: 0; -} -.panel > .table, -.panel > .table-responsive > .table, -.panel > .panel-collapse > .table { - margin-bottom: 0; -} -.panel > .table caption, -.panel > .table-responsive > .table caption, -.panel > .panel-collapse > .table caption { - padding-left: 15px; - padding-right: 15px; -} -.panel > .table:first-child, -.panel > .table-responsive:first-child > .table:first-child { - border-top-right-radius: 3px; - border-top-left-radius: 3px; -} -.panel > .table:first-child > thead:first-child > tr:first-child, -.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child, -.panel > .table:first-child > tbody:first-child > tr:first-child, -.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child { - border-top-left-radius: 3px; - border-top-right-radius: 3px; -} -.panel > .table:first-child > thead:first-child > tr:first-child td:first-child, -.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child td:first-child, -.panel > .table:first-child > tbody:first-child > tr:first-child td:first-child, -.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child td:first-child, -.panel > .table:first-child > thead:first-child > tr:first-child th:first-child, -.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child th:first-child, -.panel > .table:first-child > tbody:first-child > tr:first-child th:first-child, -.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child th:first-child { - border-top-left-radius: 3px; -} -.panel > .table:first-child > thead:first-child > tr:first-child td:last-child, -.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child td:last-child, -.panel > .table:first-child > tbody:first-child > tr:first-child td:last-child, -.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child td:last-child, -.panel > .table:first-child > thead:first-child > tr:first-child th:last-child, -.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child th:last-child, -.panel > .table:first-child > tbody:first-child > tr:first-child th:last-child, -.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child th:last-child { - border-top-right-radius: 3px; -} -.panel > .table:last-child, -.panel > .table-responsive:last-child > .table:last-child { - border-bottom-right-radius: 3px; - border-bottom-left-radius: 3px; -} -.panel > .table:last-child > tbody:last-child > tr:last-child, -.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child, -.panel > .table:last-child > tfoot:last-child > tr:last-child, -.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child { - border-bottom-left-radius: 3px; - border-bottom-right-radius: 3px; -} -.panel > .table:last-child > tbody:last-child > tr:last-child td:first-child, -.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child td:first-child, -.panel > .table:last-child > tfoot:last-child > tr:last-child td:first-child, -.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child td:first-child, -.panel > .table:last-child > tbody:last-child > tr:last-child th:first-child, -.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child th:first-child, -.panel > .table:last-child > tfoot:last-child > tr:last-child th:first-child, -.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child th:first-child { - border-bottom-left-radius: 3px; -} -.panel > .table:last-child > tbody:last-child > tr:last-child td:last-child, -.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child td:last-child, -.panel > .table:last-child > tfoot:last-child > tr:last-child td:last-child, -.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child td:last-child, -.panel > .table:last-child > tbody:last-child > tr:last-child th:last-child, -.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child th:last-child, -.panel > .table:last-child > tfoot:last-child > tr:last-child th:last-child, -.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child th:last-child { - border-bottom-right-radius: 3px; -} -.panel > .panel-body + .table, -.panel > .panel-body + .table-responsive, -.panel > .table + .panel-body, -.panel > .table-responsive + .panel-body { - border-top: 1px solid #dddddd; -} -.panel > .table > tbody:first-child > tr:first-child th, -.panel > .table > tbody:first-child > tr:first-child td { - border-top: 0; -} -.panel > .table-bordered, -.panel > .table-responsive > .table-bordered { - border: 0; -} -.panel > .table-bordered > thead > tr > th:first-child, -.panel > .table-responsive > .table-bordered > thead > tr > th:first-child, -.panel > .table-bordered > tbody > tr > th:first-child, -.panel > .table-responsive > .table-bordered > tbody > tr > th:first-child, -.panel > .table-bordered > tfoot > tr > th:first-child, -.panel > .table-responsive > .table-bordered > tfoot > tr > th:first-child, -.panel > .table-bordered > thead > tr > td:first-child, -.panel > .table-responsive > .table-bordered > thead > tr > td:first-child, -.panel > .table-bordered > tbody > tr > td:first-child, -.panel > .table-responsive > .table-bordered > tbody > tr > td:first-child, -.panel > .table-bordered > tfoot > tr > td:first-child, -.panel > .table-responsive > .table-bordered > tfoot > tr > td:first-child { - border-left: 0; -} -.panel > .table-bordered > thead > tr > th:last-child, -.panel > .table-responsive > .table-bordered > thead > tr > th:last-child, -.panel > .table-bordered > tbody > tr > th:last-child, -.panel > .table-responsive > .table-bordered > tbody > tr > th:last-child, -.panel > .table-bordered > tfoot > tr > th:last-child, -.panel > .table-responsive > .table-bordered > tfoot > tr > th:last-child, -.panel > .table-bordered > thead > tr > td:last-child, -.panel > .table-responsive > .table-bordered > thead > tr > td:last-child, -.panel > .table-bordered > tbody > tr > td:last-child, -.panel > .table-responsive > .table-bordered > tbody > tr > td:last-child, -.panel > .table-bordered > tfoot > tr > td:last-child, -.panel > .table-responsive > .table-bordered > tfoot > tr > td:last-child { - border-right: 0; -} -.panel > .table-bordered > thead > tr:first-child > td, -.panel > .table-responsive > .table-bordered > thead > tr:first-child > td, -.panel > .table-bordered > tbody > tr:first-child > td, -.panel > .table-responsive > .table-bordered > tbody > tr:first-child > td, -.panel > .table-bordered > thead > tr:first-child > th, -.panel > .table-responsive > .table-bordered > thead > tr:first-child > th, -.panel > .table-bordered > tbody > tr:first-child > th, -.panel > .table-responsive > .table-bordered > tbody > tr:first-child > th { - border-bottom: 0; -} -.panel > .table-bordered > tbody > tr:last-child > td, -.panel > .table-responsive > .table-bordered > tbody > tr:last-child > td, -.panel > .table-bordered > tfoot > tr:last-child > td, -.panel > .table-responsive > .table-bordered > tfoot > tr:last-child > td, -.panel > .table-bordered > tbody > tr:last-child > th, -.panel > .table-responsive > .table-bordered > tbody > tr:last-child > th, -.panel > .table-bordered > tfoot > tr:last-child > th, -.panel > .table-responsive > .table-bordered > tfoot > tr:last-child > th { - border-bottom: 0; -} -.panel > .table-responsive { - border: 0; - margin-bottom: 0; -} -.panel-group { - margin-bottom: 20px; -} -.panel-group .panel { - margin-bottom: 0; - border-radius: 4px; -} -.panel-group .panel + .panel { - margin-top: 5px; -} -.panel-group .panel-heading { - border-bottom: 0; -} -.panel-group .panel-heading + .panel-collapse > .panel-body, -.panel-group .panel-heading + .panel-collapse > .list-group { - border-top: 1px solid #dddddd; -} -.panel-group .panel-footer { - border-top: 0; -} -.panel-group .panel-footer + .panel-collapse .panel-body { - border-bottom: 1px solid #dddddd; -} -.panel-default { - border-color: #dddddd; -} -.panel-default > .panel-heading { - color: #333333; - background-color: #f5f5f5; - border-color: #dddddd; -} -.panel-default > .panel-heading + .panel-collapse > .panel-body { - border-top-color: #dddddd; -} -.panel-default > .panel-heading .badge { - color: #f5f5f5; - background-color: #333333; -} -.panel-default > .panel-footer + .panel-collapse > .panel-body { - border-bottom-color: #dddddd; -} -.panel-primary { - border-color: #337ab7; -} -.panel-primary > .panel-heading { - color: #ffffff; - background-color: #337ab7; - border-color: #337ab7; -} -.panel-primary > .panel-heading + .panel-collapse > .panel-body { - border-top-color: #337ab7; -} -.panel-primary > .panel-heading .badge { - color: #337ab7; - background-color: #ffffff; -} -.panel-primary > .panel-footer + .panel-collapse > .panel-body { - border-bottom-color: #337ab7; -} -.panel-success { - border-color: #d6e9c6; -} -.panel-success > .panel-heading { - color: #3c763d; - background-color: #dff0d8; - border-color: #d6e9c6; -} -.panel-success > .panel-heading + .panel-collapse > .panel-body { - border-top-color: #d6e9c6; -} -.panel-success > .panel-heading .badge { - color: #dff0d8; - background-color: #3c763d; -} -.panel-success > .panel-footer + .panel-collapse > .panel-body { - border-bottom-color: #d6e9c6; -} -.panel-info { - border-color: #bce8f1; -} -.panel-info > .panel-heading { - color: #31708f; - background-color: #d9edf7; - border-color: #bce8f1; -} -.panel-info > .panel-heading + .panel-collapse > .panel-body { - border-top-color: #bce8f1; -} -.panel-info > .panel-heading .badge { - color: #d9edf7; - background-color: #31708f; -} -.panel-info > .panel-footer + .panel-collapse > .panel-body { - border-bottom-color: #bce8f1; -} -.panel-warning { - border-color: #faebcc; -} -.panel-warning > .panel-heading { - color: #8a6d3b; - background-color: #fcf8e3; - border-color: #faebcc; -} -.panel-warning > .panel-heading + .panel-collapse > .panel-body { - border-top-color: #faebcc; -} -.panel-warning > .panel-heading .badge { - color: #fcf8e3; - background-color: #8a6d3b; -} -.panel-warning > .panel-footer + .panel-collapse > .panel-body { - border-bottom-color: #faebcc; -} -.panel-danger { - border-color: #ebccd1; -} -.panel-danger > .panel-heading { - color: #a94442; - background-color: #f2dede; - border-color: #ebccd1; -} -.panel-danger > .panel-heading + .panel-collapse > .panel-body { - border-top-color: #ebccd1; -} -.panel-danger > .panel-heading .badge { - color: #f2dede; - background-color: #a94442; -} -.panel-danger > .panel-footer + .panel-collapse > .panel-body { - border-bottom-color: #ebccd1; -} -.clearfix:before, -.clearfix:after, -.nav:before, -.nav:after, -.panel-body:before, -.panel-body:after { - content: " "; - display: table; -} -.clearfix:after, -.nav:after, -.panel-body:after { - clear: both; -} -.center-block { - display: block; - margin-left: auto; - margin-right: auto; -} -.pull-right { - float: right !important; -} -.pull-left { - float: left !important; -} -.hide { - display: none !important; -} -.show { - display: block !important; -} -.invisible { - visibility: hidden; -} -.text-hide { - font: 0/0 a; - color: transparent; - text-shadow: none; - background-color: transparent; - border: 0; -} -.hidden { - display: none !important; -} -.affix { - position: fixed; -} - -.alert { - padding: 15px; - margin-bottom: 20px; - border: 1px solid transparent; - border-radius: 4px; -} -.alert h4 { - margin-top: 0; - color: inherit; -} -.alert .alert-link { - font-weight: bold; -} -.alert > p, -.alert > ul { - margin-bottom: 0; -} -.alert > p + p { - margin-top: 5px; -} -.alert-dismissable, -.alert-dismissible { - padding-right: 35px; -} -.alert-dismissable .close, -.alert-dismissible .close { - position: relative; - top: -2px; - right: -21px; - color: inherit; -} -.alert-success { - background-color: #dff0d8; - border-color: #d6e9c6; - color: #3c763d; -} -.alert-success hr { - border-top-color: #c9e2b3; -} -.alert-success .alert-link { - color: #2b542c; -} -.alert-info { - background-color: #d9edf7; - border-color: #bce8f1; - color: #31708f; -} -.alert-info hr { - border-top-color: #a6e1ec; -} -.alert-info .alert-link { - color: #245269; -} -.alert-warning { - background-color: #fcf8e3; - border-color: #faebcc; - color: #8a6d3b; -} -.alert-warning hr { - border-top-color: #f7e1b5; -} -.alert-warning .alert-link { - color: #66512c; -} -.alert-danger { - background-color: #f2dede; - border-color: #ebccd1; - color: #a94442; -} -.alert-danger hr { - border-top-color: #e4b9c0; -} -.alert-danger .alert-link { - color: #843534; -} - -@font-face { - font-family: 'Glyphicons Halflings'; - src: url('../images/fonts/glyphicons-halflings-regular.eot'); - src: url('../images/fonts/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'), url('../images/fonts/glyphicons-halflings-regular.woff2') format('woff2'), url('../images/fonts/glyphicons-halflings-regular.woff') format('woff'), url('../images/fonts/glyphicons-halflings-regular.ttf') format('truetype'), url('../images/fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg'); -} -.glyphicon { - position: relative; - top: 1px; - display: inline-block; - font-family: 'Glyphicons Halflings'; - font-style: normal; - font-weight: normal; - line-height: 1; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; -} -.glyphicon-asterisk:before { - content: "\002a"; -} -.glyphicon-plus:before { - content: "\002b"; -} -.glyphicon-euro:before, -.glyphicon-eur:before { - content: "\20ac"; -} -.glyphicon-minus:before { - content: "\2212"; -} -.glyphicon-cloud:before { - content: "\2601"; -} -.glyphicon-envelope:before { - content: "\2709"; -} -.glyphicon-pencil:before { - content: "\270f"; -} -.glyphicon-glass:before { - content: "\e001"; -} -.glyphicon-music:before { - content: "\e002"; -} -.glyphicon-search:before { - content: "\e003"; -} -.glyphicon-heart:before { - content: "\e005"; -} -.glyphicon-star:before { - content: "\e006"; -} -.glyphicon-star-empty:before { - content: "\e007"; -} -.glyphicon-user:before { - content: "\e008"; -} -.glyphicon-film:before { - content: "\e009"; -} -.glyphicon-th-large:before { - content: "\e010"; -} -.glyphicon-th:before { - content: "\e011"; -} -.glyphicon-th-list:before { - content: "\e012"; -} -.glyphicon-ok:before { - content: "\e013"; -} -.glyphicon-remove:before { - content: "\e014"; -} -.glyphicon-zoom-in:before { - content: "\e015"; -} -.glyphicon-zoom-out:before { - content: "\e016"; -} -.glyphicon-off:before { - content: "\e017"; -} -.glyphicon-signal:before { - content: "\e018"; -} -.glyphicon-cog:before { - content: "\e019"; -} -.glyphicon-trash:before { - content: "\e020"; -} -.glyphicon-home:before { - content: "\e021"; -} -.glyphicon-file:before { - content: "\e022"; -} -.glyphicon-time:before { - content: "\e023"; -} -.glyphicon-road:before { - content: "\e024"; -} -.glyphicon-download-alt:before { - content: "\e025"; -} -.glyphicon-download:before { - content: "\e026"; -} -.glyphicon-upload:before { - content: "\e027"; -} -.glyphicon-inbox:before { - content: "\e028"; -} -.glyphicon-play-circle:before { - content: "\e029"; -} -.glyphicon-repeat:before { - content: "\e030"; -} -.glyphicon-refresh:before { - content: "\e031"; -} -.glyphicon-list-alt:before { - content: "\e032"; -} -.glyphicon-lock:before { - content: "\e033"; -} -.glyphicon-flag:before { - content: "\e034"; -} -.glyphicon-headphones:before { - content: "\e035"; -} -.glyphicon-volume-off:before { - content: "\e036"; -} -.glyphicon-volume-down:before { - content: "\e037"; -} -.glyphicon-volume-up:before { - content: "\e038"; -} -.glyphicon-qrcode:before { - content: "\e039"; -} -.glyphicon-barcode:before { - content: "\e040"; -} -.glyphicon-tag:before { - content: "\e041"; -} -.glyphicon-tags:before { - content: "\e042"; -} -.glyphicon-book:before { - content: "\e043"; -} -.glyphicon-bookmark:before { - content: "\e044"; -} -.glyphicon-print:before { - content: "\e045"; -} -.glyphicon-camera:before { - content: "\e046"; -} -.glyphicon-font:before { - content: "\e047"; -} -.glyphicon-bold:before { - content: "\e048"; -} -.glyphicon-italic:before { - content: "\e049"; -} -.glyphicon-text-height:before { - content: "\e050"; -} -.glyphicon-text-width:before { - content: "\e051"; -} -.glyphicon-align-left:before { - content: "\e052"; -} -.glyphicon-align-center:before { - content: "\e053"; -} -.glyphicon-align-right:before { - content: "\e054"; -} -.glyphicon-align-justify:before { - content: "\e055"; -} -.glyphicon-list:before { - content: "\e056"; -} -.glyphicon-indent-left:before { - content: "\e057"; -} -.glyphicon-indent-right:before { - content: "\e058"; -} -.glyphicon-facetime-video:before { - content: "\e059"; -} -.glyphicon-picture:before { - content: "\e060"; -} -.glyphicon-map-marker:before { - content: "\e062"; -} -.glyphicon-adjust:before { - content: "\e063"; -} -.glyphicon-tint:before { - content: "\e064"; -} -.glyphicon-edit:before { - content: "\e065"; -} -.glyphicon-share:before { - content: "\e066"; -} -.glyphicon-check:before { - content: "\e067"; -} -.glyphicon-move:before { - content: "\e068"; -} -.glyphicon-step-backward:before { - content: "\e069"; -} -.glyphicon-fast-backward:before { - content: "\e070"; -} -.glyphicon-backward:before { - content: "\e071"; -} -.glyphicon-play:before { - content: "\e072"; -} -.glyphicon-pause:before { - content: "\e073"; -} -.glyphicon-stop:before { - content: "\e074"; -} -.glyphicon-forward:before { - content: "\e075"; -} -.glyphicon-fast-forward:before { - content: "\e076"; -} -.glyphicon-step-forward:before { - content: "\e077"; -} -.glyphicon-eject:before { - content: "\e078"; -} -.glyphicon-chevron-left:before { - content: "\e079"; -} -.glyphicon-chevron-right:before { - content: "\e080"; -} -.glyphicon-plus-sign:before { - content: "\e081"; -} -.glyphicon-minus-sign:before { - content: "\e082"; -} -.glyphicon-remove-sign:before { - content: "\e083"; -} -.glyphicon-ok-sign:before { - content: "\e084"; -} -.glyphicon-question-sign:before { - content: "\e085"; -} -.glyphicon-info-sign:before { - content: "\e086"; -} -.glyphicon-screenshot:before { - content: "\e087"; -} -.glyphicon-remove-circle:before { - content: "\e088"; -} -.glyphicon-ok-circle:before { - content: "\e089"; -} -.glyphicon-ban-circle:before { - content: "\e090"; -} -.glyphicon-arrow-left:before { - content: "\e091"; -} -.glyphicon-arrow-right:before { - content: "\e092"; -} -.glyphicon-arrow-up:before { - content: "\e093"; -} -.glyphicon-arrow-down:before { - content: "\e094"; -} -.glyphicon-share-alt:before { - content: "\e095"; -} -.glyphicon-resize-full:before { - content: "\e096"; -} -.glyphicon-resize-small:before { - content: "\e097"; -} -.glyphicon-exclamation-sign:before { - content: "\e101"; -} -.glyphicon-gift:before { - content: "\e102"; -} -.glyphicon-leaf:before { - content: "\e103"; -} -.glyphicon-fire:before { - content: "\e104"; -} -.glyphicon-eye-open:before { - content: "\e105"; -} -.glyphicon-eye-close:before { - content: "\e106"; -} -.glyphicon-warning-sign:before { - content: "\e107"; -} -.glyphicon-plane:before { - content: "\e108"; -} -.glyphicon-calendar:before { - content: "\e109"; -} -.glyphicon-random:before { - content: "\e110"; -} -.glyphicon-comment:before { - content: "\e111"; -} -.glyphicon-magnet:before { - content: "\e112"; -} -.glyphicon-chevron-up:before { - content: "\e113"; -} -.glyphicon-chevron-down:before { - content: "\e114"; -} -.glyphicon-retweet:before { - content: "\e115"; -} -.glyphicon-shopping-cart:before { - content: "\e116"; -} -.glyphicon-folder-close:before { - content: "\e117"; -} -.glyphicon-folder-open:before { - content: "\e118"; -} -.glyphicon-resize-vertical:before { - content: "\e119"; -} -.glyphicon-resize-horizontal:before { - content: "\e120"; -} -.glyphicon-hdd:before { - content: "\e121"; -} -.glyphicon-bullhorn:before { - content: "\e122"; -} -.glyphicon-bell:before { - content: "\e123"; -} -.glyphicon-certificate:before { - content: "\e124"; -} -.glyphicon-thumbs-up:before { - content: "\e125"; -} -.glyphicon-thumbs-down:before { - content: "\e126"; -} -.glyphicon-hand-right:before { - content: "\e127"; -} -.glyphicon-hand-left:before { - content: "\e128"; -} -.glyphicon-hand-up:before { - content: "\e129"; -} -.glyphicon-hand-down:before { - content: "\e130"; -} -.glyphicon-circle-arrow-right:before { - content: "\e131"; -} -.glyphicon-circle-arrow-left:before { - content: "\e132"; -} -.glyphicon-circle-arrow-up:before { - content: "\e133"; -} -.glyphicon-circle-arrow-down:before { - content: "\e134"; -} -.glyphicon-globe:before { - content: "\e135"; -} -.glyphicon-wrench:before { - content: "\e136"; -} -.glyphicon-tasks:before { - content: "\e137"; -} -.glyphicon-filter:before { - content: "\e138"; -} -.glyphicon-briefcase:before { - content: "\e139"; -} -.glyphicon-fullscreen:before { - content: "\e140"; -} -.glyphicon-dashboard:before { - content: "\e141"; -} -.glyphicon-paperclip:before { - content: "\e142"; -} -.glyphicon-heart-empty:before { - content: "\e143"; -} -.glyphicon-link:before { - content: "\e144"; -} -.glyphicon-phone:before { - content: "\e145"; -} -.glyphicon-pushpin:before { - content: "\e146"; -} -.glyphicon-usd:before { - content: "\e148"; -} -.glyphicon-gbp:before { - content: "\e149"; -} -.glyphicon-sort:before { - content: "\e150"; -} -.glyphicon-sort-by-alphabet:before { - content: "\e151"; -} -.glyphicon-sort-by-alphabet-alt:before { - content: "\e152"; -} -.glyphicon-sort-by-order:before { - content: "\e153"; -} -.glyphicon-sort-by-order-alt:before { - content: "\e154"; -} -.glyphicon-sort-by-attributes:before { - content: "\e155"; -} -.glyphicon-sort-by-attributes-alt:before { - content: "\e156"; -} -.glyphicon-unchecked:before { - content: "\e157"; -} -.glyphicon-expand:before { - content: "\e158"; -} -.glyphicon-collapse-down:before { - content: "\e159"; -} -.glyphicon-collapse-up:before { - content: "\e160"; -} -.glyphicon-log-in:before { - content: "\e161"; -} -.glyphicon-flash:before { - content: "\e162"; -} -.glyphicon-log-out:before { - content: "\e163"; -} -.glyphicon-new-window:before { - content: "\e164"; -} -.glyphicon-record:before { - content: "\e165"; -} -.glyphicon-save:before { - content: "\e166"; -} -.glyphicon-open:before { - content: "\e167"; -} -.glyphicon-saved:before { - content: "\e168"; -} -.glyphicon-import:before { - content: "\e169"; -} -.glyphicon-export:before { - content: "\e170"; -} -.glyphicon-send:before { - content: "\e171"; -} -.glyphicon-floppy-disk:before { - content: "\e172"; -} -.glyphicon-floppy-saved:before { - content: "\e173"; -} -.glyphicon-floppy-remove:before { - content: "\e174"; -} -.glyphicon-floppy-save:before { - content: "\e175"; -} -.glyphicon-floppy-open:before { - content: "\e176"; -} -.glyphicon-credit-card:before { - content: "\e177"; -} -.glyphicon-transfer:before { - content: "\e178"; -} -.glyphicon-cutlery:before { - content: "\e179"; -} -.glyphicon-header:before { - content: "\e180"; -} -.glyphicon-compressed:before { - content: "\e181"; -} -.glyphicon-earphone:before { - content: "\e182"; -} -.glyphicon-phone-alt:before { - content: "\e183"; -} -.glyphicon-tower:before { - content: "\e184"; -} -.glyphicon-stats:before { - content: "\e185"; -} -.glyphicon-sd-video:before { - content: "\e186"; -} -.glyphicon-hd-video:before { - content: "\e187"; -} -.glyphicon-subtitles:before { - content: "\e188"; -} -.glyphicon-sound-stereo:before { - content: "\e189"; -} -.glyphicon-sound-dolby:before { - content: "\e190"; -} -.glyphicon-sound-5-1:before { - content: "\e191"; -} -.glyphicon-sound-6-1:before { - content: "\e192"; -} -.glyphicon-sound-7-1:before { - content: "\e193"; -} -.glyphicon-copyright-mark:before { - content: "\e194"; -} -.glyphicon-registration-mark:before { - content: "\e195"; -} -.glyphicon-cloud-download:before { - content: "\e197"; -} -.glyphicon-cloud-upload:before { - content: "\e198"; -} -.glyphicon-tree-conifer:before { - content: "\e199"; -} -.glyphicon-tree-deciduous:before { - content: "\e200"; -} -.glyphicon-cd:before { - content: "\e201"; -} -.glyphicon-save-file:before { - content: "\e202"; -} -.glyphicon-open-file:before { - content: "\e203"; -} -.glyphicon-level-up:before { - content: "\e204"; -} -.glyphicon-copy:before { - content: "\e205"; -} -.glyphicon-paste:before { - content: "\e206"; -} -.glyphicon-alert:before { - content: "\e209"; -} -.glyphicon-equalizer:before { - content: "\e210"; -} -.glyphicon-king:before { - content: "\e211"; -} -.glyphicon-queen:before { - content: "\e212"; -} -.glyphicon-pawn:before { - content: "\e213"; -} -.glyphicon-bishop:before { - content: "\e214"; -} -.glyphicon-knight:before { - content: "\e215"; -} -.glyphicon-baby-formula:before { - content: "\e216"; -} -.glyphicon-tent:before { - content: "\26fa"; -} -.glyphicon-blackboard:before { - content: "\e218"; -} -.glyphicon-bed:before { - content: "\e219"; -} -.glyphicon-apple:before { - content: "\f8ff"; -} -.glyphicon-erase:before { - content: "\e221"; -} -.glyphicon-hourglass:before { - content: "\231b"; -} -.glyphicon-lamp:before { - content: "\e223"; -} -.glyphicon-duplicate:before { - content: "\e224"; -} -.glyphicon-piggy-bank:before { - content: "\e225"; -} -.glyphicon-scissors:before { - content: "\e226"; -} -.glyphicon-bitcoin:before { - content: "\e227"; -} -.glyphicon-btc:before { - content: "\e227"; -} -.glyphicon-xbt:before { - content: "\e227"; -} -.glyphicon-yen:before { - content: "\00a5"; -} -.glyphicon-jpy:before { - content: "\00a5"; -} -.glyphicon-ruble:before { - content: "\20bd"; -} -.glyphicon-rub:before { - content: "\20bd"; -} -.glyphicon-scale:before { - content: "\e230"; -} -.glyphicon-ice-lolly:before { - content: "\e231"; -} -.glyphicon-ice-lolly-tasted:before { - content: "\e232"; -} -.glyphicon-education:before { - content: "\e233"; -} -.glyphicon-option-horizontal:before { - content: "\e234"; -} -.glyphicon-option-vertical:before { - content: "\e235"; -} -.glyphicon-menu-hamburger:before { - content: "\e236"; -} -.glyphicon-modal-window:before { - content: "\e237"; -} -.glyphicon-oil:before { - content: "\e238"; -} -.glyphicon-grain:before { - content: "\e239"; -} -.glyphicon-sunglasses:before { - content: "\e240"; -} -.glyphicon-text-size:before { - content: "\e241"; -} -.glyphicon-text-color:before { - content: "\e242"; -} -.glyphicon-text-background:before { - content: "\e243"; -} -.glyphicon-object-align-top:before { - content: "\e244"; -} -.glyphicon-object-align-bottom:before { - content: "\e245"; -} -.glyphicon-object-align-horizontal:before { - content: "\e246"; -} -.glyphicon-object-align-left:before { - content: "\e247"; -} -.glyphicon-object-align-vertical:before { - content: "\e248"; -} -.glyphicon-object-align-right:before { - content: "\e249"; -} -.glyphicon-triangle-right:before { - content: "\e250"; -} -.glyphicon-triangle-left:before { - content: "\e251"; -} -.glyphicon-triangle-bottom:before { - content: "\e252"; -} -.glyphicon-triangle-top:before { - content: "\e253"; -} -.glyphicon-console:before { - content: "\e254"; -} -.glyphicon-superscript:before { - content: "\e255"; -} -.glyphicon-subscript:before { - content: "\e256"; -} -.glyphicon-menu-left:before { - content: "\e257"; -} -.glyphicon-menu-right:before { - content: "\e258"; -} -.glyphicon-menu-down:before { - content: "\e259"; -} -.glyphicon-menu-up:before { - content: "\e260"; -} - -.nav { - margin-bottom: 0; - padding-left: 0; - list-style: none; -} -.nav > li { - position: relative; - display: block; -} -.nav > li > a { - position: relative; - display: block; - padding: 10px 15px; -} -.nav > li > a:hover, -.nav > li > a:focus { - text-decoration: none; - background-color: #eeeeee; -} -.nav > li.disabled > a { - color: #777777; -} -.nav > li.disabled > a:hover, -.nav > li.disabled > a:focus { - color: #777777; - text-decoration: none; - background-color: transparent; - cursor: not-allowed; -} -.nav .open > a, -.nav .open > a:hover, -.nav .open > a:focus { - background-color: #eeeeee; - border-color: #337ab7; -} -.nav .nav-divider { - height: 1px; - margin: 9px 0; - overflow: hidden; - background-color: #e5e5e5; -} -.nav > li > a > img { - max-width: none; -} -.nav-tabs { - border-bottom: 1px solid #dddddd; -} -.nav-tabs > li { - float: left; - margin-bottom: -1px; -} -.nav-tabs > li > a { - margin-right: 2px; - line-height: 1.42857143; - border: 1px solid transparent; - border-radius: 4px 4px 0 0; -} -.nav-tabs > li > a:hover { - border-color: #eeeeee #eeeeee #dddddd; -} -.nav-tabs > li.active > a, -.nav-tabs > li.active > a:hover, -.nav-tabs > li.active > a:focus { - color: #555555; - background-color: #ffffff; - border: 1px solid #dddddd; - border-bottom-color: transparent; - cursor: default; -} -.nav-tabs.nav-justified { - width: 100%; - border-bottom: 0; -} -.nav-tabs.nav-justified > li { - float: none; -} -.nav-tabs.nav-justified > li > a { - text-align: center; - margin-bottom: 5px; -} -.nav-tabs.nav-justified > .dropdown .dropdown-menu { - top: auto; - left: auto; -} -@media (min-width: 768px) { - .nav-tabs.nav-justified > li { - display: table-cell; - width: 1%; - } - .nav-tabs.nav-justified > li > a { - margin-bottom: 0; - } -} -.nav-tabs.nav-justified > li > a { - margin-right: 0; - border-radius: 4px; -} -.nav-tabs.nav-justified > .active > a, -.nav-tabs.nav-justified > .active > a:hover, -.nav-tabs.nav-justified > .active > a:focus { - border: 1px solid #dddddd; -} -@media (min-width: 768px) { - .nav-tabs.nav-justified > li > a { - border-bottom: 1px solid #dddddd; - border-radius: 4px 4px 0 0; - } - .nav-tabs.nav-justified > .active > a, - .nav-tabs.nav-justified > .active > a:hover, - .nav-tabs.nav-justified > .active > a:focus { - border-bottom-color: #ffffff; - } -} -.nav-pills > li { - float: left; -} -.nav-pills > li > a { - border-radius: 4px; -} -.nav-pills > li + li { - margin-left: 2px; -} -.nav-pills > li.active > a, -.nav-pills > li.active > a:hover, -.nav-pills > li.active > a:focus { - color: #ffffff; - background-color: #337ab7; -} -.nav-stacked > li { - float: none; -} -.nav-stacked > li + li { - margin-top: 2px; - margin-left: 0; -} -.nav-justified { - width: 100%; -} -.nav-justified > li { - float: none; -} -.nav-justified > li > a { - text-align: center; - margin-bottom: 5px; -} -.nav-justified > .dropdown .dropdown-menu { - top: auto; - left: auto; -} -@media (min-width: 768px) { - .nav-justified > li { - display: table-cell; - width: 1%; - } - .nav-justified > li > a { - margin-bottom: 0; - } -} -.nav-tabs-justified { - border-bottom: 0; -} -.nav-tabs-justified > li > a { - margin-right: 0; - border-radius: 4px; -} -.nav-tabs-justified > .active > a, -.nav-tabs-justified > .active > a:hover, -.nav-tabs-justified > .active > a:focus { - border: 1px solid #dddddd; -} -@media (min-width: 768px) { - .nav-tabs-justified > li > a { - border-bottom: 1px solid #dddddd; - border-radius: 4px 4px 0 0; - } - .nav-tabs-justified > .active > a, - .nav-tabs-justified > .active > a:hover, - .nav-tabs-justified > .active > a:focus { - border-bottom-color: #ffffff; - } -} -/* -* List Group -*/ -.list-group { - margin-bottom: 20px; - padding-left: 0; -} -.list-group-item { - position: relative; - display: block; - padding: 10px 15px; - margin-bottom: -1px; - background-color: #ffffff; - border: 1px solid #dddddd; -} -.list-group-item:first-child { - border-top-right-radius: 4px; - border-top-left-radius: 4px; -} -.list-group-item:last-child { - margin-bottom: 0; - border-bottom-right-radius: 4px; - border-bottom-left-radius: 4px; -} -a.list-group-item, -button.list-group-item { - color: #555555; -} -a.list-group-item .list-group-item-heading, -button.list-group-item .list-group-item-heading { - color: #333333; -} -a.list-group-item:hover, -button.list-group-item:hover, -a.list-group-item:focus, -button.list-group-item:focus { - text-decoration: none; - color: #555555; - background-color: #f5f5f5; -} -button.list-group-item { - width: 100%; - text-align: left; -} -.list-group-item.disabled, -.list-group-item.disabled:hover, -.list-group-item.disabled:focus { - background-color: #eeeeee; - color: #777777; - cursor: not-allowed; -} -.list-group-item.disabled .list-group-item-heading, -.list-group-item.disabled:hover .list-group-item-heading, -.list-group-item.disabled:focus .list-group-item-heading { - color: inherit; -} -.list-group-item.disabled .list-group-item-text, -.list-group-item.disabled:hover .list-group-item-text, -.list-group-item.disabled:focus .list-group-item-text { - color: #777777; -} -.list-group-item.active, -.list-group-item.active:hover, -.list-group-item.active:focus { - z-index: 2; - color: #ffffff; - background-color: #337ab7; - border-color: #337ab7; -} -.list-group-item.active .list-group-item-heading, -.list-group-item.active:hover .list-group-item-heading, -.list-group-item.active:focus .list-group-item-heading, -.list-group-item.active .list-group-item-heading > small, -.list-group-item.active:hover .list-group-item-heading > small, -.list-group-item.active:focus .list-group-item-heading > small, -.list-group-item.active .list-group-item-heading > .small, -.list-group-item.active:hover .list-group-item-heading > .small, -.list-group-item.active:focus .list-group-item-heading > .small { - color: inherit; -} -.list-group-item.active .list-group-item-text, -.list-group-item.active:hover .list-group-item-text, -.list-group-item.active:focus .list-group-item-text { - color: #c7ddef; -} -.list-group-item-success { - color: #3c763d; - background-color: #dff0d8; -} -a.list-group-item-success, -button.list-group-item-success { - color: #3c763d; -} -a.list-group-item-success .list-group-item-heading, -button.list-group-item-success .list-group-item-heading { - color: inherit; -} -a.list-group-item-success:hover, -button.list-group-item-success:hover, -a.list-group-item-success:focus, -button.list-group-item-success:focus { - color: #3c763d; - background-color: #d0e9c6; -} -a.list-group-item-success.active, -button.list-group-item-success.active, -a.list-group-item-success.active:hover, -button.list-group-item-success.active:hover, -a.list-group-item-success.active:focus, -button.list-group-item-success.active:focus { - color: #fff; - background-color: #3c763d; - border-color: #3c763d; -} -.list-group-item-info { - color: #31708f; - background-color: #d9edf7; -} -a.list-group-item-info, -button.list-group-item-info { - color: #31708f; -} -a.list-group-item-info .list-group-item-heading, -button.list-group-item-info .list-group-item-heading { - color: inherit; -} -a.list-group-item-info:hover, -button.list-group-item-info:hover, -a.list-group-item-info:focus, -button.list-group-item-info:focus { - color: #31708f; - background-color: #c4e3f3; -} -a.list-group-item-info.active, -button.list-group-item-info.active, -a.list-group-item-info.active:hover, -button.list-group-item-info.active:hover, -a.list-group-item-info.active:focus, -button.list-group-item-info.active:focus { - color: #fff; - background-color: #31708f; - border-color: #31708f; -} -.list-group-item-warning { - color: #8a6d3b; - background-color: #fcf8e3; -} -a.list-group-item-warning, -button.list-group-item-warning { - color: #8a6d3b; -} -a.list-group-item-warning .list-group-item-heading, -button.list-group-item-warning .list-group-item-heading { - color: inherit; -} -a.list-group-item-warning:hover, -button.list-group-item-warning:hover, -a.list-group-item-warning:focus, -button.list-group-item-warning:focus { - color: #8a6d3b; - background-color: #faf2cc; -} -a.list-group-item-warning.active, -button.list-group-item-warning.active, -a.list-group-item-warning.active:hover, -button.list-group-item-warning.active:hover, -a.list-group-item-warning.active:focus, -button.list-group-item-warning.active:focus { - color: #fff; - background-color: #8a6d3b; - border-color: #8a6d3b; -} -.list-group-item-danger { - color: #a94442; - background-color: #f2dede; -} -a.list-group-item-danger, -button.list-group-item-danger { - color: #a94442; -} -a.list-group-item-danger .list-group-item-heading, -button.list-group-item-danger .list-group-item-heading { - color: inherit; -} -a.list-group-item-danger:hover, -button.list-group-item-danger:hover, -a.list-group-item-danger:focus, -button.list-group-item-danger:focus { - color: #a94442; - background-color: #ebcccc; -} -a.list-group-item-danger.active, -button.list-group-item-danger.active, -a.list-group-item-danger.active:hover, -button.list-group-item-danger.active:hover, -a.list-group-item-danger.active:focus, -button.list-group-item-danger.active:focus { - color: #fff; - background-color: #a94442; - border-color: #a94442; -} -.list-group-item-heading { - margin-top: 0; - margin-bottom: 5px; -} -.list-group-item-text { - margin-bottom: 0; - line-height: 1.3; -} - -.well { - min-height: 20px; - padding: 19px; - margin-bottom: 20px; - background-color: #f5f5f5; - border: 1px solid #e3e3e3; - border-radius: 4px; - -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05); - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05); -} -.well blockquote { - border-color: #ddd; - border-color: rgba(0, 0, 0, 0.15); -} -.well-lg { - padding: 24px; - border-radius: 6px; -} -.well-sm { - padding: 9px; - border-radius: 3px; -} diff --git a/OpenRefine/extensions/database/module/styles/database-import.less b/OpenRefine/extensions/database/module/styles/database-import.less deleted file mode 100644 index 770958405..000000000 --- a/OpenRefine/extensions/database/module/styles/database-import.less +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -@import-less url("theme.less"); - -.database-container { -// margin-left: 200px; - // width: 50%; -// min-height: 50%; -} - -.cleared { - clear: both; -} - -.custom-restricted-width { - /* To limit the menu width to the content of the menu: */ - display: inline-block; - /* Or set the width explicitly: */ - /* width: 10em; */ -} - - -.scrollable { - height: 400px; - overflow: auto; -} - -.database-importing-wizard-header { - font-size: 1.3em; - background: @chrome_primary; - padding: @padding_tight; - } - -.database-importing-parsing-data-panel { - font-size: 1.1em; - position: absolute; - overflow: auto; - } - -.database-importing-progress-data-panel { - position: absolute; - overflow: auto; - font-size: 200%; - padding: 3em; - background: rgba(255, 255, 255, 0.7); - text-align: center; - } - -.database-importing-parsing-control-panel { - font-size: 1.3em; - position: absolute; - overflow: auto; - border-top: 5px solid @chrome_primary; - background: white; - padding: @padding_looser; - } - -.context-menu-text { - padding: 4px 4px 2px 2px; -} - - -.custom-restricted { - height: 300px; - width: 100%; - border: 1px solid #bcf; - border-radius: 4px; -} - -// FIXME: This is a duplicate -.custom-restricted-width { - /* To limit the menu width to the content of the menu: */ - display: inline-block; - /* Or set the width explicitly: */ - /* width: 10em; */ - margin-right:10px; - padding-right:10px; - margin-bottom:10px; -} -.no-resize { - resize: none; -} -.layout-div { - width : 100%; - -} -.layout-div .connection-div-layout { - min-width: 400px; -} -.new-connection-legend { - padding-left: 2px; - margin-left: 2px; - border-color: #bcf; -} - -.sql-editor-div { - margin-left:40px; - padding-left:20px; -} -.new-connection-div { - margin-left:40px; - padding-left:20px; -} -.new-connection-fieldset { - border: 1px solid gray; -} - -.sc-context-more-vert:after { - content: ""; - display: inline-block; - background: url("../images/more-option-horiz-16.png") no-repeat top right; - // line-height: 1; - width: 16px; - height: 16px; - -webkit-font-smoothing: antialiased; - -} diff --git a/OpenRefine/extensions/database/module/styles/jquery.contextMenu.css b/OpenRefine/extensions/database/module/styles/jquery.contextMenu.css deleted file mode 100644 index 0253e6e1b..000000000 --- a/OpenRefine/extensions/database/module/styles/jquery.contextMenu.css +++ /dev/null @@ -1,292 +0,0 @@ -@charset "UTF-8"; -/*! - * jQuery contextMenu - Plugin for simple contextMenu handling - * - * Version: v2.6.3 - * - * Authors: Björn Brala (SWIS.nl), Rodney Rehm, Addy Osmani (patches for FF) - * Web: http://swisnl.github.io/jQuery-contextMenu/ - * - * Copyright (c) 2011-2017 SWIS BV and contributors - * - * Licensed under - * MIT License http://www.opensource.org/licenses/mit-license - * - * Date: 2017-10-30T19:03:13.936Z - */ -@-webkit-keyframes cm-spin { - 0% { - -webkit-transform: translateY(-50%) rotate(0deg); - transform: translateY(-50%) rotate(0deg); - } - 100% { - -webkit-transform: translateY(-50%) rotate(359deg); - transform: translateY(-50%) rotate(359deg); - } -} -@-o-keyframes cm-spin { - 0% { - -webkit-transform: translateY(-50%) rotate(0deg); - -o-transform: translateY(-50%) rotate(0deg); - transform: translateY(-50%) rotate(0deg); - } - 100% { - -webkit-transform: translateY(-50%) rotate(359deg); - -o-transform: translateY(-50%) rotate(359deg); - transform: translateY(-50%) rotate(359deg); - } -} -@keyframes cm-spin { - 0% { - -webkit-transform: translateY(-50%) rotate(0deg); - -o-transform: translateY(-50%) rotate(0deg); - transform: translateY(-50%) rotate(0deg); - } - 100% { - -webkit-transform: translateY(-50%) rotate(359deg); - -o-transform: translateY(-50%) rotate(359deg); - transform: translateY(-50%) rotate(359deg); - } -} - -@font-face { - font-family: "context-menu-icons"; - font-style: normal; - font-weight: normal; - - src: url("font/context-menu-icons.eot?2wp27"); - src: url("font/context-menu-icons.eot?2wp27#iefix") format("embedded-opentype"), url("font/context-menu-icons.woff2?2wp27") format("woff2"), url("font/context-menu-icons.woff?2wp27") format("woff"), url("font/context-menu-icons.ttf?2wp27") format("truetype"); -} - -.context-menu-icon-add:before { - content: "\EA01"; -} - -.context-menu-icon-copy:before { - content: "\EA02"; -} - -.context-menu-icon-cut:before { - content: "\EA03"; -} - -.context-menu-icon-delete:before { - content: "\EA04"; -} - -.context-menu-icon-edit:before { - content: "\EA05"; -} - -.context-menu-icon-loading:before { - content: "\EA06"; -} - -.context-menu-icon-paste:before { - content: "\EA07"; -} - -.context-menu-icon-quit:before { - content: "\EA08"; -} - -.context-menu-icon::before { - position: absolute; - top: 50%; - left: 0; - width: 2em; - font-family: "context-menu-icons"; - font-size: 1em; - font-style: normal; - font-weight: normal; - line-height: 1; - color: #2980b9; - text-align: center; - -webkit-transform: translateY(-50%); - -ms-transform: translateY(-50%); - -o-transform: translateY(-50%); - transform: translateY(-50%); - - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; -} - -.context-menu-icon.context-menu-hover:before { - color: #fff; -} - -.context-menu-icon.context-menu-disabled::before { - color: #bbb; -} - -.context-menu-icon.context-menu-icon-loading:before { - -webkit-animation: cm-spin 2s infinite; - -o-animation: cm-spin 2s infinite; - animation: cm-spin 2s infinite; -} - -.context-menu-icon.context-menu-icon--fa { - display: list-item; - font-family: inherit; -} -.context-menu-icon.context-menu-icon--fa::before { - position: absolute; - top: 50%; - left: 0; - width: 2em; - font-family: FontAwesome; - font-size: 1em; - font-style: normal; - font-weight: normal; - line-height: 1; - color: #2980b9; - text-align: center; - -webkit-transform: translateY(-50%); - -ms-transform: translateY(-50%); - -o-transform: translateY(-50%); - transform: translateY(-50%); - - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; -} -.context-menu-icon.context-menu-icon--fa.context-menu-hover:before { - color: #fff; -} -.context-menu-icon.context-menu-icon--fa.context-menu-disabled::before { - color: #bbb; -} - -.context-menu-list { - position: absolute; - display: inline-block; - min-width: 13em; - max-width: 26em; - padding: .25em 0; - margin: .3em; - font-family: inherit; - font-size: inherit; - list-style-type: none; - background: #fff; - border: 1px solid #bebebe; - border-radius: .2em; - -webkit-box-shadow: 0 2px 5px rgba(0, 0, 0, .5); - box-shadow: 0 2px 5px rgba(0, 0, 0, .5); -} - -.context-menu-item { - position: relative; - padding: .2em 2em; - color: #2f2f2f; - -webkit-user-select: none; - -moz-user-select: none; - -ms-user-select: none; - user-select: none; - background-color: #fff; - font: 15px "Helvetica Neue", Helvetica, Arial, sans-serif; -} - -.context-menu-separator { - padding: 0; - margin: .35em 0; - border-bottom: 1px solid #e6e6e6; -} - -.context-menu-item > label > input, -.context-menu-item > label > textarea { - -webkit-user-select: text; - -moz-user-select: text; - -ms-user-select: text; - user-select: text; -} - -.context-menu-item.context-menu-hover { - color: #fff; - cursor: pointer; - background-color: #2980b9; -} - -.context-menu-item.context-menu-disabled { - color: #bbb; - cursor: default; - background-color: #fff; -} - -.context-menu-input.context-menu-hover { - color: #2f2f2f; - cursor: default; -} - -.context-menu-submenu:after { - position: absolute; - top: 50%; - right: .5em; - z-index: 1; - width: 0; - height: 0; - content: ''; - border-color: transparent transparent transparent #2f2f2f; - border-style: solid; - border-width: .25em 0 .25em .25em; - -webkit-transform: translateY(-50%); - -ms-transform: translateY(-50%); - -o-transform: translateY(-50%); - transform: translateY(-50%); -} - -/** - * Inputs - */ -.context-menu-item.context-menu-input { - padding: .3em .6em; -} - -/* vertically align inside labels */ -.context-menu-input > label > * { - vertical-align: top; -} - -/* position checkboxes and radios as icons */ -.context-menu-input > label > input[type="checkbox"], -.context-menu-input > label > input[type="radio"] { - position: relative; - top: .12em; - margin-right: .4em; -} - -.context-menu-input > label { - margin: 0; -} - -.context-menu-input > label, -.context-menu-input > label > input[type="text"], -.context-menu-input > label > textarea, -.context-menu-input > label > select { - display: block; - width: 100%; - -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - box-sizing: border-box; -} - -.context-menu-input > label > textarea { - height: 7em; -} - -.context-menu-item > .context-menu-list { - top: .3em; - /* re-positioned by js */ - right: -.3em; - display: none; -} - -.context-menu-item.context-menu-visible > .context-menu-list { - display: block; -} - -.context-menu-accesskey { - text-decoration: underline; -} -/*Custom to display icons*/ - - - diff --git a/OpenRefine/extensions/database/module/styles/pure.css b/OpenRefine/extensions/database/module/styles/pure.css deleted file mode 100644 index 854e02cad..000000000 --- a/OpenRefine/extensions/database/module/styles/pure.css +++ /dev/null @@ -1,1549 +0,0 @@ -/*! -Pure v1.0.0 -Copyright 2013 Yahoo! -Licensed under the BSD License. -https://github.com/yahoo/pure/blob/master/LICENSE.md -*/ -/*! -normalize.css v^3.0 | MIT License | git.io/normalize -Copyright (c) Nicolas Gallagher and Jonathan Neal -*/ -/*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */ - -/** - * 1. Set default font family to sans-serif. - * 2. Prevent iOS and IE text size adjust after device orientation change, - * without disabling user zoom. - */ - -html { - font-family: sans-serif; /* 1 */ - -ms-text-size-adjust: 100%; /* 2 */ - -webkit-text-size-adjust: 100%; /* 2 */ -} - -/** - * Remove default margin. - */ - -body { - margin: 0; -} - -/* HTML5 display definitions - ========================================================================== */ - -/** - * Correct `block` display not defined for any HTML5 element in IE 8/9. - * Correct `block` display not defined for `details` or `summary` in IE 10/11 - * and Firefox. - * Correct `block` display not defined for `main` in IE 11. - */ - -article, -aside, -details, -figcaption, -figure, -footer, -header, -hgroup, -main, -menu, -nav, -section, -summary { - display: block; -} - -/** - * 1. Correct `inline-block` display not defined in IE 8/9. - * 2. Normalize vertical alignment of `progress` in Chrome, Firefox, and Opera. - */ - -audio, -canvas, -progress, -video { - display: inline-block; /* 1 */ - vertical-align: baseline; /* 2 */ -} - -/** - * Prevent modern browsers from displaying `audio` without controls. - * Remove excess height in iOS 5 devices. - */ - -audio:not([controls]) { - display: none; - height: 0; -} - -/** - * Address `[hidden]` styling not present in IE 8/9/10. - * Hide the `template` element in IE 8/9/10/11, Safari, and Firefox < 22. - */ - -[hidden], -template { - display: none; -} - -/* Links - ========================================================================== */ - -/** - * Remove the gray background color from active links in IE 10. - */ - -a { - background-color: transparent; -} - -/** - * Improve readability of focused elements when they are also in an - * active/hover state. - */ - -a:active, -a:hover { - outline: 0; -} - -/* Text-level semantics - ========================================================================== */ - -/** - * Address styling not present in IE 8/9/10/11, Safari, and Chrome. - */ - -abbr[title] { - border-bottom: 1px dotted; -} - -/** - * Address style set to `bolder` in Firefox 4+, Safari, and Chrome. - */ - -b, -strong { - font-weight: bold; -} - -/** - * Address styling not present in Safari and Chrome. - */ - -dfn { - font-style: italic; -} - -/** - * Address variable `h1` font-size and margin within `section` and `article` - * contexts in Firefox 4+, Safari, and Chrome. - */ - -h1 { - font-size: 2em; - margin: 0.67em 0; -} - -/** - * Address styling not present in IE 8/9. - */ - -mark { - background: #ff0; - color: #000; -} - -/** - * Address inconsistent and variable font size in all browsers. - */ - -small { - font-size: 80%; -} - -/** - * Prevent `sub` and `sup` affecting `line-height` in all browsers. - */ - -sub, -sup { - font-size: 75%; - line-height: 0; - position: relative; - vertical-align: baseline; -} - -sup { - top: -0.5em; -} - -sub { - bottom: -0.25em; -} - -/* Embedded content - ========================================================================== */ - -/** - * Remove border when inside `a` element in IE 8/9/10. - */ - -img { - border: 0; -} - -/** - * Correct overflow not hidden in IE 9/10/11. - */ - -svg:not(:root) { - overflow: hidden; -} - -/* Grouping content - ========================================================================== */ - -/** - * Address margin not present in IE 8/9 and Safari. - */ - -figure { - margin: 1em 40px; -} - -/** - * Address differences between Firefox and other browsers. - */ - -hr { - box-sizing: content-box; - height: 0; -} - -/** - * Contain overflow in all browsers. - */ - -pre { - overflow: auto; -} - -/** - * Address odd `em`-unit font size rendering in all browsers. - */ - -code, -kbd, -pre, -samp { - font-family: monospace, monospace; - font-size: 1em; -} - -/* Forms - ========================================================================== */ - -/** - * Known limitation: by default, Chrome and Safari on OS X allow very limited - * styling of `select`, unless a `border` property is set. - */ - -/** - * 1. Correct color not being inherited. - * Known issue: affects color of disabled elements. - * 2. Correct font properties not being inherited. - * 3. Address margins set differently in Firefox 4+, Safari, and Chrome. - */ - -button, -input, -optgroup, -select, -textarea { - color: inherit; /* 1 */ - font: inherit; /* 2 */ - margin: 0; /* 3 */ -} - -/** - * Address `overflow` set to `hidden` in IE 8/9/10/11. - */ - -button { - overflow: visible; -} - -/** - * Address inconsistent `text-transform` inheritance for `button` and `select`. - * All other form control elements do not inherit `text-transform` values. - * Correct `button` style inheritance in Firefox, IE 8/9/10/11, and Opera. - * Correct `select` style inheritance in Firefox. - */ - -button, -select { - text-transform: none; -} - -/** - * 1. Avoid the WebKit bug in Android 4.0.* where (2) destroys native `audio` - * and `video` controls. - * 2. Correct inability to style clickable `input` types in iOS. - * 3. Improve usability and consistency of cursor style between image-type - * `input` and others. - */ - -button, -html input[type="button"], /* 1 */ -input[type="reset"], -input[type="submit"] { - -webkit-appearance: button; /* 2 */ - cursor: pointer; /* 3 */ -} - -/** - * Re-set default cursor for disabled elements. - */ - -button[disabled], -html input[disabled] { - cursor: default; -} - -/** - * Remove inner padding and border in Firefox 4+. - */ - -button::-moz-focus-inner, -input::-moz-focus-inner { - border: 0; - padding: 0; -} - -/** - * Address Firefox 4+ setting `line-height` on `input` using `!important` in - * the UA stylesheet. - */ - -input { - line-height: normal; -} - -/** - * It's recommended that you don't attempt to style these elements. - * Firefox's implementation doesn't respect box-sizing, padding, or width. - * - * 1. Address box sizing set to `content-box` in IE 8/9/10. - * 2. Remove excess padding in IE 8/9/10. - */ - -input[type="checkbox"], -input[type="radio"] { - box-sizing: border-box; /* 1 */ - padding: 0; /* 2 */ -} - -/** - * Fix the cursor style for Chrome's increment/decrement buttons. For certain - * `font-size` values of the `input`, it causes the cursor style of the - * decrement button to change from `default` to `text`. - */ - -input[type="number"]::-webkit-inner-spin-button, -input[type="number"]::-webkit-outer-spin-button { - height: auto; -} - -/** - * 1. Address `appearance` set to `searchfield` in Safari and Chrome. - * 2. Address `box-sizing` set to `border-box` in Safari and Chrome. - */ - -input[type="search"] { - -webkit-appearance: textfield; /* 1 */ - box-sizing: content-box; /* 2 */ -} - -/** - * Remove inner padding and search cancel button in Safari and Chrome on OS X. - * Safari (but not Chrome) clips the cancel button when the search input has - * padding (and `textfield` appearance). - */ - -input[type="search"]::-webkit-search-cancel-button, -input[type="search"]::-webkit-search-decoration { - -webkit-appearance: none; -} - -/** - * Define consistent border, margin, and padding. - */ - -fieldset { - border: 1px solid #c0c0c0; - margin: 0 2px; - padding: 0.35em 0.625em 0.75em; -} - -/** - * 1. Correct `color` not being inherited in IE 8/9/10/11. - * 2. Remove padding so people aren't caught out if they zero out fieldsets. - */ - -legend { - border: 0; /* 1 */ - padding: 0; /* 2 */ -} - -/** - * Remove default vertical scrollbar in IE 8/9/10/11. - */ - -textarea { - overflow: auto; -} - -/** - * Don't inherit the `font-weight` (applied by a rule above). - * NOTE: the default cannot safely be changed in Chrome and Safari on OS X. - */ - -optgroup { - font-weight: bold; -} - -/* Tables - ========================================================================== */ - -/** - * Remove most spacing between table cells. - */ - -table { - border-collapse: collapse; - border-spacing: 0; -} - -td, -th { - padding: 0; -} - -/*csslint important:false*/ - -/* ========================================================================== - Pure Base Extras - ========================================================================== */ - -/** - * Extra rules that Pure adds on top of Normalize.css - */ - -/** - * Always hide an element when it has the `hidden` HTML attribute. - */ - -.hidden, -[hidden] { - display: none !important; -} - -/** - * Add this class to an image to make it fit within it's fluid parent wrapper while maintaining - * aspect ratio. - */ -.pure-img { - max-width: 100%; - height: auto; - display: block; -} - -/*csslint regex-selectors:false, known-properties:false, duplicate-properties:false*/ - -.pure-g { - letter-spacing: -0.31em; /* Webkit: collapse white-space between units */ - *letter-spacing: normal; /* reset IE < 8 */ - *word-spacing: -0.43em; /* IE < 8: collapse white-space between units */ - text-rendering: optimizespeed; /* Webkit: fixes text-rendering: optimizeLegibility */ - - /* - Sets the font stack to fonts known to work properly with the above letter - and word spacings. See: https://github.com/yahoo/pure/issues/41/ - - The following font stack makes Pure Grids work on all known environments. - - * FreeSans: Ships with many Linux distros, including Ubuntu - - * Arimo: Ships with Chrome OS. Arimo has to be defined before Helvetica and - Arial to get picked up by the browser, even though neither is available - in Chrome OS. - - * Droid Sans: Ships with all versions of Android. - - * Helvetica, Arial, sans-serif: Common font stack on OS X and Windows. - */ - font-family: FreeSans, Arimo, "Droid Sans", Helvetica, Arial, sans-serif; - - /* Use flexbox when possible to avoid `letter-spacing` side-effects. */ - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - -webkit-flex-flow: row wrap; - -ms-flex-flow: row wrap; - flex-flow: row wrap; - - /* Prevents distributing space between rows */ - -webkit-align-content: flex-start; - -ms-flex-line-pack: start; - align-content: flex-start; -} - -/* IE10 display: -ms-flexbox (and display: flex in IE 11) does not work inside a table; fall back to block and rely on font hack */ -@media all and (-ms-high-contrast: none), (-ms-high-contrast: active) { - table .pure-g { - display: block; - } -} - -/* Opera as of 12 on Windows needs word-spacing. - The ".opera-only" selector is used to prevent actual prefocus styling - and is not required in markup. -*/ -.opera-only :-o-prefocus, -.pure-g { - word-spacing: -0.43em; -} - -.pure-u { - display: inline-block; - *display: inline; /* IE < 8: fake inline-block */ - zoom: 1; - letter-spacing: normal; - word-spacing: normal; - vertical-align: top; - text-rendering: auto; -} - -/* -Resets the font family back to the OS/browser's default sans-serif font, -this the same font stack that Normalize.css sets for the `body`. -*/ -.pure-g [class *= "pure-u"] { - font-family: sans-serif; -} - -.pure-u-1, -.pure-u-1-1, -.pure-u-1-2, -.pure-u-1-3, -.pure-u-2-3, -.pure-u-1-4, -.pure-u-3-4, -.pure-u-1-5, -.pure-u-2-5, -.pure-u-3-5, -.pure-u-4-5, -.pure-u-5-5, -.pure-u-1-6, -.pure-u-5-6, -.pure-u-1-8, -.pure-u-3-8, -.pure-u-5-8, -.pure-u-7-8, -.pure-u-1-12, -.pure-u-5-12, -.pure-u-7-12, -.pure-u-11-12, -.pure-u-1-24, -.pure-u-2-24, -.pure-u-3-24, -.pure-u-4-24, -.pure-u-5-24, -.pure-u-6-24, -.pure-u-7-24, -.pure-u-8-24, -.pure-u-9-24, -.pure-u-10-24, -.pure-u-11-24, -.pure-u-12-24, -.pure-u-13-24, -.pure-u-14-24, -.pure-u-15-24, -.pure-u-16-24, -.pure-u-17-24, -.pure-u-18-24, -.pure-u-19-24, -.pure-u-20-24, -.pure-u-21-24, -.pure-u-22-24, -.pure-u-23-24, -.pure-u-24-24 { - display: inline-block; - *display: inline; - zoom: 1; - letter-spacing: normal; - word-spacing: normal; - vertical-align: top; - text-rendering: auto; -} - -.pure-u-1-24 { - width: 4.1667%; - *width: 4.1357%; -} - -.pure-u-1-12, -.pure-u-2-24 { - width: 8.3333%; - *width: 8.3023%; -} - -.pure-u-1-8, -.pure-u-3-24 { - width: 12.5000%; - *width: 12.4690%; -} - -.pure-u-1-6, -.pure-u-4-24 { - width: 16.6667%; - *width: 16.6357%; -} - -.pure-u-1-5 { - width: 20%; - *width: 19.9690%; -} - -.pure-u-5-24 { - width: 20.8333%; - *width: 20.8023%; -} - -.pure-u-1-4, -.pure-u-6-24 { - width: 25%; - *width: 24.9690%; -} - -.pure-u-7-24 { - width: 29.1667%; - *width: 29.1357%; -} - -.pure-u-1-3, -.pure-u-8-24 { - width: 33.3333%; - *width: 33.3023%; -} - -.pure-u-3-8, -.pure-u-9-24 { - width: 37.5000%; - *width: 37.4690%; -} - -.pure-u-2-5 { - width: 40%; - *width: 39.9690%; -} - -.pure-u-5-12, -.pure-u-10-24 { - width: 41.6667%; - *width: 41.6357%; -} - -.pure-u-11-24 { - width: 45.8333%; - *width: 45.8023%; -} - -.pure-u-1-2, -.pure-u-12-24 { - width: 50%; - *width: 49.9690%; -} - -.pure-u-13-24 { - width: 54.1667%; - *width: 54.1357%; -} - -.pure-u-7-12, -.pure-u-14-24 { - width: 58.3333%; - *width: 58.3023%; -} - -.pure-u-3-5 { - width: 60%; - *width: 59.9690%; -} - -.pure-u-5-8, -.pure-u-15-24 { - width: 62.5000%; - *width: 62.4690%; -} - -.pure-u-2-3, -.pure-u-16-24 { - width: 66.6667%; - *width: 66.6357%; -} - -.pure-u-17-24 { - width: 70.8333%; - *width: 70.8023%; -} - -.pure-u-3-4, -.pure-u-18-24 { - width: 75%; - *width: 74.9690%; -} - -.pure-u-19-24 { - width: 79.1667%; - *width: 79.1357%; -} - -.pure-u-4-5 { - width: 80%; - *width: 79.9690%; -} - -.pure-u-5-6, -.pure-u-20-24 { - width: 83.3333%; - *width: 83.3023%; -} - -.pure-u-7-8, -.pure-u-21-24 { - width: 87.5000%; - *width: 87.4690%; -} - -.pure-u-11-12, -.pure-u-22-24 { - width: 91.6667%; - *width: 91.6357%; -} - -.pure-u-23-24 { - width: 95.8333%; - *width: 95.8023%; -} - -.pure-u-1, -.pure-u-1-1, -.pure-u-5-5, -.pure-u-24-24 { - width: 100%; -} -.pure-button { - /* Structure */ - display: inline-block; - zoom: 1; - line-height: normal; - white-space: nowrap; - vertical-align: middle; - text-align: center; - cursor: pointer; - -webkit-user-drag: none; - -webkit-user-select: none; - -moz-user-select: none; - -ms-user-select: none; - user-select: none; - box-sizing: border-box; -} - -/* Firefox: Get rid of the inner focus border */ -.pure-button::-moz-focus-inner { - padding: 0; - border: 0; -} - -/* Inherit .pure-g styles */ -.pure-button-group { - letter-spacing: -0.31em; /* Webkit: collapse white-space between units */ - *letter-spacing: normal; /* reset IE < 8 */ - *word-spacing: -0.43em; /* IE < 8: collapse white-space between units */ - text-rendering: optimizespeed; /* Webkit: fixes text-rendering: optimizeLegibility */ -} - -.opera-only :-o-prefocus, -.pure-button-group { - word-spacing: -0.43em; -} - -.pure-button-group .pure-button { - letter-spacing: normal; - word-spacing: normal; - vertical-align: top; - text-rendering: auto; -} - -/*csslint outline-none:false*/ - -.pure-button { - font-family: inherit; - font-size: 100%; - padding: 0.5em 1em; - color: #444; /* rgba not supported (IE 8) */ - color: rgba(0, 0, 0, 0.80); /* rgba supported */ - border: 1px solid #999; /*IE 6/7/8*/ - border: none rgba(0, 0, 0, 0); /*IE9 + everything else*/ - background-color: #E6E6E6; - text-decoration: none; - border-radius: 2px; -} - -.pure-button-hover, -.pure-button:hover, -.pure-button:focus { - /* csslint ignore:start */ - filter: alpha(opacity=90); - /* csslint ignore:end */ - background-image: -webkit-linear-gradient(transparent, rgba(0,0,0, 0.05) 40%, rgba(0,0,0, 0.10)); - background-image: linear-gradient(transparent, rgba(0,0,0, 0.05) 40%, rgba(0,0,0, 0.10)); -} -.pure-button:focus { - outline: 0; -} -.pure-button-active, -.pure-button:active { - box-shadow: 0 0 0 1px rgba(0,0,0, 0.15) inset, 0 0 6px rgba(0,0,0, 0.20) inset; - border-color: #000\9; -} - -.pure-button[disabled], -.pure-button-disabled, -.pure-button-disabled:hover, -.pure-button-disabled:focus, -.pure-button-disabled:active { - border: none; - background-image: none; - /* csslint ignore:start */ - filter: alpha(opacity=40); - /* csslint ignore:end */ - opacity: 0.40; - cursor: not-allowed; - box-shadow: none; - pointer-events: none; -} - -.pure-button-hidden { - display: none; -} - -.pure-button-primary, -.pure-button-selected, -a.pure-button-primary, -a.pure-button-selected { - background-color: rgb(0, 120, 231); - color: #fff; -} - -/* Button Groups */ -.pure-button-group .pure-button { - margin: 0; - border-radius: 0; - border-right: 1px solid #111; /* fallback color for rgba() for IE7/8 */ - border-right: 1px solid rgba(0, 0, 0, 0.2); - -} - -.pure-button-group .pure-button:first-child { - border-top-left-radius: 2px; - border-bottom-left-radius: 2px; -} -.pure-button-group .pure-button:last-child { - border-top-right-radius: 2px; - border-bottom-right-radius: 2px; - border-right: none; -} - -/*csslint box-model:false*/ -/* -Box-model set to false because we're setting a height on select elements, which -also have border and padding. This is done because some browsers don't render -the padding. We explicitly set the box-model for select elements to border-box, -so we can ignore the csslint warning. -*/ - -.pure-form input[type="text"], -.pure-form input[type="password"], -.pure-form input[type="email"], -.pure-form input[type="url"], -.pure-form input[type="date"], -.pure-form input[type="month"], -.pure-form input[type="time"], -.pure-form input[type="datetime"], -.pure-form input[type="datetime-local"], -.pure-form input[type="week"], -.pure-form input[type="number"], -.pure-form input[type="search"], -.pure-form input[type="tel"], -.pure-form input[type="color"], -.pure-form select, -.pure-form textarea { - padding: 0.5em 0.6em; - display: inline-block; - border: 1px solid #ccc; - box-shadow: inset 0 1px 3px #ddd; - border-radius: 4px; - vertical-align: middle; - box-sizing: border-box; -} - -/* -Need to separate out the :not() selector from the rest of the CSS 2.1 selectors -since IE8 won't execute CSS that contains a CSS3 selector. -*/ -.pure-form input:not([type]) { - padding: 0.5em 0.6em; - display: inline-block; - border: 1px solid #ccc; - box-shadow: inset 0 1px 3px #ddd; - border-radius: 4px; - box-sizing: border-box; -} - - -/* Chrome (as of v.32/34 on OS X) needs additional room for color to display. */ -/* May be able to remove this tweak as color inputs become more standardized across browsers. */ -.pure-form input[type="color"] { - padding: 0.2em 0.5em; -} - - -.pure-form input[type="text"]:focus, -.pure-form input[type="password"]:focus, -.pure-form input[type="email"]:focus, -.pure-form input[type="url"]:focus, -.pure-form input[type="date"]:focus, -.pure-form input[type="month"]:focus, -.pure-form input[type="time"]:focus, -.pure-form input[type="datetime"]:focus, -.pure-form input[type="datetime-local"]:focus, -.pure-form input[type="week"]:focus, -.pure-form input[type="number"]:focus, -.pure-form input[type="search"]:focus, -.pure-form input[type="tel"]:focus, -.pure-form input[type="color"]:focus, -.pure-form select:focus, -.pure-form textarea:focus { - outline: 0; - border-color: #129FEA; -} - -/* -Need to separate out the :not() selector from the rest of the CSS 2.1 selectors -since IE8 won't execute CSS that contains a CSS3 selector. -*/ -.pure-form input:not([type]):focus { - outline: 0; - border-color: #129FEA; -} - -.pure-form input[type="file"]:focus, -.pure-form input[type="radio"]:focus, -.pure-form input[type="checkbox"]:focus { - outline: thin solid #129FEA; - outline: 1px auto #129FEA; -} -.pure-form .pure-checkbox, -.pure-form .pure-radio { - margin: 0.5em 0; - display: block; -} - -.pure-form input[type="text"][disabled], -.pure-form input[type="password"][disabled], -.pure-form input[type="email"][disabled], -.pure-form input[type="url"][disabled], -.pure-form input[type="date"][disabled], -.pure-form input[type="month"][disabled], -.pure-form input[type="time"][disabled], -.pure-form input[type="datetime"][disabled], -.pure-form input[type="datetime-local"][disabled], -.pure-form input[type="week"][disabled], -.pure-form input[type="number"][disabled], -.pure-form input[type="search"][disabled], -.pure-form input[type="tel"][disabled], -.pure-form input[type="color"][disabled], -.pure-form select[disabled], -.pure-form textarea[disabled] { - cursor: not-allowed; - background-color: #eaeded; - color: #cad2d3; -} - -/* -Need to separate out the :not() selector from the rest of the CSS 2.1 selectors -since IE8 won't execute CSS that contains a CSS3 selector. -*/ -.pure-form input:not([type])[disabled] { - cursor: not-allowed; - background-color: #eaeded; - color: #cad2d3; -} -.pure-form input[readonly], -.pure-form select[readonly], -.pure-form textarea[readonly] { - background-color: #eee; /* menu hover bg color */ - color: #777; /* menu text color */ - border-color: #ccc; -} - -.pure-form input:focus:invalid, -.pure-form textarea:focus:invalid, -.pure-form select:focus:invalid { - color: #b94a48; - border-color: #e9322d; -} -.pure-form input[type="file"]:focus:invalid:focus, -.pure-form input[type="radio"]:focus:invalid:focus, -.pure-form input[type="checkbox"]:focus:invalid:focus { - outline-color: #e9322d; -} -.pure-form select { - /* Normalizes the height; padding is not sufficient. */ - height: 2.25em; - border: 1px solid #ccc; - background-color: white; -} -.pure-form select[multiple] { - height: auto; -} -.pure-form label { - margin: 0.5em 0 0.2em; -} -.pure-form fieldset { - margin: 0; - padding: 0.35em 0 0.75em; - border: 0; -} -.pure-form legend { - display: block; - width: 50%; - padding: 0.3em 0; - margin-bottom: 0.3em; - margin-left: 5px; - color: #333; - /* border-bottom: 1px solid #e5e5e5; */ - border-bottom: 1px solid #bcf; -} - -.pure-form-stacked input[type="text"], -.pure-form-stacked input[type="password"], -.pure-form-stacked input[type="email"], -.pure-form-stacked input[type="url"], -.pure-form-stacked input[type="date"], -.pure-form-stacked input[type="month"], -.pure-form-stacked input[type="time"], -.pure-form-stacked input[type="datetime"], -.pure-form-stacked input[type="datetime-local"], -.pure-form-stacked input[type="week"], -.pure-form-stacked input[type="number"], -.pure-form-stacked input[type="search"], -.pure-form-stacked input[type="tel"], -.pure-form-stacked input[type="color"], -.pure-form-stacked input[type="file"], -.pure-form-stacked select, -.pure-form-stacked label, -.pure-form-stacked textarea { - display: block; - margin: 0.25em 0; -} - -/* -Need to separate out the :not() selector from the rest of the CSS 2.1 selectors -since IE8 won't execute CSS that contains a CSS3 selector. -*/ -.pure-form-stacked input:not([type]) { - display: block; - margin: 0.25em 0; -} -.pure-form-aligned input, -.pure-form-aligned textarea, -.pure-form-aligned select, -/* NOTE: pure-help-inline is deprecated. Use .pure-form-message-inline instead. */ -.pure-form-aligned .pure-help-inline, -.pure-form-message-inline { - display: inline-block; - *display: inline; - *zoom: 1; - vertical-align: middle; -} -.pure-form-aligned textarea { - vertical-align: top; -} - -/* Aligned Forms */ -.pure-form-aligned .pure-control-group { - margin-bottom: 0.5em; -} -.pure-form-aligned .pure-control-group label { - text-align: right; - display: inline-block; - vertical-align: middle; - width: 10em; - margin: 0 1em 0 0; -} -.pure-form-aligned .pure-controls { - margin: 1.5em 0 0 11em; -} - -/* Rounded Inputs */ -.pure-form input.pure-input-rounded, -.pure-form .pure-input-rounded { - border-radius: 2em; - padding: 0.5em 1em; -} - -/* Grouped Inputs */ -.pure-form .pure-group fieldset { - margin-bottom: 10px; -} -.pure-form .pure-group input, -.pure-form .pure-group textarea { - display: block; - padding: 10px; - margin: 0 0 -1px; - border-radius: 0; - position: relative; - top: -1px; -} -.pure-form .pure-group input:focus, -.pure-form .pure-group textarea:focus { - z-index: 3; -} -.pure-form .pure-group input:first-child, -.pure-form .pure-group textarea:first-child { - top: 1px; - border-radius: 4px 4px 0 0; - margin: 0; -} -.pure-form .pure-group input:first-child:last-child, -.pure-form .pure-group textarea:first-child:last-child { - top: 1px; - border-radius: 4px; - margin: 0; -} -.pure-form .pure-group input:last-child, -.pure-form .pure-group textarea:last-child { - top: -2px; - border-radius: 0 0 4px 4px; - margin: 0; -} -.pure-form .pure-group button { - margin: 0.35em 0; -} - -.pure-form .pure-input-1 { - width: 100%; -} -.pure-form .pure-input-3-4 { - width: 75%; -} -.pure-form .pure-input-2-3 { - width: 66%; -} -.pure-form .pure-input-1-2 { - width: 50%; -} -.pure-form .pure-input-1-3 { - width: 33%; -} -.pure-form .pure-input-1-4 { - width: 25%; -} - -/* Inline help for forms */ -/* NOTE: pure-help-inline is deprecated. Use .pure-form-message-inline instead. */ -.pure-form .pure-help-inline, -.pure-form-message-inline { - display: inline-block; - padding-left: 0.3em; - color: #666; - vertical-align: middle; - font-size: 0.875em; -} - -/* Block help for forms */ -.pure-form-message { - display: block; - color: #666; - font-size: 0.875em; -} - -@media only screen and (max-width : 480px) { - .pure-form button[type="submit"] { - margin: 0.7em 0 0; - } - - .pure-form input:not([type]), - .pure-form input[type="text"], - .pure-form input[type="password"], - .pure-form input[type="email"], - .pure-form input[type="url"], - .pure-form input[type="date"], - .pure-form input[type="month"], - .pure-form input[type="time"], - .pure-form input[type="datetime"], - .pure-form input[type="datetime-local"], - .pure-form input[type="week"], - .pure-form input[type="number"], - .pure-form input[type="search"], - .pure-form input[type="tel"], - .pure-form input[type="color"], - .pure-form label { - margin-bottom: 0.3em; - display: block; - } - - .pure-group input:not([type]), - .pure-group input[type="text"], - .pure-group input[type="password"], - .pure-group input[type="email"], - .pure-group input[type="url"], - .pure-group input[type="date"], - .pure-group input[type="month"], - .pure-group input[type="time"], - .pure-group input[type="datetime"], - .pure-group input[type="datetime-local"], - .pure-group input[type="week"], - .pure-group input[type="number"], - .pure-group input[type="search"], - .pure-group input[type="tel"], - .pure-group input[type="color"] { - margin-bottom: 0; - } - - .pure-form-aligned .pure-control-group label { - margin-bottom: 0.3em; - text-align: left; - display: block; - width: 100%; - } - - .pure-form-aligned .pure-controls { - margin: 1.5em 0 0 0; - } - - /* NOTE: pure-help-inline is deprecated. Use .pure-form-message-inline instead. */ - .pure-form .pure-help-inline, - .pure-form-message-inline, - .pure-form-message { - display: block; - font-size: 0.75em; - /* Increased bottom padding to make it group with its related input element. */ - padding: 0.2em 0 0.8em; - } -} - -/*csslint adjoining-classes: false, box-model:false*/ -.pure-menu { - box-sizing: border-box; -} - -.pure-menu-fixed { - position: fixed; - left: 0; - top: 0; - z-index: 3; -} - -.pure-menu-list, -.pure-menu-item { - position: relative; -} - -.pure-menu-list { - list-style: none; - margin: 0; - padding: 0; -} - -.pure-menu-item { - padding: 0; - margin: 0; - height: 100%; -} - -.pure-menu-link, -.pure-menu-heading { - display: block; - text-decoration: none; - white-space: nowrap; -} - -/* HORIZONTAL MENU */ -.pure-menu-horizontal { - width: 100%; - white-space: nowrap; -} - -.pure-menu-horizontal .pure-menu-list { - display: inline-block; -} - -/* Initial menus should be inline-block so that they are horizontal */ -.pure-menu-horizontal .pure-menu-item, -.pure-menu-horizontal .pure-menu-heading, -.pure-menu-horizontal .pure-menu-separator { - display: inline-block; - *display: inline; - zoom: 1; - vertical-align: middle; -} - -/* Submenus should still be display: block; */ -.pure-menu-item .pure-menu-item { - display: block; -} - -.pure-menu-children { - display: none; - position: absolute; - left: 100%; - top: 0; - margin: 0; - padding: 0; - z-index: 3; -} - -.pure-menu-horizontal .pure-menu-children { - left: 0; - top: auto; - width: inherit; -} - -.pure-menu-allow-hover:hover > .pure-menu-children, -.pure-menu-active > .pure-menu-children { - display: block; - position: absolute; -} - -/* Vertical Menus - show the dropdown arrow */ -.pure-menu-has-children > .pure-menu-link:after { - padding-left: 0.5em; - content: "\25B8"; - font-size: small; -} - -/* Horizontal Menus - show the dropdown arrow */ -.pure-menu-horizontal .pure-menu-has-children > .pure-menu-link:after { - content: "\25BE"; -} - -/* scrollable menus */ -.pure-menu-scrollable { - overflow-y: scroll; - overflow-x: scroll; -} - -.pure-menu-scrollable .pure-menu-list { - display: block; -} - -.pure-menu-horizontal.pure-menu-scrollable .pure-menu-list { - display: inline-block; -} - -.pure-menu-horizontal.pure-menu-scrollable { - white-space: nowrap; - overflow-y: hidden; - overflow-x: auto; - -ms-overflow-style: none; - -webkit-overflow-scrolling: touch; - /* a little extra padding for this style to allow for scrollbars */ - padding: .5em 0; -} - -.pure-menu-horizontal.pure-menu-scrollable::-webkit-scrollbar { - display: none; -} - -/* misc default styling */ - -.pure-menu-separator, -.pure-menu-horizontal .pure-menu-children .pure-menu-separator { - background-color: #ccc; - height: 1px; - margin: .3em 0; -} - -.pure-menu-horizontal .pure-menu-separator { - width: 1px; - height: 1.3em; - margin: 0 .3em ; -} - -/* Need to reset the separator since submenu is vertical */ -.pure-menu-horizontal .pure-menu-children .pure-menu-separator { - display: block; - width: auto; -} - -.pure-menu-heading { - text-transform: uppercase; - color: #565d64; -} - -.pure-menu-link { - /* color: #777; */ -} - -.pure-menu-children { - background-color: #fff; -} - -.pure-menu-link, -.pure-menu-disabled, -.pure-menu-heading { - padding: .5em 1em; -} - -.pure-menu-disabled { - opacity: .5; -} - -.pure-menu-disabled .pure-menu-link:hover { - background-color: transparent; -} - -.pure-menu-active > .pure-menu-link, -.pure-menu-link:hover, -.pure-menu-link:focus { - background-color: #eee; -} - -.pure-menu-selected .pure-menu-link, -.pure-menu-selected .pure-menu-link:visited { - color: #000; -} - -.pure-table { - /* Remove spacing between table cells (from Normalize.css) */ - border-collapse: collapse; - border-spacing: 0; - empty-cells: show; - border: 1px solid #cbcbcb; -} - -.pure-table caption { - color: #000; - font: italic 85%/1 arial, sans-serif; - padding: 1em 0; - text-align: center; -} - -.pure-table td, -.pure-table th { - border-left: 1px solid #cbcbcb;/* inner column border */ - border-width: 0 0 0 1px; - font-size: inherit; - margin: 0; - overflow: visible; /*to make ths where the title is really long work*/ - padding: 0.5em 1em; /* cell padding */ -} - -/* Consider removing this next declaration block, as it causes problems when -there's a rowspan on the first cell. Case added to the tests. issue#432 */ -.pure-table td:first-child, -.pure-table th:first-child { - border-left-width: 0; -} - -.pure-table thead { - background-color: #e0e0e0; - color: #000; - text-align: left; - vertical-align: bottom; -} - -/* -striping: - even - #fff (white) - odd - #f2f2f2 (light gray) -*/ -.pure-table td { - background-color: transparent; -} -.pure-table-odd td { - background-color: #f2f2f2; -} - -/* nth-child selector for modern browsers */ -.pure-table-striped tr:nth-child(2n-1) td { - background-color: #f2f2f2; -} - -/* BORDERED TABLES */ -.pure-table-bordered td { - border-bottom: 1px solid #cbcbcb; -} -.pure-table-bordered tbody > tr:last-child > td { - border-bottom-width: 0; -} - - -/* HORIZONTAL BORDERED TABLES */ - -.pure-table-horizontal td, -.pure-table-horizontal th { - border-width: 0 0 1px 0; - border-bottom: 1px solid #cbcbcb; -} -.pure-table-horizontal tbody > tr:last-child > td { - border-bottom-width: 0; -} - - -/* - * -- HELPER STYLES -- - * Over-riding some of the .pure-button styles to make my buttons look unique - */ -.button-success, -.button-error, -.button-warning, -.button-secondary { - color: white; - border-radius: 4px; - text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2); -} - -.button-success { - background: rgb(28, 184, 65); /* this is a green */ -} - -.button-error { - background: rgb(202, 60, 60); /* this is a maroon */ -} - -.button-warning { - background: rgb(223, 117, 20); /* this is an orange */ -} - -.button-secondary { - background: rgb(66, 184, 221); /* this is a light blue */ -} - - -.custom-restricted { - height: 160px; - width: 150px; - border: 1px solid gray; - border-radius: 4px; -} - diff --git a/OpenRefine/extensions/database/module/styles/theme.less b/OpenRefine/extensions/database/module/styles/theme.less deleted file mode 100644 index 443a411bc..000000000 --- a/OpenRefine/extensions/database/module/styles/theme.less +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -@import-less url("../../../../main/webapp/modules/core/styles/theme.less"); diff --git a/OpenRefine/extensions/database/pom.xml b/OpenRefine/extensions/database/pom.xml deleted file mode 100644 index 47984c6d2..000000000 --- a/OpenRefine/extensions/database/pom.xml +++ /dev/null @@ -1,150 +0,0 @@ - - 4.0.0 - - database - jar - - OpenRefine - Database extension - Connections to SQL databases for import and export - http://openrefine.org/ - - org.openrefine - extensions - 3.6-SNAPSHOT - - - - openrefine-database - - - src - - - tests/src - module/MOD-INF/classes - - - org.apache.maven.plugins - maven-surefire-plugin - ${surefire.version} - - - tests/conf/tests.xml - - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - ${maven-dependency-plugin.version} - - - compile - - copy-dependencies - - - module/MOD-INF/lib - runtime - - - - - - - - - - - ${project.groupId} - main - ${project.version} - provided - - - javax.servlet - javax.servlet-api - ${servlet-api.version} - provided - - - org.postgresql - postgresql - 42.3.1 - - - org.apache.httpcomponents - httpclient - ${httpclient.version} - - - com.google.http-client - google-http-client-jackson2 - 1.41.2 - - - mysql - mysql-connector-java - 8.0.28 - - - org.jasypt - jasypt - ${jasypt.version} - - - org.mariadb.jdbc - mariadb-java-client - 2.7.5 - - - org.xerial - sqlite-jdbc - 3.36.0.3 - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - provided - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - - - org.owasp.encoder - encoder - ${owasp-encoder.version} - - - - - - org.testng - testng - ${testng.version} - test - - - org.mockito - mockito-core - ${mockito.version} - test - - - - - - - diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DBQueryResultImportReader.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/DBQueryResultImportReader.java deleted file mode 100644 index 0da3fa055..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DBQueryResultImportReader.java +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseQueryInfo; -import com.google.refine.extension.database.model.DatabaseRow; -import com.google.refine.importers.TabularImportingParserBase.TableDataReader; -import com.google.refine.importing.ImportingJob; - - -public class DBQueryResultImportReader implements TableDataReader { - - private static final Logger logger = LoggerFactory.getLogger("DBQueryResultImportReader"); - - private final ImportingJob job; - private final String querySource; - private List dbColumns; - private final int batchSize; - - private int nextRow = 0; // 0-based - private int batchRowStart = 0; // 0-based - private boolean end = false; - private List> rowsOfCells = null; - private boolean usedHeaders = false; - private DatabaseService databaseService; - private DatabaseQueryInfo dbQueryInfo; - private int processedRows = 0; - private static int progress = 0; - - - public DBQueryResultImportReader( - ImportingJob job, - DatabaseService databaseService, - String querySource, - List columns, - DatabaseQueryInfo dbQueryInfo, - int batchSize) { - - this.job = job; - this.querySource = querySource; - this.batchSize = batchSize; - this.dbColumns = columns; - this.databaseService = databaseService; - this.dbQueryInfo = dbQueryInfo; - if(logger.isDebugEnabled()) { - logger.debug("batchSize:" + batchSize); - } - - } - - @Override - public List getNextRowOfCells() throws IOException { - - try { - - if (!usedHeaders) { - List row = new ArrayList(dbColumns.size()); - for (DatabaseColumn cd : dbColumns) { - row.add(cd.getName()); - } - usedHeaders = true; - //logger.info("Exit::getNextRowOfCells return header::row:" + row); - return row; - } - - if (rowsOfCells == null || (nextRow >= batchRowStart + rowsOfCells.size() && !end)) { - int newBatchRowStart = batchRowStart + (rowsOfCells == null ? 0 : rowsOfCells.size()); - rowsOfCells = getRowsOfCells(newBatchRowStart); - processedRows = processedRows + rowsOfCells.size(); - batchRowStart = newBatchRowStart; - setProgress(job, querySource, -1 /* batchRowStart * 100 / totalRows */); - } - - if (rowsOfCells != null && nextRow - batchRowStart < rowsOfCells.size()) { - List result = rowsOfCells.get(nextRow++ - batchRowStart); - if(nextRow >= batchSize) { - rowsOfCells = getRowsOfCells(processedRows); - processedRows = processedRows + rowsOfCells.size(); - - if(logger.isDebugEnabled()) { - logger.debug("[[ Returning last row in batch:nextRow::{}, processedRows:{} ]]", nextRow, processedRows); - } - - nextRow = 0; - if(processedRows % 100 == 0) { - setProgress(job, querySource, progress++); - } - if(processedRows % 10000 == 0) { - if(logger.isDebugEnabled()) { - logger.debug("[[ {} rows processed... ]]",processedRows); - } - } - } - return result; - } else { - if(logger.isDebugEnabled()) { - logger.debug("[[processedRows:{} ]]", processedRows); - } - return null; - } - - - }catch(DatabaseServiceException e) { - logger.error("DatabaseServiceException::{}", e); - throw new IOException(e); - - } - - - } - - /** - * @param startRow - * @return - * @throws IOException - * @throws DatabaseServiceException - */ - private List> getRowsOfCells(int startRow) throws IOException, DatabaseServiceException { - //logger.info("Entry getRowsOfCells::startRow:" + startRow); - - List> rowsOfCells = new ArrayList>(batchSize); - - String query = databaseService.buildLimitQuery(batchSize, startRow, dbQueryInfo.getQuery()); - //logger.info("batchSize::" + batchSize + " startRow::" + startRow + " query::" + query ); - - List dbRows = databaseService.getRows(dbQueryInfo.getDbConfig(), query); - - if(dbRows != null && !dbRows.isEmpty() && dbRows.size() > 0) { - - for(DatabaseRow dbRow: dbRows) { - List row = dbRow.getValues(); - List rowOfCells = new ArrayList(row.size()); - - for (int j = 0; j < row.size() && j < dbColumns.size(); j++) { - - String text = row.get(j); - if (text == null || text.isEmpty()) { - rowOfCells.add(null); - }else { - DatabaseColumn col = dbColumns.get(j); - if(col.getType() == DatabaseColumnType.NUMBER) { - try { - rowOfCells.add(Long.parseLong(text)); - continue; - } catch (NumberFormatException e) {} - - }else if(col.getType() == DatabaseColumnType.DOUBLE || col.getType() == DatabaseColumnType.FLOAT ) { - try { - double d = Double.parseDouble(text); - if (!Double.isInfinite(d) && !Double.isNaN(d)) { - rowOfCells.add(d); - continue; - } - } catch (NumberFormatException e) {} - - } - - rowOfCells.add(text); - } - - } - - rowsOfCells.add(rowOfCells); - - } - - } - end = dbRows.size() < batchSize + 1; - //logger.info("Exit::getRowsOfCells::rowsOfCells:{}", rowsOfCells); - return rowsOfCells; - - } - - private static void setProgress(ImportingJob job, String querySource, int percent) { - job.setProgress(percent, "Reading " + querySource); - } - - public List getColumns() { - return dbColumns; - } - - - public void setColumns(List columns) { - this.dbColumns = columns; - } - - - public int getNextRow() { - return nextRow; - } - - - public void setNextRow(int nextRow) { - this.nextRow = nextRow; - } - - - public int getBatchRowStart() { - return batchRowStart; - } - - - public void setBatchRowStart(int batchRowStart) { - this.batchRowStart = batchRowStart; - } - - - public boolean isEnd() { - return end; - } - - - public void setEnd(boolean end) { - this.end = end; - } - - - public List> getRowsOfCells() { - return rowsOfCells; - } - - - public void setRowsOfCells(List> rowsOfCells) { - this.rowsOfCells = rowsOfCells; - } - - - public boolean isUsedHeaders() { - return usedHeaders; - } - - - public void setUsedHeaders(boolean usedHeaders) { - this.usedHeaders = usedHeaders; - } - - - public ImportingJob getJob() { - return job; - } - - - public String getQuerySource() { - return querySource; - } - - - public int getBatchSize() { - return batchSize; - } - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DBQueryResultPreviewReader.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/DBQueryResultPreviewReader.java deleted file mode 100644 index 95f8a6cf7..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DBQueryResultPreviewReader.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseQueryInfo; -import com.google.refine.extension.database.model.DatabaseRow; -import com.google.refine.importers.TabularImportingParserBase.TableDataReader; -import com.google.refine.importing.ImportingJob; - - -public class DBQueryResultPreviewReader implements TableDataReader { - - private static final Logger logger = LoggerFactory.getLogger("DBQueryResultPreviewReader"); - - private final ImportingJob job; - private final String querySource; - private List dbColumns; - private final int batchSize; - - private int nextRow = 0; // 0-based - private int batchRowStart = 0; // 0-based - private boolean end = false; - private List> rowsOfCells = null; - private boolean usedHeaders = false; - private DatabaseService databaseService; - private DatabaseQueryInfo dbQueryInfo; - - - public DBQueryResultPreviewReader( - ImportingJob job, - DatabaseService databaseService, - String querySource, - List columns, - DatabaseQueryInfo dbQueryInfo, - int batchSize) { - - this.job = job; - this.querySource = querySource; - this.batchSize = batchSize; - this.dbColumns = columns; - this.databaseService = databaseService; - this.dbQueryInfo = dbQueryInfo; - logger.debug("DBQueryResultPreviewReader::batchSize:" + batchSize); - - } - - @Override - public List getNextRowOfCells() throws IOException { - - // logger.info("Entry::getNextRowOfCells"); - - try { - - if (!usedHeaders) { - List row = new ArrayList(dbColumns.size()); - for (DatabaseColumn cd : dbColumns) { - row.add(cd.getName()); - } - usedHeaders = true; - // logger.debug("Exit::getNextRowOfCells return header::row:" + row); - return row; - } - - if (rowsOfCells == null || (nextRow >= batchRowStart + rowsOfCells.size() && !end)) { - int newBatchRowStart = batchRowStart + (rowsOfCells == null ? 0 : rowsOfCells.size()); - rowsOfCells = getRowsOfCells(newBatchRowStart); - batchRowStart = newBatchRowStart; - setProgress(job, querySource, -1 /* batchRowStart * 100 / totalRows */); - // logger.info("getNextRowOfCells:: rowsOfCellsIsNull::rowsOfCells size:" + rowsOfCells.size() + ":batchRowStart:" + batchRowStart + " ::nextRow:" + nextRow); - } - - if (rowsOfCells != null && nextRow - batchRowStart < rowsOfCells.size()) { - //logger.info("Exit::getNextRowOfCells :rowsOfCellsNotNull::rowsOfCells size:" + rowsOfCells.size() + ":batchRowStart:" + batchRowStart + " ::nextRow:" + nextRow); - return rowsOfCells.get(nextRow++ - batchRowStart); - } else { - if(logger.isDebugEnabled()) { - logger.debug("nextRow:{}, batchRowStart:{}", nextRow, batchRowStart); - } - - return null; - } - - - }catch(DatabaseServiceException e) { - logger.error("DatabaseServiceException::preview:{}", e.getMessage()); - IOException ioEx = new IOException(e.getMessage(), e); - throw ioEx; - - } - - } - - /** - * - * @param startRow - * @return - * @throws IOException - * @throws DatabaseServiceException - */ - private List> getRowsOfCells(int startRow) throws IOException, DatabaseServiceException { - //logger.info("Entry getRowsOfCells::startRow:" + startRow); - - List> rowsOfCells = new ArrayList>(batchSize); - - String query = databaseService.buildLimitQuery(batchSize, startRow, dbQueryInfo.getQuery()); - if(logger.isDebugEnabled()) { - logger.debug("batchSize::" + batchSize + " startRow::" + startRow + " query::" + query ); - } - - List dbRows = databaseService.getRows(dbQueryInfo.getDbConfig(), query); - - if(dbRows != null && !dbRows.isEmpty() && dbRows.size() > 0) { - - for(DatabaseRow dbRow: dbRows) { - List row = dbRow.getValues(); - List rowOfCells = new ArrayList(row.size()); - - for (int j = 0; j < row.size() && j < dbColumns.size(); j++) { - - String text = row.get(j); - if (text == null || text.isEmpty()) { - rowOfCells.add(null); - }else { - DatabaseColumn col = dbColumns.get(j); - if(col.getType() == DatabaseColumnType.NUMBER) { - try { - rowOfCells.add(Long.parseLong(text)); - continue; - } catch (NumberFormatException e) {} - - }else if(col.getType() == DatabaseColumnType.DOUBLE || col.getType() == DatabaseColumnType.FLOAT ) { - try { - double d = Double.parseDouble(text); - if (!Double.isInfinite(d) && !Double.isNaN(d)) { - rowOfCells.add(d); - continue; - } - } catch (NumberFormatException e) {} - - } - - rowOfCells.add(text); - } - - } - rowsOfCells.add(rowOfCells); - - } - - } - end = dbRows.size() < batchSize + 1; - //logger.info("Exit::getRowsOfCells::rowsOfCells:{}", rowsOfCells); - return rowsOfCells; - - } - - private static void setProgress(ImportingJob job, String querySource, int percent) { - job.setProgress(percent, "Reading " + querySource); - } - - public List getColumns() { - return dbColumns; - } - - - public void setColumns(List columns) { - this.dbColumns = columns; - } - - - public int getNextRow() { - return nextRow; - } - - - public void setNextRow(int nextRow) { - this.nextRow = nextRow; - } - - - public int getBatchRowStart() { - return batchRowStart; - } - - - public void setBatchRowStart(int batchRowStart) { - this.batchRowStart = batchRowStart; - } - - - public boolean isEnd() { - return end; - } - - - public void setEnd(boolean end) { - this.end = end; - } - - - public List> getRowsOfCells() { - return rowsOfCells; - } - - - public void setRowsOfCells(List> rowsOfCells) { - this.rowsOfCells = rowsOfCells; - } - - - public boolean isUsedHeaders() { - return usedHeaders; - } - - - public void setUsedHeaders(boolean usedHeaders) { - this.usedHeaders = usedHeaders; - } - - - public ImportingJob getJob() { - return job; - } - - - public String getQuerySource() { - return querySource; - } - - - public int getBatchSize() { - return batchSize; - } - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseColumnType.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseColumnType.java deleted file mode 100644 index 79a0c8f3a..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseColumnType.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database; - - -public enum DatabaseColumnType { - - STRING, - NUMBER, - DATETIME, - LOCATION, - BOOLEAN, - DATE, - DOUBLE, - FLOAT - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseConfiguration.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseConfiguration.java deleted file mode 100644 index 47dad7f4e..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseConfiguration.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database; - - -public class DatabaseConfiguration { - - private String connectionName; - private String databaseType; - private String databaseHost; - private int databasePort; - private String databaseUser; - private String databasePassword; - private String databaseName; - private String databaseSchema; - - //optional parameters - private boolean useSSL; - - - public String getConnectionName() { - return connectionName; - } - - public void setConnectionName(String connectionName) { - this.connectionName = connectionName; - } - - public String getDatabaseType() { - return databaseType; - } - - public void setDatabaseType(String databaseType) { - this.databaseType = databaseType; - } - - public String getDatabaseHost() { - return databaseHost; - } - - public void setDatabaseHost(String databaseServer) { - this.databaseHost = databaseServer; - } - - public int getDatabasePort() { - return databasePort; - } - - public void setDatabasePort(int databasePort) { - this.databasePort = databasePort; - } - - public String getDatabaseUser() { - return databaseUser; - } - - public void setDatabaseUser(String databaseUser) { - this.databaseUser = databaseUser; - } - - public String getDatabasePassword() { - return databasePassword; - } - - public void setDatabasePassword(String databasePassword) { - this.databasePassword = databasePassword; - } - - public String getDatabaseName() { - return databaseName; - } - - public void setDatabaseName(String initialDatabase) { - this.databaseName = initialDatabase; - } - - public String getDatabaseSchema() { - return databaseSchema; - } - - public void setDatabaseSchema(String initialSchema) { - this.databaseSchema = initialSchema; - } - - - - public boolean isUseSSL() { - return useSSL; - } - - public void setUseSSL(boolean useSSL) { - this.useSSL = useSSL; - } - - @Override - public String toString() { - return "DatabaseConfiguration [connectionName=" + connectionName + ", databaseType=" + databaseType - + ", databaseHost=" + databaseHost + ", databasePort=" + databasePort + ", databaseUser=" + databaseUser - + ", databaseName=" + databaseName + ", databaseSchema=" - + databaseSchema + ", useSSL=" + useSSL + "]"; - } - - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseImportController.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseImportController.java deleted file mode 100644 index a7126e925..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseImportController.java +++ /dev/null @@ -1,490 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package com.google.refine.extension.database; - -import java.io.IOException; -import java.io.Writer; -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.RefineServlet; -import com.google.refine.commands.HttpUtilities; -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseQueryInfo; -import com.google.refine.importers.TabularImportingParserBase; -import com.google.refine.importing.ImportingController; -import com.google.refine.importing.ImportingJob; -import com.google.refine.importing.ImportingManager; -import com.google.refine.model.Project; -import com.google.refine.util.JSONUtilities; -import com.google.refine.util.ParsingUtilities; - - -public class DatabaseImportController implements ImportingController { - - private static final Logger logger = LoggerFactory.getLogger("DatabaseImportController"); - protected RefineServlet servlet; - public static int DEFAULT_PREVIEW_LIMIT = 100; - public static String OPTIONS_KEY = "options"; - - @Override - public void init(RefineServlet servlet) { - this.servlet = servlet; - } - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - HttpUtilities.respond(response, "error", "GET not implemented"); - } - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - if(logger.isDebugEnabled()){ - logger.debug("doPost Query String::{}", request.getQueryString()); - } - response.setCharacterEncoding("UTF-8"); - Properties parameters = ParsingUtilities.parseUrlParameters(request); - - String subCommand = parameters.getProperty("subCommand"); - - if(logger.isDebugEnabled()){ - logger.info("doPost::subCommand::{}", subCommand); - } - - if ("initialize-parser-ui".equals(subCommand)) { - doInitializeParserUI(request, response, parameters); - } else if ("parse-preview".equals(subCommand)) { - try { - - doParsePreview(request, response, parameters); - - } catch (DatabaseServiceException e) { - logger.error("doPost::DatabaseServiceException::{}", e); - HttpUtilities.respond(response, "error", getDbServiceException(e)); - } - } else if ("create-project".equals(subCommand)) { - doCreateProject(request, response, parameters); - } else { - HttpUtilities.respond(response, "error", "No such sub command"); - } - - } - - private String getDbServiceException(Exception ex) { - String message = ""; - if(ex instanceof DatabaseServiceException) { - DatabaseServiceException dbEx = (DatabaseServiceException) ex; - if(dbEx.isSqlException()) { - message = message + dbEx.getSqlCode() + " " + dbEx.getSqlState(); - } - } - message = message + ex.getMessage(); - - return message; - } - - /** - * - * @param request - * @param response - * @param parameters - * @throws ServletException - * @throws IOException - */ - private void doInitializeParserUI(HttpServletRequest request, HttpServletResponse response, Properties parameters) - throws ServletException, IOException { - if(logger.isDebugEnabled()) { - logger.debug("::doInitializeParserUI::"); - } - - - ObjectNode result = ParsingUtilities.mapper.createObjectNode(); - ObjectNode options = ParsingUtilities.mapper.createObjectNode(); - JSONUtilities.safePut(result, "status", "ok"); - JSONUtilities.safePut(result, OPTIONS_KEY, options); - - JSONUtilities.safePut(options, "skipDataLines", 0); - JSONUtilities.safePut(options, "storeBlankRows", true); - JSONUtilities.safePut(options, "storeBlankCellsAsNulls", true); - if(logger.isDebugEnabled()) { - logger.debug("doInitializeParserUI:::{}", result.toString()); - } - - HttpUtilities.respond(response, result.toString()); - - } - - - /** - * doParsePreview - * @param request - * @param response - * @param parameters - * @throws ServletException - * @throws IOException - * @throws DatabaseServiceException - */ - private void doParsePreview( - HttpServletRequest request, HttpServletResponse response, Properties parameters) - throws ServletException, IOException, DatabaseServiceException { - if(logger.isDebugEnabled()) { - logger.debug("JobID::{}", parameters.getProperty("jobID")); - } - - - long jobID = Long.parseLong(parameters.getProperty("jobID")); - ImportingJob job = ImportingManager.getJob(jobID); - if (job == null) { - HttpUtilities.respond(response, "error", "No such import job"); - return; - } - - - DatabaseQueryInfo databaseQueryInfo = getQueryInfo(request); - - - if(databaseQueryInfo == null) { - HttpUtilities.respond(response, "error", "Invalid or missing Query Info"); - } - - job.updating = true; - try { - ObjectNode optionObj = ParsingUtilities.evaluateJsonStringToObjectNode( - request.getParameter("options")); - - List exceptions = new LinkedList(); - - job.prepareNewProject(); - - parsePreview( - databaseQueryInfo, - job.project, - job.metadata, - job, - DEFAULT_PREVIEW_LIMIT , - optionObj, - exceptions - ); - Writer w = response.getWriter(); - JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w); - try { - writer.writeStartObject(); - if (exceptions.size() == 0) { - job.project.update(); // update all internal models, indexes, caches, etc. - writer.writeStringField("status", "ok"); - } else { - writer.writeStringField("status", "error"); - writer.writeStringField("message", getExceptionString(exceptions)); - } - writer.writeEndObject(); - } catch (IOException e) { - throw new ServletException(e); - } finally { - writer.flush(); - writer.close(); - w.flush(); - w.close(); - } - - } catch (IOException e) { - throw new ServletException(e); - } finally { - job.touch(); - job.updating = false; - } - } - - - - private String getExceptionString(List exceptions) { - String ex = ""; - for(Exception e: exceptions) { - ex = ex + e.getLocalizedMessage() + "\n"; - } - // TODO Auto-generated method stub - return ex; - } - - /** - * - * @param dbQueryInfo - * @param project - * @param metadata - * @param job - * @param limit - * @param options - * @param exceptions - * @throws DatabaseServiceException - */ - private static void parsePreview( - DatabaseQueryInfo dbQueryInfo, - Project project, - ProjectMetadata metadata, - final ImportingJob job, - int limit, - ObjectNode options, - List exceptions) throws DatabaseServiceException{ - - - DatabaseService databaseService = DatabaseService.get(dbQueryInfo.getDbConfig().getDatabaseType()); - String querySource = getQuerySource(dbQueryInfo); - - List columns = databaseService.getColumns(dbQueryInfo.getDbConfig(), dbQueryInfo.getQuery()); - - - setProgress(job, querySource, -1); - - JSONUtilities.safePut(options, "ignoreLines", 0); // number of blank lines at the beginning to ignore - JSONUtilities.safePut(options, "headerLines", 1); // number of header lines - - - TabularImportingParserBase.readTable( - project, - metadata, - job, - new DBQueryResultPreviewReader(job, databaseService, querySource, columns, dbQueryInfo, 100), - querySource, - limit, - options, - exceptions - ); - - setProgress(job, querySource, 100); - - } - - - /** - * doCreateProject - * @param request - * @param response - * @param parameters - */ - private void doCreateProject(HttpServletRequest request, HttpServletResponse response, Properties parameters) - throws ServletException, IOException{ - if(logger.isDebugEnabled()) { - logger.debug("DatabaseImportController::doCreateProject:::{}", parameters.getProperty("jobID")); - } - - long jobID = Long.parseLong(parameters.getProperty("jobID")); - final ImportingJob job = ImportingManager.getJob(jobID); - if (job == null) { - HttpUtilities.respond(response, "error", "No such import job"); - return; - } - - final DatabaseQueryInfo databaseQueryInfo = getQueryInfo(request); - if(databaseQueryInfo == null) { - HttpUtilities.respond(response, "error", "Invalid or missing Query Info"); - } - - job.updating = true; - try { - final ObjectNode optionObj = ParsingUtilities.evaluateJsonStringToObjectNode( - request.getParameter("options")); - - final List exceptions = new LinkedList(); - - job.setState("creating-project"); - - final Project project = new Project(); - - new Thread() { - @Override - public void run() { - ProjectMetadata pm = new ProjectMetadata(); - pm.setName(JSONUtilities.getString(optionObj, "projectName", "Untitled")); - pm.setEncoding(JSONUtilities.getString(optionObj, "encoding", "UTF-8")); - - try { - parseCreate( - databaseQueryInfo, - project, - pm, - job, - -1, - optionObj, - exceptions - ); - } catch (DatabaseServiceException e) { - logger.error("DatabaseImportController::doCreateProject:::run{}", e); - // throw new RuntimeException("DatabaseServiceException::", e); - } - - if (!job.canceled) { - if (exceptions.size() > 0) { - job.setError(exceptions); - } else { - project.update(); // update all internal models, indexes, caches, etc. - ProjectManager.singleton.registerProject(project, pm); - job.setState("created-project"); - job.setProjectID(project.id); - // logger.info("DatabaseImportController::doCreateProject:::run::projectID :{}", project.id); - } - - job.touch(); - job.updating = false; - } - } - }.start(); - - HttpUtilities.respond(response, "ok", "done"); - } catch (IOException e) { - throw new ServletException(e); - } - } - - - /** - * @param dbQueryInfo - * @param project - * @param metadata - * @param job - * @param limit - * @param options - * @param exceptions - * @throws DatabaseServiceException - */ - private static void parseCreate( - DatabaseQueryInfo dbQueryInfo, - Project project, - ProjectMetadata metadata, - final ImportingJob job, - int limit, - ObjectNode options, - List exceptions) throws DatabaseServiceException{ - - - DatabaseService databaseService = DatabaseService.get(dbQueryInfo.getDbConfig().getDatabaseType()); - String querySource = getQuerySource(dbQueryInfo); - - List columns = databaseService.getColumns(dbQueryInfo.getDbConfig(), dbQueryInfo.getQuery()); - - setProgress(job, querySource, -1); - - JSONUtilities.safePut(options, "ignoreLines", 0); // number of blank lines at the beginning to ignore - JSONUtilities.safePut(options, "headerLines", 1); // number of header lines - - long startTime = System.currentTimeMillis() ; - - TabularImportingParserBase.readTable( - project, - metadata, - job, - new DBQueryResultImportReader(job, databaseService, querySource, columns, dbQueryInfo, getCreateBatchSize()), - querySource, - limit, - options, - exceptions - ); - - long endTime = System.currentTimeMillis() ; - if(logger.isDebugEnabled()) { - logger.debug("Execution Time: {}", endTime - startTime); - } - - setProgress(job, querySource, 100); - - } - - private static int getCreateBatchSize() { - String propBatchSize = DatabaseModuleImpl.getImportCreateBatchSize(); - int batchSize = 100; - if(propBatchSize != null && !propBatchSize.isEmpty()) { - try { - batchSize = Integer.parseInt(propBatchSize); - }catch(NumberFormatException nfe) { - - } - } - return batchSize; - } - - /** - * @param request - * @return - */ - private DatabaseQueryInfo getQueryInfo(HttpServletRequest request) { - DatabaseConfiguration jdbcConfig = new DatabaseConfiguration(); - jdbcConfig.setConnectionName(request.getParameter("connectionName")); - jdbcConfig.setDatabaseType(request.getParameter("databaseType")); - jdbcConfig.setDatabaseHost(request.getParameter("databaseServer")); - try { - jdbcConfig.setDatabasePort(Integer.parseInt(request.getParameter("databasePort"))); - } catch(NumberFormatException nfE) { - logger.error("getQueryInfo :: invalid database port ::{}", nfE); - } - jdbcConfig.setDatabaseUser(request.getParameter("databaseUser")); - jdbcConfig.setDatabasePassword(request.getParameter("databasePassword")); - jdbcConfig.setDatabaseName(request.getParameter("initialDatabase")); - jdbcConfig.setDatabaseSchema(request.getParameter("initialSchema")); - - String query = request.getParameter("query"); - if(logger.isDebugEnabled()) { - logger.debug("jdbcConfig::{}, query::{}", jdbcConfig, query); - } - if (jdbcConfig.getDatabaseHost() == null || jdbcConfig.getDatabaseName() == null - || jdbcConfig.getDatabasePassword() == null || jdbcConfig.getDatabaseType() == null - || jdbcConfig.getDatabaseUser() == null || query == null - || (jdbcConfig.getDatabasePort() == 0 && !"SQLite".equalsIgnoreCase(jdbcConfig.getDatabaseType()))) { - if(logger.isDebugEnabled()) { - logger.debug("Missing Database Configuration::{}", jdbcConfig); - } - return null; - } - - return new DatabaseQueryInfo(jdbcConfig, query); - } - - - private static String getQuerySource(DatabaseQueryInfo dbQueryInfo) { - String dbType = dbQueryInfo.getDbConfig().getDatabaseType(); - return DatabaseService.get(dbType).getDatabaseUrl(dbQueryInfo.getDbConfig()); - } - - - private static void setProgress(ImportingJob job, String querySource, int percent) { - job.setProgress(percent, "Reading " + querySource); - } -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseModuleImpl.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseModuleImpl.java deleted file mode 100644 index b40805238..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseModuleImpl.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.util.Properties; - -import javax.servlet.ServletConfig; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import edu.mit.simile.butterfly.ButterflyModuleImpl; - - -public class DatabaseModuleImpl extends ButterflyModuleImpl { - - private static final Logger logger = LoggerFactory.getLogger("DatabaseModuleImpl"); - - public static DatabaseModuleImpl instance; - - public static Properties extensionProperties; - - private static String DEFAULT_CREATE_PROJ_BATCH_SIZE = "100"; - private static String DEFAULT_PREVIEW_BATCH_SIZE = "100"; - - - - @Override - public void init(ServletConfig config) - throws Exception { - // TODO Auto-generated method stub - super.init(config); - - - readModuleProperty(); - - // Set the singleton. - instance = this; - - logger.trace("Database Extension module initialization completed"); - } - - public static String getImportCreateBatchSize() { - if(extensionProperties == null) { - return DEFAULT_CREATE_PROJ_BATCH_SIZE; - } - return extensionProperties.getProperty("create.batchSize", DEFAULT_CREATE_PROJ_BATCH_SIZE); - } - - public static String getImportPreviewBatchSize() { - if(extensionProperties == null) { - return DEFAULT_PREVIEW_BATCH_SIZE; - } - return extensionProperties.getProperty("preview.batchSize", DEFAULT_PREVIEW_BATCH_SIZE); - } - - private void readModuleProperty() { - // The module path - File f = getPath(); - if(logger.isDebugEnabled()) { - logger.debug("Module getPath(): {}", f.getPath()); - } - - // Load our custom properties. - File modFile = new File(f,"MOD-INF"); - if(logger.isDebugEnabled()) { - logger.debug("Module File: {}", modFile.getPath()); - } - - if (modFile.exists()) { - - extensionProperties = loadProperties (new File(modFile,"dbextension.properties")); - - } - - } - - private Properties loadProperties(File propFile) { - Properties ps = new Properties(); - try { - if (propFile.exists()) { - if(logger.isDebugEnabled()) { - logger.debug("Loading Extension properties ({})", propFile); - } - BufferedInputStream stream = null; - try { - ps = new Properties(); - stream = new BufferedInputStream(new FileInputStream(propFile)); - ps.load(stream); - - } finally { - // Close the stream. - if (stream != null) stream.close(); - } - - } - } catch (Exception e) { - logger.error("Error loading Database properties", e); - } - return ps; - } - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseService.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseService.java deleted file mode 100644 index f25d5b114..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseService.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database; - -import java.sql.Connection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.google.refine.extension.database.sqlite.SQLiteDatabaseService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.extension.database.mariadb.MariaDBDatabaseService; -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.extension.database.model.DatabaseRow; -import com.google.refine.extension.database.mysql.MySQLDatabaseService; -import com.google.refine.extension.database.pgsql.PgSQLDatabaseService; - -public abstract class DatabaseService { - - private static final Logger logger = LoggerFactory.getLogger("DatabaseService"); - - - public static class DBType { - private static Map databaseServiceMap = new HashMap(); - - static { - try { - - DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); - DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance()); - DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); - DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance()); - - } catch (Exception e) { - logger.error("Exception occurred while trying to prepare databases!", e); - } - } - - public static void registerDatabase(String name, DatabaseService db) { - - if (!databaseServiceMap.containsKey(name)) { - //throw new DatabaseServiceException(name + " cannot be registered. Database Type already exists"); - databaseServiceMap.put(name, db); - logger.info(String.format("Registered %s Database", name)); - }else { - if(logger.isDebugEnabled()) { - logger.debug(name + " Database Type already exists"); - } - - } - - } - - public static DatabaseService getJdbcServiceFromType(String name) { - return databaseServiceMap.get(name); - } - - } - - protected String getDatabaseUrl(DatabaseConfiguration dbConfig) { - int port = dbConfig.getDatabasePort(); - return "jdbc:" + dbConfig.getDatabaseType() + "://" + dbConfig.getDatabaseHost() - + ((port == 0) ? "" : (":" + port)) + "/" + dbConfig.getDatabaseName(); - } - - /** - * get Database - * @param dbType - * @return - */ - public static DatabaseService get(String dbType) { - logger.debug("get called on DatabaseService with, {}", dbType); - DatabaseService databaseService = DatabaseService.DBType.getJdbcServiceFromType(dbType.toLowerCase()); - - logger.debug("DatabaseService found: {}", databaseService.getClass()); - return databaseService; - - } - - - //Database Service APIs - public abstract Connection getConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException; - - public abstract boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException; - - public abstract DatabaseInfo connect(DatabaseConfiguration dbConfig) throws DatabaseServiceException; - - public abstract DatabaseInfo executeQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException; - - public abstract DatabaseInfo testQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException; - - public String buildLimitQuery(Integer limit, Integer offset, String query) { - if(logger.isDebugEnabled()) { - logger.info( "<<< original input query::{} >>>" , query ); - } - - final int len = query.length(); - String parsedQuery = len > 0 && query.endsWith(";") ? query.substring(0, len - 1) : query; - - - StringBuilder sb = new StringBuilder(); - sb.append("SELECT * FROM ("); - sb.append(parsedQuery); - sb.append(") data"); - - if(limit != null) { - sb.append(" LIMIT" + " " + limit); - } - - if(offset != null) { - sb.append(" OFFSET" + " " + offset); - } - sb.append(";"); - String parsedQueryOut = sb.toString(); - - if(logger.isDebugEnabled()) { - logger.info( "<<>>" , parsedQueryOut ); - } - - return parsedQueryOut; - } - - public abstract List getColumns(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException; - - public abstract List getRows(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException; - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseServiceException.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseServiceException.java deleted file mode 100644 index cb37f9040..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseServiceException.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database; - -import java.sql.SQLException; - -public class DatabaseServiceException extends Exception { - - - private static final long serialVersionUID = 1L; - - private boolean sqlException; - private String sqlState; - private int sqlCode; - - - public DatabaseServiceException(String exception) { - super(exception); - } - - - public DatabaseServiceException(boolean sqlException, String sqlState, int sqlCode, String message) { - super(message); - this.sqlException = sqlException; - this.sqlState = sqlState; - this.sqlCode = sqlCode; - - } - - - public boolean isSqlException() { - return sqlException; - } - - - public void setSqlException(boolean sqlException) { - this.sqlException = sqlException; - } - - - public String getSqlState() { - return sqlState; - } - - - public void setSqlState(String sqlState) { - this.sqlState = sqlState; - } - - - public int getSqlCode() { - return sqlCode; - } - - - public void setSqlCode(int sqlCode) { - this.sqlCode = sqlCode; - } - - public DatabaseServiceException(String string, SQLException e) { - super(string, e); - } - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseUtils.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseUtils.java deleted file mode 100644 index a5d0a06bd..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/DatabaseUtils.java +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.ListIterator; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.JsonGenerationException; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.refine.ProjectManager; -import com.google.refine.io.FileProjectManager; - -public class DatabaseUtils { - - private static final Logger logger = LoggerFactory.getLogger("DatabaseUtils"); - - - public final static String DATABASE_EXTENSION_DIR = "dbextension"; - public final static String SETTINGS_FILE_NAME = ".saved-db-connections.json"; - public final static String SAVED_CONNECTION_KEY = "savedConnections"; - - private static SimpleTextEncryptor textEncryptor = new SimpleTextEncryptor("Aa1Gb@tY7_Y"); - - - public static int getSavedConnectionsSize() { - List scList = getSavedConnections(); - if(scList == null || scList.isEmpty()) { - return 0; - } - - return scList.size(); - } - /** - * GET saved connections - * @return - */ - public static List getSavedConnections() { - ObjectMapper mapper = new ObjectMapper(); - try { - String filename = getExtensionFilePath(); - - File file = new File(filename); - if (!file.exists()) { - //logger.debug("saved connections file not found, creating new: {}", filename); - - String dirPath = getExtensionFolder(); - File dirFile = new File(dirPath); - boolean dirExists = true; - if(!dirFile.exists()) { - dirExists = dirFile.mkdir(); - } - - if(dirExists) { - - SavedConnectionContainer sc = new SavedConnectionContainer(new ArrayList()); - mapper.writerWithDefaultPrettyPrinter().writeValue(new File(filename), sc); - return sc.getSavedConnections(); - //return decryptAll(sc.getSavedConnections()); - - } - - } - //logger.debug("saved connections file found {}", filename); - SavedConnectionContainer savedConnectionContainer = mapper.readValue(new File(filename), SavedConnectionContainer.class); - //return decryptAll(savedConnectionContainer.getSavedConnections()); - return savedConnectionContainer.getSavedConnections(); - - } catch (JsonParseException e) { - logger.error("JsonParseException: {}", e); - } catch (JsonMappingException e) { - logger.error("JsonMappingException: {}", e); - } catch (IOException e) { - logger.error("IOException: {}", e); - } - return null; - } - - - - - - /** - * GET one saved connection - * @param connectionName - * @return - */ - public static DatabaseConfiguration getSavedConnection(String connectionName) { - //logger.debug("get saved connection called with connectionName: {}", connectionName); - List savedConfigurations = getSavedConnections(); - - for (DatabaseConfiguration dc : savedConfigurations) { - //logger.debug("Saved Connection : {}", dc.getConnectionName()); - if (dc.getConnectionName().equalsIgnoreCase(connectionName.trim())) { - //logger.debug("Saved Connection Found : {}", dc); - //dc.setDatabasePassword(decrypt(dc.getDatabasePassword())); - return dc; - } - } - - return null; - } - - public static String encrypt(String plainPassword) { - return textEncryptor.encrypt(plainPassword); - } - - public static String decrypt(String encodedPassword) { - return textEncryptor.decrypt(encodedPassword); - } - - public static List decryptAll(List savedConnections) { - List dbConfigs = new ArrayList(savedConnections.size()); - - for(DatabaseConfiguration d: savedConnections) { - d.setDatabasePassword(decrypt(d.getDatabasePassword())); - dbConfigs.add(d); - - } - return dbConfigs; - } - - - /** - * ADD to saved connections - * @param dbConfig - */ - public static void addToSavedConnections(DatabaseConfiguration dbConfig){ - - try { - ObjectMapper mapper = new ObjectMapper(); - String savedConnectionFile = getExtensionFilePath(); - SavedConnectionContainer savedConnectionContainer = mapper.readValue(new File(savedConnectionFile), SavedConnectionContainer.class); - savedConnectionContainer.getSavedConnections().add(dbConfig); - - mapper.writerWithDefaultPrettyPrinter().writeValue(new File(savedConnectionFile), savedConnectionContainer); - - } catch (JsonGenerationException e1) { - logger.error("JsonGenerationException: {}", e1); - // e1.printStackTrace(); - } catch (JsonMappingException e1) { - logger.error("JsonMappingException: {}", e1); - // e1.printStackTrace(); - } catch (IOException e1) { - logger.error("IOException: {}", e1); - // e1.printStackTrace(); - } - } - - - public static void deleteAllSavedConnections() { - if(logger.isDebugEnabled()) { - logger.debug("delete All Saved Connections called..."); - } - - try { - - List savedConnections = getSavedConnections(); - if(logger.isDebugEnabled()) { - logger.debug("Size before delete SavedConnections :: {}", savedConnections.size()); - } - - ArrayList newSavedConns = new ArrayList(); - - ObjectMapper mapper = new ObjectMapper(); - String savedConnectionFile = getExtensionFilePath(); - SavedConnectionContainer savedConnectionContainer = mapper.readValue(new File(savedConnectionFile), SavedConnectionContainer.class); - savedConnectionContainer.setSavedConnections(newSavedConns); - - if(logger.isDebugEnabled()) { - logger.debug("Size after delete SavedConnections :: {}", savedConnectionContainer.getSavedConnections().size()); - } - mapper.writerWithDefaultPrettyPrinter().writeValue(new File(savedConnectionFile), savedConnectionContainer); - - } catch (JsonGenerationException e1) { - logger.error("JsonGenerationException: {}", e1); - // e1.printStackTrace(); - } catch (JsonMappingException e1) { - logger.error("JsonMappingException: {}", e1); - // e1.printStackTrace(); - } catch (IOException e1) { - logger.error("IOException: {}", e1); - // e1.printStackTrace(); - } - - } - - /** - * DELETE saved connections - * @param connectionName - */ - public static void deleteSavedConnections(String connectionName) { - if(logger.isDebugEnabled()) { - logger.debug("deleteSavedConnections called with: {}", connectionName); - } - - try { - - List savedConnections = getSavedConnections();; - if(logger.isDebugEnabled()) { - logger.debug("Size before delete SavedConnections :: {}", savedConnections.size()); - } - - ArrayList newSavedConns = new ArrayList(); - for(DatabaseConfiguration dc: savedConnections) { - if(!dc.getConnectionName().equalsIgnoreCase(connectionName.trim())) { - newSavedConns.add(dc); - } - - } - - ObjectMapper mapper = new ObjectMapper(); - String savedConnectionFile = getExtensionFilePath(); - SavedConnectionContainer savedConnectionContainer = mapper.readValue(new File(savedConnectionFile), SavedConnectionContainer.class); - savedConnectionContainer.setSavedConnections(newSavedConns); - - if(logger.isDebugEnabled()) { - logger.debug("Size after delete SavedConnections :: {}", savedConnectionContainer.getSavedConnections().size()); - } - mapper.writerWithDefaultPrettyPrinter().writeValue(new File(savedConnectionFile), savedConnectionContainer); - - } catch (JsonGenerationException e1) { - logger.error("JsonGenerationException: {}", e1); - // e1.printStackTrace(); - } catch (JsonMappingException e1) { - logger.error("JsonMappingException: {}", e1); - // e1.printStackTrace(); - } catch (IOException e1) { - logger.error("IOException: {}", e1); - // e1.printStackTrace(); - } - } - - /** - * EDIT saved connections - * @param jdbcConfig - */ - public static void editSavedConnections(DatabaseConfiguration jdbcConfig) { - if(logger.isDebugEnabled()) { - logger.debug("Edit SavedConnections called with: {}", jdbcConfig); - } - - - try { - ObjectMapper mapper = new ObjectMapper(); - String savedConnectionFile = getExtensionFilePath(); - SavedConnectionContainer savedConnectionContainer = mapper.readValue(new File(savedConnectionFile), SavedConnectionContainer.class); - - List savedConnections = savedConnectionContainer.getSavedConnections(); - - ListIterator savedConnArrayIter = (ListIterator) savedConnections.listIterator(); - - while (savedConnArrayIter.hasNext()) { - DatabaseConfiguration sc = (DatabaseConfiguration) savedConnArrayIter.next(); - - if (sc.getConnectionName().equals(jdbcConfig.getConnectionName())) { - savedConnArrayIter.remove(); - } - - } - - savedConnections.add(jdbcConfig); - savedConnectionContainer.setSavedConnections(savedConnections); - - mapper.writerWithDefaultPrettyPrinter().writeValue(new File(savedConnectionFile), savedConnectionContainer); - - } catch (JsonGenerationException e1) { - logger.error("JsonGenerationException: {}", e1); - e1.printStackTrace(); - } catch (JsonMappingException e1) { - logger.error("JsonMappingException: {}", e1); - e1.printStackTrace(); - } catch (IOException e1) { - logger.error("IOException: {}", e1); - e1.printStackTrace(); - } - } - - public static String getExtensionFilePath(){ - File dir = ((FileProjectManager) ProjectManager.singleton).getWorkspaceDir(); - String fileSep = System.getProperty("file.separator"); - String filename = dir.getPath() + fileSep + DATABASE_EXTENSION_DIR + fileSep + SETTINGS_FILE_NAME; - - logger.debug("** extension file name: {} **", filename); - return filename; - } - public static String getExtensionFolder(){ - File dir = ((FileProjectManager) ProjectManager.singleton).getWorkspaceDir(); - String fileSep = System.getProperty("file.separator"); - String filename = dir.getPath() + fileSep + DATABASE_EXTENSION_DIR; - return filename; - } - - public static DatabaseColumnType getDbColumnType(int dbColumnType) { - - switch (dbColumnType) { - case java.sql.Types.BIGINT: - return DatabaseColumnType.NUMBER; - case java.sql.Types.FLOAT: - return DatabaseColumnType.FLOAT; - case java.sql.Types.REAL: - return DatabaseColumnType.DOUBLE; - case java.sql.Types.DOUBLE: - return DatabaseColumnType.DOUBLE; - case java.sql.Types.NUMERIC: - return DatabaseColumnType.NUMBER; - case java.sql.Types.DECIMAL: - return DatabaseColumnType.STRING; - case java.sql.Types.CHAR: - return DatabaseColumnType.STRING; - case java.sql.Types.VARCHAR: - return DatabaseColumnType.STRING; - case java.sql.Types.LONGVARCHAR: - return DatabaseColumnType.STRING; - case java.sql.Types.DATE: - return DatabaseColumnType.DATE; - case java.sql.Types.TIME: - return DatabaseColumnType.DATETIME; - case java.sql.Types.TIMESTAMP: - return DatabaseColumnType.DATETIME; - case java.sql.Types.BINARY: - return DatabaseColumnType.STRING; - case java.sql.Types.VARBINARY: - return DatabaseColumnType.STRING; - case java.sql.Types.LONGVARBINARY: - return DatabaseColumnType.STRING; - case java.sql.Types.NULL: - return DatabaseColumnType.STRING; - case java.sql.Types.OTHER: - return DatabaseColumnType.STRING; - case java.sql.Types.JAVA_OBJECT: - return DatabaseColumnType.STRING; - case java.sql.Types.DISTINCT: - return DatabaseColumnType.STRING; - case java.sql.Types.STRUCT: - return DatabaseColumnType.STRING; - case java.sql.Types.ARRAY: - return DatabaseColumnType.STRING; - case java.sql.Types.BLOB: - return DatabaseColumnType.STRING; - case java.sql.Types.CLOB: - return DatabaseColumnType.STRING; - case java.sql.Types.REF: - return DatabaseColumnType.STRING; - case java.sql.Types.BOOLEAN: - return DatabaseColumnType.BOOLEAN; - case java.sql.Types.INTEGER: - return DatabaseColumnType.NUMBER; - - default: - return DatabaseColumnType.STRING; - } - - } - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/SQLType.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/SQLType.java deleted file mode 100644 index 2871b6abd..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/SQLType.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database; - - - -import java.util.HashMap; -import java.util.Map; - - -public final class SQLType { - - private static final Map jdbcDriverRegistry = new HashMap(); - private final DriverContainer driverContainer; - - private SQLType(DriverContainer container) { - this.driverContainer = container; - } - - public static SQLType forName(String name) { - for (SQLType sqlType : jdbcDriverRegistry.values()) { - if (sqlType.getIdentifier().equalsIgnoreCase(name)) { - return sqlType; - } - } - return null; - } - - public static SQLType registerSQLDriver(String identifier, String classpath) { - return registerSQLDriver(identifier, classpath, true); - } - - public static SQLType registerSQLDriver(String identifier, String classpath, boolean useJDBCManager) { - DriverContainer driverContainer = new DriverContainer(identifier, classpath, useJDBCManager); - if (!jdbcDriverRegistry.containsKey(driverContainer)) { - SQLType newType = new SQLType(driverContainer); - jdbcDriverRegistry.put(driverContainer, newType); - return newType; - } - return null; - } - - - public String getClassPath() { - return this.driverContainer.classpath; - } - - public String getIdentifier() { - return this.driverContainer.identifier; - } - - public boolean usesJDBCManager() { - return this.driverContainer.useJDBCManager; - } - - - private static class DriverContainer { - - public final String classpath; - public final String identifier; - public final boolean useJDBCManager; - - private DriverContainer(String identifier, String classpath, boolean useJDBCManager) { - this.classpath = classpath; - this.identifier = identifier; - this.useJDBCManager = useJDBCManager; - } - - public final boolean equals(Object obj) { - return obj instanceof DriverContainer && ((DriverContainer) obj).classpath.equals(this.classpath) - && ((DriverContainer) obj).identifier.equals(this.identifier) - && ((DriverContainer) obj).useJDBCManager == this.useJDBCManager; - } - } -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/SavedConnectionContainer.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/SavedConnectionContainer.java deleted file mode 100644 index 15a45f731..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/SavedConnectionContainer.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database; - -import java.util.List; - -public class SavedConnectionContainer { - private List savedConnections; - - public List getSavedConnections() { - return savedConnections; - } - - - public void setSavedConnections(List savedConnections) { - this.savedConnections = savedConnections; - } - - - - public SavedConnectionContainer(List savedConnections) { - super(); - this.savedConnections = savedConnections; - } - - - public SavedConnectionContainer() { - - } - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/SimpleTextEncryptor.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/SimpleTextEncryptor.java deleted file mode 100644 index ad5f20d95..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/SimpleTextEncryptor.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database; - -import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; -import org.jasypt.util.text.TextEncryptor; - - -public class SimpleTextEncryptor implements TextEncryptor { - - private final StandardPBEStringEncryptor encryptor; - - @Override - public String decrypt(String encryptedMessage) { - return this.encryptor.decrypt(encryptedMessage); - } - - @Override - public String encrypt(String message) { - return this.encryptor.encrypt(message); - } - - public SimpleTextEncryptor(String passwordChars) { - super(); - - this.encryptor = new StandardPBEStringEncryptor(); - this.encryptor.setAlgorithm("PBEWithMD5AndDES"); - this.encryptor.setPasswordCharArray(passwordChars.toCharArray()); - } - - - - public void setPassword(final String password) { - this.encryptor.setPassword(password); - } - - - /** - * Sets a password, as a char[] - * - * @since 1.8 - * @param password the password to be set. - */ - public void setPasswordCharArray(final char[] password) { - this.encryptor.setPasswordCharArray(password); - } - - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/ConnectCommand.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/ConnectCommand.java deleted file mode 100644 index 0e1fa1e6a..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/ConnectCommand.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.cmd; - -import java.io.IOException; -import java.io.Writer; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.http.HttpStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -//import com.google.refine.ProjectManager; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.util.ParsingUtilities; - - -public class ConnectCommand extends DatabaseCommand { - - private static final Logger logger = LoggerFactory.getLogger("ConnectCommand"); - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - if(!hasValidCSRFToken(request)) { - respondCSRFError(response); - return; - } - - DatabaseConfiguration databaseConfiguration = getJdbcConfiguration(request); - if(logger.isDebugEnabled()) { - logger.debug("ConnectCommand::Post::{}", databaseConfiguration); - } - // ProjectManager.singleton.setBusy(true); - try { - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - Writer w = response.getWriter(); - JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w); - ObjectMapper mapperObj = new ObjectMapper(); - - try { - DatabaseInfo databaseInfo = DatabaseService.get(databaseConfiguration.getDatabaseType()) - .connect(databaseConfiguration); - String databaseInfoString = mapperObj.writeValueAsString(databaseInfo); - response.setStatus(HttpStatus.SC_OK); - writer.writeStartObject(); - writer.writeStringField("code", "ok"); - writer.writeStringField("databaseInfo", databaseInfoString); - writer.writeEndObject(); - - } catch (DatabaseServiceException e) { - logger.error("ConnectCommand::Post::DatabaseServiceException::{}", e); - sendError(HttpStatus.SC_UNAUTHORIZED,response, e); - }catch (Exception e) { - logger.error("ConnectCommand::Post::Exception::{}", e); - sendError(HttpStatus.SC_UNAUTHORIZED,response, e); - } finally { - writer.flush(); - writer.close(); - w.close(); - } - } catch (Exception e) { - logger.error("ConnectCommand::Post::Exception::{}", e); - throw new ServletException(e); - } -// finally { -// // ProjectManager.singleton.setBusy(false); -// } - - - } - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/DatabaseCommand.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/DatabaseCommand.java deleted file mode 100644 index 171dccac1..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/DatabaseCommand.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.cmd; - -import java.io.IOException; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.commands.Command; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseServiceException; - -public abstract class DatabaseCommand extends Command { - - private static final Logger logger = LoggerFactory.getLogger("DatabaseCommand"); - - /** - * - * @param request - * @return - */ - protected DatabaseConfiguration getJdbcConfiguration(HttpServletRequest request) { - DatabaseConfiguration jdbcConfig = new DatabaseConfiguration(); - - jdbcConfig.setConnectionName(request.getParameter("connectionName")); - jdbcConfig.setDatabaseType(request.getParameter("databaseType")); - jdbcConfig.setDatabaseHost(request.getParameter("databaseServer")); - - String dbPort = request.getParameter("databasePort"); - if(dbPort != null) { - try { - jdbcConfig.setDatabasePort(Integer.parseInt(dbPort)); - }catch(NumberFormatException nfe) {} - } - - jdbcConfig.setDatabaseUser(request.getParameter("databaseUser")); - jdbcConfig.setDatabasePassword(request.getParameter("databasePassword")); - jdbcConfig.setDatabaseName(request.getParameter("initialDatabase")); - jdbcConfig.setDatabaseSchema(request.getParameter("initialSchema")); - - if(logger.isDebugEnabled()) { - logger.debug("JDBC Configuration: {}", jdbcConfig); - } - return jdbcConfig; - } - /** - * - * @param status - * @param response - * @param writer - * @param e - * @throws IOException - */ - protected void sendError(int status, HttpServletResponse response, Exception e) - throws IOException { - - //logger.info("sendError::{}", writer); - response.sendError(status, e.getMessage()); - - } - /** - * - * @param status - * @param response - * @param writer - * @param e - * @throws IOException - */ - protected void sendError(int status, HttpServletResponse response, DatabaseServiceException e) - throws IOException { - - String message = ""; - - if(e.getSqlState() != null) { - - message = message + "SqlCode:" + e.getSqlCode() + "SqlState" + e.getSqlState(); - } - - message = message + e.getMessage(); - - response.sendError(status, e.getMessage()); - - } - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/ExecuteQueryCommand.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/ExecuteQueryCommand.java deleted file mode 100644 index 863ec9423..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/ExecuteQueryCommand.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.cmd; - -import java.io.IOException; -import java.io.Writer; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.http.HttpStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -//import com.google.refine.ProjectManager; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.util.ParsingUtilities; - - -public class ExecuteQueryCommand extends DatabaseCommand { - - private static final Logger logger = LoggerFactory.getLogger("ExecuteQueryCommand"); - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - if(!hasValidCSRFToken(request)) { - respondCSRFError(response); - return; - } - - DatabaseConfiguration databaseConfiguration = getJdbcConfiguration(request); - String query = request.getParameter("queryString"); - if(logger.isDebugEnabled()) { - logger.debug("QueryCommand::Post::DatabaseConfiguration::{}::Query::{} " ,databaseConfiguration, query); - } - - //ProjectManager.singleton.setBusy(true); - try { - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - Writer w = response.getWriter(); - JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w); - - try { - DatabaseInfo databaseInfo = DatabaseService.get(databaseConfiguration.getDatabaseType()) - .executeQuery(databaseConfiguration, query); - ObjectMapper mapperObj = new ObjectMapper(); - - response.setStatus(HttpStatus.SC_OK); - String jsonStr = mapperObj.writeValueAsString(databaseInfo); - - if(logger.isDebugEnabled()) { - logger.debug("QueryCommand::Post::Result::{} " ,jsonStr); - } - - - writer.writeStartObject(); - writer.writeStringField("code", "ok"); - writer.writeStringField("QueryResult", jsonStr); - writer.writeEndObject(); - - - } catch (DatabaseServiceException e) { - logger.error("QueryCommand::Post::DatabaseServiceException::{}", e); - sendError(HttpStatus.SC_BAD_REQUEST, response, e); - - } catch (Exception e) { - logger.error("QueryCommand::Post::Exception::{}", e); - sendError(HttpStatus.SC_BAD_REQUEST,response, e); - } finally { - writer.flush(); - writer.close(); - w.close(); - } - } catch (Exception e) { - logger.error("QueryCommand::Post::Exception::{}", e); - throw new ServletException(e); - } -// finally { -// // ProjectManager.singleton.setBusy(false); -// } - - - } - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/SavedConnectionCommand.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/SavedConnectionCommand.java deleted file mode 100644 index 80e4955d2..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/SavedConnectionCommand.java +++ /dev/null @@ -1,358 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.cmd; - -import java.io.IOException; -import java.io.Writer; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.http.HttpStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseUtils; -import com.google.refine.util.ParsingUtilities; - - -public class SavedConnectionCommand extends DatabaseCommand { - - private static final Logger logger = LoggerFactory.getLogger("SavedConnectionCommand"); - - private static final Pattern CONN_NAME_PATTERN = Pattern.compile("^[a-zA-Z0-9._-]*"); - private static final Pattern DATABASE_PORT_PATTERN = Pattern.compile("^[0-9]*"); - - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - if(logger.isDebugEnabled()) { - logger.debug("SavedConnectionCommand::Get::connectionName::{}", request.getParameter("connectionName")); - } - - String connectionName = request.getParameter("connectionName"); - try { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - if(connectionName == null || connectionName.isEmpty()) { - writeSavedConnectionResponse(response); - }else { - - DatabaseConfiguration savedConnection = DatabaseUtils.getSavedConnection(connectionName); - writeSavedConnectionResponse(response, savedConnection); - - } - - } catch (Exception e) { - logger.error("Exception while loading settings {}", e); - } - } - - - @Override - public void doDelete(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - - if(logger.isDebugEnabled()) { - logger.debug("SavedConnectionCommand::Delete Connection: {}", request.getParameter("connectionName")); - } - - String connectionName = request.getParameter("connectionName"); - - DatabaseConfiguration savedConn = DatabaseUtils.getSavedConnection(connectionName); - if(savedConn == null) { - //logger.error("Connection With name:: {} does not exist!", request.getParameter("connectionName")); - response.sendError(HttpStatus.SC_BAD_REQUEST, "Connection with name " + connectionName + " does not exists!"); - response.flushBuffer(); - return; - } - - try { - - DatabaseUtils.deleteSavedConnections(connectionName); - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - writeSavedConnectionResponse(response); - - } catch (Exception e) { - logger.error("Exception while Deleting Connection with name: {}, error:{}",connectionName, e); - } - } - - /** - * - * @param response - * @param savedConnection - * @throws IOException - * @throws JSONException - */ - private void writeSavedConnectionResponse(HttpServletResponse response, DatabaseConfiguration savedConnection) throws IOException { - Writer w = response.getWriter(); - try { - JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w); - - writer.writeStartObject(); - writer.writeArrayFieldStart(DatabaseUtils.SAVED_CONNECTION_KEY); - - writer.writeStartObject(); - - - writer.writeStringField("connectionName", savedConnection.getConnectionName()); - writer.writeStringField("databaseType", savedConnection.getDatabaseType()); - writer.writeStringField("databaseHost", savedConnection.getDatabaseHost()); - writer.writeNumberField("databasePort", savedConnection.getDatabasePort()); - writer.writeStringField("databaseName", savedConnection.getDatabaseName()); - - - String dbPasswd = savedConnection.getDatabasePassword(); - if(dbPasswd != null && !dbPasswd.isEmpty()) { - dbPasswd = DatabaseUtils.decrypt(savedConnection.getDatabasePassword()); - } - writer.writeStringField("databasePassword", dbPasswd); - writer.writeStringField("databaseSchema", savedConnection.getDatabaseSchema()); - writer.writeStringField("databaseUser", savedConnection.getDatabaseUser()); - - - writer.writeEndObject(); - writer.writeEndArray(); - - writer.writeEndObject(); - writer.flush(); - writer.close(); - - }finally { - w.flush(); - w.close(); - } - - } - /** - * - * @param response - * @throws IOException - * @throws JSONException - */ - private void writeSavedConnectionResponse(HttpServletResponse response) throws IOException { - Writer w = response.getWriter(); - try { - - List savedConnections = DatabaseUtils.getSavedConnections(); - JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w); - - writer.writeStartObject(); - writer.writeArrayFieldStart(DatabaseUtils.SAVED_CONNECTION_KEY); - - int size = savedConnections.size(); - - for (int i = 0; i < size; i++) { - - writer.writeStartObject(); - DatabaseConfiguration dbConfig = (DatabaseConfiguration) savedConnections.get(i); - - writer.writeStringField("connectionName", dbConfig.getConnectionName()); - - writer.writeStringField("databaseType", dbConfig.getDatabaseType()); - - writer.writeStringField("databaseHost", dbConfig.getDatabaseHost()); - - writer.writeNumberField("databasePort", dbConfig.getDatabasePort()); - - writer.writeStringField("databaseName", dbConfig.getDatabaseName()); - - - String dbPasswd = dbConfig.getDatabasePassword(); - if(dbPasswd != null && !dbPasswd.isEmpty()) { - dbPasswd = DatabaseUtils.decrypt(dbConfig.getDatabasePassword()); - } - // writer.value(dbConfig.getDatabasePassword()); - writer.writeStringField("databasePassword", dbPasswd); - - writer.writeStringField("databaseSchema", dbConfig.getDatabaseSchema()); - - writer.writeStringField("databaseUser", dbConfig.getDatabaseUser()); - - writer.writeEndObject(); - - } - writer.writeEndArray(); - writer.writeEndObject(); - writer.flush(); - writer.close(); - // logger.info("Saved Connection Get Response sent"); - } finally { - w.flush(); - w.close(); - } - } - - /** - * Add a new Saved JDBC connection configuration - */ - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - if(!hasValidCSRFToken(request)) { - respondCSRFError(response); - return; - } - - if(logger.isDebugEnabled()) { - logger.debug("doPost Connection: {}", request.getParameter("connectionName")); - } - - - DatabaseConfiguration jdbcConfig = getJdbcConfiguration(request); - - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - if(jdbcConfig.getConnectionName() == null) { - response.sendError(HttpStatus.SC_BAD_REQUEST, "Connection Name is Required!"); - response.flushBuffer(); - return; - } - - if(!validateInput(jdbcConfig.getConnectionName(), CONN_NAME_PATTERN)) { - logger.warn("Invalid Connection Name: {}", jdbcConfig.getConnectionName()); - response.sendError(HttpStatus.SC_BAD_REQUEST, "Connection Name is Invalid. Expecting [a-zA-Z0-9._-]"); - response.flushBuffer(); - return; - } - - if(!validateInput("" + jdbcConfig.getDatabasePort(), DATABASE_PORT_PATTERN)) { - logger.warn("Invalid Database Port: {}", jdbcConfig.getDatabasePort()); - response.sendError(HttpStatus.SC_BAD_REQUEST, "Database Port Invalid. Expecting Numeric values only"); - response.flushBuffer(); - return; - } - - - DatabaseConfiguration savedConn = DatabaseUtils.getSavedConnection(jdbcConfig.getConnectionName()); - if(savedConn != null) { - response.sendError(HttpStatus.SC_BAD_REQUEST, "Connection with name " + jdbcConfig.getConnectionName() + " already exists!"); - response.flushBuffer(); - return; - } - - - if(jdbcConfig.getDatabasePassword() != null) { - //logger.debug("SavedConnectionCommand::Post::password::{}", jdbcConfig.getDatabasePassword()); - jdbcConfig.setDatabasePassword(DatabaseUtils.encrypt(jdbcConfig.getDatabasePassword())); - } - - DatabaseUtils.addToSavedConnections(jdbcConfig); - - try { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - writeSavedConnectionResponse(response); - } catch (Exception e) { - logger.error("Exception while loading settings {}", e); - } - - } - - private boolean validateInput(String input, Pattern pattern){ - Matcher matcher = pattern.matcher(input); - if(matcher.matches()){ - return true; - } - return false; - } - - @Override - public void doPut(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - - - if(logger.isDebugEnabled()) { - logger.debug("databaseType::{} " , request.getParameter("databaseHost")); - } - // logger.info("databaseHost::{} " , request.getParameter("databaseServer")); - - DatabaseConfiguration jdbcConfig = getJdbcConfiguration(request); - StringBuilder sb = new StringBuilder(); - boolean error = false; - if(jdbcConfig.getConnectionName() == null) { - sb.append("Connection Name, "); - error = true; - } - if(jdbcConfig.getDatabaseHost() == null) { - sb.append("Database Host, "); - error = true; - } - if(jdbcConfig.getDatabaseUser() == null) { - sb.append("Database User, "); - error = true; - } - if(jdbcConfig.getDatabaseName() == null) { - sb.append("Database Name, "); - error = true; - } - if(error) { - sb.append(" is missing"); - logger.debug("Connection Parameter errors::{}", sb.toString()); - response.sendError(HttpStatus.SC_BAD_REQUEST, sb.toString()); - } - - if(logger.isDebugEnabled()) { - logger.debug("Connection Config:: {}", jdbcConfig.getConnectionName()); - } - - if(jdbcConfig.getDatabasePassword() != null) { - jdbcConfig.setDatabasePassword(DatabaseUtils.encrypt(jdbcConfig.getDatabasePassword())); - } - - DatabaseUtils.editSavedConnections(jdbcConfig); - - try { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - writeSavedConnectionResponse(response); - - } catch (Exception e) { - logger.error("Exception while loading settings {}", e); - } - } - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/TestConnectCommand.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/TestConnectCommand.java deleted file mode 100644 index af7b6f0f9..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/TestConnectCommand.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.cmd; - -import java.io.IOException; -import java.io.Writer; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.http.HttpStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.util.ParsingUtilities; - - - -public class TestConnectCommand extends DatabaseCommand { - - private static final Logger logger = LoggerFactory.getLogger("TestConnectCommand"); - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - if(!hasValidCSRFToken(request)) { - respondCSRFError(response); - return; - } - - DatabaseConfiguration databaseConfiguration = getJdbcConfiguration(request); - if(logger.isDebugEnabled()) { - logger.debug("TestConnectCommand::Post::{}", databaseConfiguration); - } - - - //ProjectManager.singleton.setBusy(true); - try { - - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - Writer w = response.getWriter(); - JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w); - - try { - - boolean connectionTestResult = DatabaseService.get(databaseConfiguration.getDatabaseType()) - .testConnection(databaseConfiguration); - - response.setStatus(HttpStatus.SC_OK); - writer.writeStartObject(); - - writer.writeBooleanField("connectionResult", connectionTestResult); - writer.writeStringField("code", "ok"); - writer.writeEndObject(); - - } catch (DatabaseServiceException e) { - logger.error("TestConnectCommand::Post::DatabaseServiceException::{}", e); - sendError(HttpStatus.SC_UNAUTHORIZED,response, e); - } finally { - writer.flush(); - writer.close(); - w.close(); - } - } catch (Exception e) { - logger.error("TestConnectCommand::Post::Exception::{}", e); - throw new ServletException(e); - } finally { - //ProjectManager.singleton.setBusy(false); - } - - - } - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/TestQueryCommand.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/TestQueryCommand.java deleted file mode 100644 index 82610f38b..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/cmd/TestQueryCommand.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.cmd; - -import java.io.IOException; -import java.io.Writer; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.http.HttpStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -//import com.google.refine.ProjectManager; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.util.ParsingUtilities; - - -public class TestQueryCommand extends DatabaseCommand { - - private static final Logger logger = LoggerFactory.getLogger("TestQueryCommand"); - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - if(!hasValidCSRFToken(request)) { - respondCSRFError(response); - return; - } - - DatabaseConfiguration dbConfig = getJdbcConfiguration(request); - String query = request.getParameter("query"); - - if(logger.isDebugEnabled()) { - logger.debug("TestQueryCommand::Post::DatabaseConfiguration::{}::Query::{} " ,dbConfig, query); - } - - - //ProjectManager.singleton.setBusy(true); - try { - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - Writer w = response.getWriter(); - JsonGenerator writer = ParsingUtilities.mapper.getFactory().createGenerator(w); - - try { - DatabaseInfo databaseInfo = DatabaseService.get(dbConfig.getDatabaseType()) - .testQuery(dbConfig, query); - ObjectMapper mapperObj = new ObjectMapper(); - - response.setStatus(HttpStatus.SC_OK); - String jsonStr = mapperObj.writeValueAsString(databaseInfo); - if(logger.isDebugEnabled()) { - logger.debug("TestQueryCommand::Post::Result::{} " ,jsonStr); - } - - writer.writeStartObject(); - writer.writeStringField("code", "ok"); - writer.writeStringField("QueryResult", jsonStr); - writer.writeEndObject(); - - - } catch (DatabaseServiceException e) { - logger.error("TestQueryCommand::Post::DatabaseServiceException::{}", e); - sendError(HttpStatus.SC_BAD_REQUEST, response, e); - - } catch (Exception e) { - logger.error("TestQueryCommand::Post::Exception::{}", e); - sendError(HttpStatus.SC_BAD_REQUEST,response, e); - } finally { - writer.flush(); - writer.close(); - w.close(); - } - } catch (Exception e) { - logger.error("TestQueryCommand::Post::Exception::{}", e); - throw new ServletException(e); - } -// finally { -// // ProjectManager.singleton.setBusy(false); -// } - - - } - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManager.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManager.java deleted file mode 100644 index 4af014afc..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManager.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.mariadb; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.SQLType; - - - -public class MariaDBConnectionManager { - - private static final Logger logger = LoggerFactory.getLogger("MariaDBConnectionManager"); - private Connection connection; - private SQLType type; - - private static MariaDBConnectionManager instance; - - /** - * - * @param type - * @param databaseConfiguration - * @throws SQLException - */ - private MariaDBConnectionManager() { - type = SQLType.forName(MariaDBDatabaseService.DB_NAME); - - } - - - - - /** - * Create a new instance of this connection manager. - * - * @return an instance of the manager - * - * @throws DatabaseServiceException - */ - public static MariaDBConnectionManager getInstance() throws DatabaseServiceException { - if (instance == null) { - //logger.info("::Creating new MariaDB Connection Manager ::"); - instance = new MariaDBConnectionManager(); - - } - return instance; - } - - - /** - * Get the SQL Database type. - * - * @return the type - */ - public SQLType getType() { - return this.type; - } - - /** - * testConnection - * @param databaseConfiguration - * @return - */ - public boolean testConnection(DatabaseConfiguration databaseConfiguration) throws DatabaseServiceException{ - - try { - boolean connResult = false; - - Connection conn = getConnection(databaseConfiguration, true); - if(conn != null) { - connResult = true; - conn.close(); - } - - return connResult; - - } - catch (SQLException e) { - logger.error("Test connection Failed!", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - - } - - /** - * Get a connection form the connection pool. - * - * @return connection from the pool - */ - public Connection getConnection(DatabaseConfiguration databaseConfiguration, boolean forceNewConnection) throws DatabaseServiceException{ - try { - - // logger.info("connection::{}, forceNewConnection: {}", connection, forceNewConnection); - - if (connection != null && !forceNewConnection) { - // logger.debug("connection closed::{}", connection.isClosed()); - if (!connection.isClosed()) { - if(logger.isDebugEnabled()) { - logger.debug("Returning existing connection::{}", connection); - } - - - return connection; - } - } - - Class.forName(type.getClassPath()); - DriverManager.setLoginTimeout(10); - String dbURL = getDatabaseUrl(databaseConfiguration); - connection = DriverManager.getConnection(dbURL, databaseConfiguration.getDatabaseUser(), - databaseConfiguration.getDatabasePassword()); - - if(logger.isDebugEnabled()) { - logger.debug("*** Acquired New connection for ::{} **** ", dbURL); - } - - - return connection; - - - } catch (ClassNotFoundException e) { - logger.error("Jdbc Driver not found", e); - throw new DatabaseServiceException(e.getMessage()); - } catch (SQLException e) { - logger.error("SQLException::Couldn't get a Connection!", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - - public void shutdown() { - - if (connection != null) { - try { - connection.close(); - } - catch (SQLException e) { - logger.warn("Non-Managed connection could not be closed. Whoops!", e); - } - } - - } - - - - private static String getDatabaseUrl(DatabaseConfiguration dbConfig) { - - int port = dbConfig.getDatabasePort(); - return "jdbc:" + dbConfig.getDatabaseType().toLowerCase() + "://" + dbConfig.getDatabaseHost() - + ((port == 0) ? "" : (":" + port)) + "/" + dbConfig.getDatabaseName(); - - } -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseService.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseService.java deleted file mode 100644 index af8b4a948..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseService.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.mariadb; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; - -import org.mariadb.jdbc.MariaDbResultSetMetaData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.DatabaseUtils; -import com.google.refine.extension.database.SQLType; -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.extension.database.model.DatabaseRow; - -public class MariaDBDatabaseService extends DatabaseService { - private static final Logger logger = LoggerFactory.getLogger("MariaDBDatabaseService"); - public static final String DB_NAME = "mariadb"; - public static final String DB_DRIVER = "org.mariadb.jdbc.Driver"; - private static MariaDBDatabaseService instance; - - private MariaDBDatabaseService() { - } - - public static MariaDBDatabaseService getInstance() { - if (instance == null) { - SQLType.registerSQLDriver(DB_NAME, DB_DRIVER); - instance = new MariaDBDatabaseService(); - if(logger.isDebugEnabled()) { - logger.debug("MariaDBDatabaseService Instance: {}", instance); - } - } - return instance; - } - - @Override - public boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException{ - return MariaDBConnectionManager.getInstance().testConnection(dbConfig); - } - - @Override - public DatabaseInfo connect(DatabaseConfiguration dbConfig) throws DatabaseServiceException{ - return getMetadata(dbConfig); - } - - @Override - public DatabaseInfo executeQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException{ - try { - Connection connection = MariaDBConnectionManager.getInstance().getConnection(dbConfig, false); - Statement statement = connection.createStatement(); - ResultSet queryResult = statement.executeQuery(query); - MariaDbResultSetMetaData metadata = (MariaDbResultSetMetaData)queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - ArrayList columns = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - DatabaseColumn dc = new DatabaseColumn( - metadata.getColumnName(i), - metadata.getColumnLabel(i), - DatabaseUtils.getDbColumnType(metadata.getColumnType(i)), - metadata.getColumnDisplaySize(i)); - columns.add(dc); - } - int index = 0; - List rows = new ArrayList(); - while (queryResult.next()) { - DatabaseRow row = new DatabaseRow(); - row.setIndex(index); - List values = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - values.add(queryResult.getString(i)); - } - row.setValues(values); - rows.add(row); - index++; - } - DatabaseInfo dbInfo = new DatabaseInfo(); - dbInfo.setColumns(columns); - dbInfo.setRows(rows); - return dbInfo; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } finally { - MariaDBConnectionManager.getInstance().shutdown(); - } - } - - /** - * @param connectionInfo - * @return - * @throws DatabaseServiceException - */ - private DatabaseInfo getMetadata(DatabaseConfiguration connectionInfo) throws DatabaseServiceException { - try { - Connection connection = MariaDBConnectionManager.getInstance().getConnection(connectionInfo, true); - if(connection != null) { - java.sql.DatabaseMetaData metadata = connection.getMetaData(); - int dbMajorVersion = metadata.getDatabaseMajorVersion(); - int dbMinorVersion = metadata.getDatabaseMinorVersion(); - String dbProductVersion = metadata.getDatabaseProductVersion(); - String dbProductName = metadata.getDatabaseProductName(); - DatabaseInfo dbInfo = new DatabaseInfo(); - dbInfo.setDatabaseMajorVersion(dbMajorVersion); - dbInfo.setDatabaseMinorVersion(dbMinorVersion); - dbInfo.setDatabaseProductVersion(dbProductVersion); - dbInfo.setDatabaseProductName(dbProductName); - return dbInfo; - } - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - - return null; - } - - @Override - public ArrayList getColumns(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException{ - try { - Connection connection = MariaDBConnectionManager.getInstance().getConnection(dbConfig, true); - Statement statement = connection.createStatement(); - ResultSet queryResult = statement.executeQuery(query); - MariaDbResultSetMetaData metadata = (MariaDbResultSetMetaData) queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - ArrayList columns = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - DatabaseColumn dc = new DatabaseColumn(metadata.getColumnName(i), metadata.getColumnLabel(i), - DatabaseUtils.getDbColumnType(metadata.getColumnType(i)), metadata.getColumnDisplaySize(i)); - columns.add(dc); - } - return columns; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - @Override - public List getRows(DatabaseConfiguration dbConfig, String query) - throws DatabaseServiceException { - try { - Connection connection = MariaDBConnectionManager.getInstance().getConnection(dbConfig, false); - Statement statement = connection.createStatement(); - ResultSet queryResult = statement.executeQuery(query); - MariaDbResultSetMetaData metadata = (MariaDbResultSetMetaData)queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - int index = 0; - List rows = new ArrayList(); - while (queryResult.next()) { - DatabaseRow row = new DatabaseRow(); - row.setIndex(index); - List values = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - values.add(queryResult.getString(i)); - } - row.setValues(values); - rows.add(row); - index++; - } - return rows; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - @Override - protected String getDatabaseUrl(DatabaseConfiguration dbConfig) { - int port = dbConfig.getDatabasePort(); - return "jdbc:" + dbConfig.getDatabaseType() + "://" + dbConfig.getDatabaseHost() - + ((port == 0) ? "" : (":" + port)) + "/" + dbConfig.getDatabaseName() + "?useSSL=" + dbConfig.isUseSSL(); - } - - @Override - public Connection getConnection(DatabaseConfiguration dbConfig) - throws DatabaseServiceException { - // TODO Auto-generated method stub - return MariaDBConnectionManager.getInstance().getConnection(dbConfig, true); - } - - @Override - public DatabaseInfo testQuery(DatabaseConfiguration dbConfig, String query) - throws DatabaseServiceException { - Statement statement = null; - ResultSet queryResult = null; - try { - Connection connection = MariaDBConnectionManager.getInstance().getConnection(dbConfig, true); - statement = connection.createStatement(); - queryResult = statement.executeQuery(query); - DatabaseInfo dbInfo = new DatabaseInfo(); - return dbInfo; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } finally { - try { - if (queryResult != null) { - queryResult.close(); - } - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - MariaDBConnectionManager.getInstance().shutdown(); - } - } -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseColumn.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseColumn.java deleted file mode 100644 index 35620146c..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseColumn.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.model; - -import com.google.refine.extension.database.DatabaseColumnType; - -public class DatabaseColumn { - - - private String name; - private int size; - private DatabaseColumnType type; - private String label; - - public DatabaseColumnType getType() { - return type; - } - - - - public void setType(DatabaseColumnType type) { - this.type = type; - } - - - - public String getLabel() { - return label; - } - - - - public void setLabel(String label) { - this.label = label; - } - - - public DatabaseColumn(String name, int size, DatabaseColumnType type) { - super(); - this.name = name; - this.size = size; - this.type = type; - } - - - public DatabaseColumn(String name, String label, DatabaseColumnType type, int size) { - this.name = name; - this.label = label; - this.size = size; - this.type = type; - } - - - public String getName() { - return name; - } - - - public void setName(String name) { - this.name = name; - } - - - public int getSize() { - return size; - } - - - public void setSize(int size) { - this.size = size; - } - - - - - @Override - public String toString() { - return "DatabaseColumn [name=" + name + ", size=" + size + ", type=" + type + ", label=" + label + "]"; - } - - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseInfo.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseInfo.java deleted file mode 100644 index c5755e9a7..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseInfo.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.model; - -import java.util.ArrayList; -import java.util.List; - -public class DatabaseInfo { - - private List tables; - private int dbMajorVersion; - private int dbMinorVersion; - private String dbProductVersion; - private String dbProductName; - - private ArrayList columns; - private List rows; - - public DatabaseInfo() { - // TODO Auto-generated constructor stub - } - - public List getTables() { - return tables; - } - - public void setTables(List tables) { - this.tables = tables; - } - - public void setDatabaseMajorVersion(int dbMajorVersion) { - this.dbMajorVersion = dbMajorVersion; - - } - - public void setDatabaseMinorVersion(int dbMinorVersion) { - this.dbMinorVersion = dbMinorVersion; - - } - - public void setDatabaseProductVersion(String dbProductVersion) { - this.dbProductVersion = dbProductVersion; - - } - - public void setDatabaseProductName(String dbProductName) { - this.dbProductName = dbProductName; - - } - - public int getDbMajorVersion() { - return dbMajorVersion; - } - - public void setDbMajorVersion(int dbMajorVersion) { - this.dbMajorVersion = dbMajorVersion; - } - - public int getDbMinorVersion() { - return dbMinorVersion; - } - - public void setDbMinorVersion(int dbMinorVersion) { - this.dbMinorVersion = dbMinorVersion; - } - - public String getDbProductVersion() { - return dbProductVersion; - } - - public void setDbProductVersion(String dbProductVersion) { - this.dbProductVersion = dbProductVersion; - } - - public String getDbProductName() { - return dbProductName; - } - - public void setDbProductName(String dbProductName) { - this.dbProductName = dbProductName; - } - - public void setColumns(ArrayList columns) { - this.columns = columns; - - } - - public void setRows(List rows) { - this.rows = rows; - - } - - public ArrayList getColumns() { - return columns; - } - - - public List getRows() { - return rows; - } - - - @Override - public String toString() { - return "DatabaseInfo [tables=" + tables + ", dbMajorVersion=" + dbMajorVersion + ", dbMinorVersion=" - + dbMinorVersion + ", dbProductVersion=" + dbProductVersion + ", dbProductName=" + dbProductName + "]"; - } - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseQueryInfo.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseQueryInfo.java deleted file mode 100644 index e8f1fb8c8..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseQueryInfo.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.model; - -import com.google.refine.extension.database.DatabaseConfiguration; - -public class DatabaseQueryInfo { - - private DatabaseConfiguration dbConfig; - - private String query; - - public DatabaseQueryInfo(DatabaseConfiguration databaseConfig, String query) { - super(); - this.dbConfig = databaseConfig; - this.query = query; - } - - - public DatabaseConfiguration getDbConfig() { - return dbConfig; - } - - - public void setDbConfig(DatabaseConfiguration databaseConfig) { - this.dbConfig = databaseConfig; - } - - - public String getQuery() { - return query; - } - - - public void setQuery(String query) { - this.query = query; - } - - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseRow.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseRow.java deleted file mode 100644 index 3fd6b9979..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseRow.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.model; - -import java.util.List; - -public class DatabaseRow { - - private int index; - - private List values; - - - public int getIndex() { - return index; - } - - - public void setIndex(int index) { - this.index = index; - } - - - public List getValues() { - return values; - } - - - public void setValues(List values) { - this.values = values; - } - - - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseTable.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseTable.java deleted file mode 100644 index c7758de27..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/model/DatabaseTable.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.model; - -import java.util.ArrayList; -import java.util.List; - -public class DatabaseTable { - - - private List columns = new ArrayList(); - - private String name; - - - public DatabaseTable(String name) { - this.name = name; - } - - - public List getColumns() { - return columns; - } - - - public void setColumns(List columns) { - this.columns = columns; - } - - - public String getName() { - return name; - } - - - public void setName(String name) { - this.name = name; - } - - @Override - public String toString() { - return "DatabaseTable [columns=" + columns + ", name=" + name + "]"; - } - - - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/mysql/MySQLConnectionManager.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/mysql/MySQLConnectionManager.java deleted file mode 100644 index 9e81fd264..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/mysql/MySQLConnectionManager.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.mysql; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.SQLType; - - -public class MySQLConnectionManager { - - private static final Logger logger = LoggerFactory.getLogger("MySQLConnectionManager"); - private Connection connection; - private SQLType type; - - private static MySQLConnectionManager instance; - - /** - * - * @param type - * @param databaseConfiguration - * @throws SQLException - */ - private MySQLConnectionManager() { - type = SQLType.forName(MySQLDatabaseService.DB_NAME); - - } - - - - /** - * Create a new instance of this connection manager. - * - * @return an instance of the manager - * - * @throws DatabaseServiceException - */ - public static MySQLConnectionManager getInstance() throws DatabaseServiceException { - if (instance == null) { - logger.debug("::Creating new MySQLConnectionManager ::"); - instance = new MySQLConnectionManager(); - - } - return instance; - } - - - /** - * Get the SQL Database type. - * - * @return the type - */ - public SQLType getType() { - return this.type; - } - - /** - * testConnection - * @param databaseConfiguration - * @return - */ - public boolean testConnection(DatabaseConfiguration databaseConfiguration) throws DatabaseServiceException{ - - try { - boolean connResult = false; - - Connection conn = getConnection(databaseConfiguration, true); - if(conn != null) { - connResult = true; - conn.close(); - } - - return connResult; - - } - catch (SQLException e) { - logger.error("Test connection Failed!", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - - } - - /** - * Get a connection form the connection pool. - * - * @return connection from the pool - */ - public Connection getConnection(DatabaseConfiguration databaseConfiguration, boolean forceNewConnection) throws DatabaseServiceException{ - try { - - if (connection != null && !forceNewConnection) { - //logger.info("connection closed::{}", connection.isClosed()); - if (!connection.isClosed()) { - if(logger.isDebugEnabled()){ - logger.debug("Returning existing connection::{}", connection); - } - - return connection; - } - } - String dbURL = getDatabaseUrl(databaseConfiguration); - Class.forName(type.getClassPath()); - - //logger.info("*** type.getClassPath() ::{}, {}**** ", type.getClassPath()); - - DriverManager.setLoginTimeout(10); - - connection = DriverManager.getConnection(dbURL, databaseConfiguration.getDatabaseUser(), - databaseConfiguration.getDatabasePassword()); - - if(logger.isDebugEnabled()) { - logger.debug("*** Acquired New connection for ::{} **** ", dbURL); - } - - - return connection; - - - } catch (ClassNotFoundException e) { - logger.error("Jdbc Driver not found", e); - throw new DatabaseServiceException(e.getMessage()); - } catch (SQLException e) { - logger.error("SQLException::Couldn't get a Connection!", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - - public void shutdown() { - - if (connection != null) { - try { - connection.close(); - } - catch (SQLException e) { - logger.warn("Non-Managed connection could not be closed. Whoops!", e); - } - } - - } - - - private String getDatabaseUrl(DatabaseConfiguration dbConfig) { - - int port = dbConfig.getDatabasePort(); - return "jdbc:" + dbConfig.getDatabaseType() + "://" + dbConfig.getDatabaseHost() - + ((port == 0) ? "" : (":" + port)) + "/" + dbConfig.getDatabaseName() + "?useSSL=" + dbConfig.isUseSSL(); - - } -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/mysql/MySQLDatabaseService.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/mysql/MySQLDatabaseService.java deleted file mode 100644 index 8f4492f0b..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/mysql/MySQLDatabaseService.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.mysql; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.DatabaseUtils; -import com.google.refine.extension.database.SQLType; -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.extension.database.model.DatabaseRow; - -public class MySQLDatabaseService extends DatabaseService { - private static final Logger logger = LoggerFactory.getLogger("MySQLDatabaseService"); - public static final String DB_NAME = "mysql"; - public static final String DB_DRIVER = "com.mysql.jdbc.Driver"; - - private static MySQLDatabaseService instance; - - private MySQLDatabaseService() { - } - - public static MySQLDatabaseService getInstance() { - if (instance == null) { - SQLType.registerSQLDriver(DB_NAME, DB_DRIVER, false); - instance = new MySQLDatabaseService(); - if(logger.isDebugEnabled()) { - logger.debug("MySQLDatabaseService Instance: {}", instance); - } - } - return instance; - } - - @Override - public boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException{ - return MySQLConnectionManager.getInstance().testConnection(dbConfig); - - } - - @Override - public DatabaseInfo connect(DatabaseConfiguration dbConfig) throws DatabaseServiceException{ - return getMetadata(dbConfig); - } - - @Override - public DatabaseInfo executeQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException{ - try { - Connection connection = MySQLConnectionManager.getInstance().getConnection(dbConfig, false); - Statement statement = connection.createStatement(); - ResultSet queryResult = statement.executeQuery(query); - java.sql.ResultSetMetaData metadata = queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - ArrayList columns = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - DatabaseColumn dc = new DatabaseColumn( - metadata.getColumnName(i), - metadata.getColumnLabel(i), - DatabaseUtils.getDbColumnType(metadata.getColumnType(i)), - metadata.getColumnDisplaySize(i)); - columns.add(dc); - } - int index = 0; - List rows = new ArrayList(); - while (queryResult.next()) { - DatabaseRow row = new DatabaseRow(); - row.setIndex(index); - List values = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - - values.add(queryResult.getString(i)); - - } - row.setValues(values); - rows.add(row); - index++; - - } - DatabaseInfo dbInfo = new DatabaseInfo(); - dbInfo.setColumns(columns); - dbInfo.setRows(rows); - return dbInfo; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - }finally { - MySQLConnectionManager.getInstance().shutdown(); - } - } - - /** - * @param connectionInfo - * @return - * @throws DatabaseServiceException - */ - private DatabaseInfo getMetadata(DatabaseConfiguration connectionInfo) throws DatabaseServiceException { - try { - Connection connection = MySQLConnectionManager.getInstance().getConnection(connectionInfo, true); - if(connection != null) { - java.sql.DatabaseMetaData metadata; - metadata = connection.getMetaData(); - int dbMajorVersion = metadata.getDatabaseMajorVersion(); - int dbMinorVersion = metadata.getDatabaseMinorVersion(); - String dbProductVersion = metadata.getDatabaseProductVersion(); - String dbProductName = metadata.getDatabaseProductName(); - DatabaseInfo dbInfo = new DatabaseInfo(); - dbInfo.setDatabaseMajorVersion(dbMajorVersion); - dbInfo.setDatabaseMinorVersion(dbMinorVersion); - dbInfo.setDatabaseProductVersion(dbProductVersion); - dbInfo.setDatabaseProductName(dbProductName); - return dbInfo; - } - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - return null; - } - - @Override - public ArrayList getColumns(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException{ - try { - Connection connection = MySQLConnectionManager.getInstance().getConnection(dbConfig, true); - Statement statement = connection.createStatement(); - ResultSet queryResult = statement.executeQuery(query); - java.sql.ResultSetMetaData metadata = queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - ArrayList columns = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - DatabaseColumn dc = new DatabaseColumn(metadata.getColumnName(i), metadata.getColumnLabel(i), - DatabaseUtils.getDbColumnType(metadata.getColumnType(i)), metadata.getColumnDisplaySize(i)); - columns.add(dc); - } - return columns; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - @Override - public List getRows(DatabaseConfiguration dbConfig, String query) - throws DatabaseServiceException { - try { - Connection connection = MySQLConnectionManager.getInstance().getConnection(dbConfig, false); - Statement statement = connection.createStatement(); - statement.setFetchSize(10); - ResultSet queryResult = statement.executeQuery(query); - java.sql.ResultSetMetaData metadata = queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - int index = 0; - List rows = new ArrayList(); - while (queryResult.next()) { - DatabaseRow row = new DatabaseRow(); - row.setIndex(index); - List values = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - values.add(queryResult.getString(i)); - } - row.setValues(values); - rows.add(row); - index++; - } - return rows; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - @Override - protected String getDatabaseUrl(DatabaseConfiguration dbConfig) { - int port = dbConfig.getDatabasePort(); - return "jdbc:" + dbConfig.getDatabaseType() + "://" + dbConfig.getDatabaseHost() - + ((port == 0) ? "" : (":" + port)) + "/" + dbConfig.getDatabaseName() + "?useSSL=" + dbConfig.isUseSSL(); - } - - @Override - public Connection getConnection(DatabaseConfiguration dbConfig) - throws DatabaseServiceException { - // TODO Auto-generated method stub - return MySQLConnectionManager.getInstance().getConnection(dbConfig, true); - } - - @Override - public DatabaseInfo testQuery(DatabaseConfiguration dbConfig, String query) - throws DatabaseServiceException { - Statement statement = null; - ResultSet queryResult = null; - try { - Connection connection = MySQLConnectionManager.getInstance().getConnection(dbConfig, true); - statement = connection.createStatement(); - queryResult = statement.executeQuery(query); - DatabaseInfo dbInfo = new DatabaseInfo(); - return dbInfo; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } finally { - try { - if (queryResult != null) { - queryResult.close(); - } - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - MySQLConnectionManager.getInstance().shutdown(); - } - } -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/pgsql/PgSQLConnectionManager.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/pgsql/PgSQLConnectionManager.java deleted file mode 100644 index bef6c9a69..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/pgsql/PgSQLConnectionManager.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.pgsql; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.SQLType; - - - -public class PgSQLConnectionManager { - - private static final Logger logger = LoggerFactory.getLogger("PgSQLConnectionManager"); - private Connection connection; - private SQLType type; - - private static PgSQLConnectionManager instance; - - - - /** - * - * @param type - * @param databaseConfiguration - * @throws SQLException - */ - private PgSQLConnectionManager() { - type = SQLType.forName(PgSQLDatabaseService.DB_NAME); - - } - - - - - /** - * Create a new instance of this connection manager. - * - * @return an instance of the manager - * - * @throws DatabaseServiceException - */ - public static PgSQLConnectionManager getInstance() throws DatabaseServiceException { - if (instance == null) { - if(logger.isDebugEnabled()) { - logger.debug("::Creating new PgSQL ConnectionManager ::"); - } - - instance = new PgSQLConnectionManager(); - - } - return instance; - } - - - /** - * Get the SQL Database type. - * - * @return the type - */ - public SQLType getType() { - return this.type; - } - - /** - * testConnection - * @param databaseConfiguration - * @return - */ - public boolean testConnection(DatabaseConfiguration databaseConfiguration) throws DatabaseServiceException{ - - try { - boolean connResult = false; - - Connection conn = getConnection(databaseConfiguration, true); - if(conn != null) { - connResult = true; - conn.close(); - } - - return connResult; - - } - catch (SQLException e) { - logger.error("Test connection Failed!", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - - } - - /** - * Get a connection form the connection pool. - * - * @return connection from the pool - */ - public Connection getConnection(DatabaseConfiguration databaseConfiguration, boolean forceNewConnection) throws DatabaseServiceException{ - try { - - // logger.info("connection::{}, forceNewConnection: {}", connection, forceNewConnection); - - if (connection != null && !forceNewConnection) { - // logger.info("connection closed::{}", connection.isClosed()); - if (!connection.isClosed()) { - if(logger.isDebugEnabled()){ - logger.debug("Returning existing connection::{}", connection); - } - return connection; - } - } - - Class.forName(type.getClassPath()); - DriverManager.setLoginTimeout(10); - String dbURL = getDatabaseUrl(databaseConfiguration); - connection = DriverManager.getConnection(dbURL, databaseConfiguration.getDatabaseUser(), - databaseConfiguration.getDatabasePassword()); - - logger.debug("*** Acquired New connection for ::{} **** ", dbURL); - - return connection; - - - } catch (ClassNotFoundException e) { - logger.error("Jdbc Driver not found", e); - throw new DatabaseServiceException(e.getMessage()); - } catch (SQLException e) { - logger.error("SQLException::Couldn't get a Connection!", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - - public void shutdown() { - - if (connection != null) { - try { - connection.close(); - } - catch (SQLException e) { - logger.warn("Non-Managed connection could not be closed. Whoops!", e); - } - } - - } - - - private static String getDatabaseUrl(DatabaseConfiguration dbConfig) { - - int port = dbConfig.getDatabasePort(); - return "jdbc:" + dbConfig.getDatabaseType().toLowerCase() + "://" + dbConfig.getDatabaseHost() - + ((port == 0) ? "" : (":" + port)) + "/" + dbConfig.getDatabaseName(); - - } -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/pgsql/PgSQLDatabaseService.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/pgsql/PgSQLDatabaseService.java deleted file mode 100644 index 10051ca16..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/pgsql/PgSQLDatabaseService.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright (c) 2017, Tony Opara - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.database.pgsql; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; - -import org.postgresql.jdbc.PgResultSetMetaData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.DatabaseUtils; -import com.google.refine.extension.database.SQLType; -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.extension.database.model.DatabaseRow; - -public class PgSQLDatabaseService extends DatabaseService { - private static final Logger logger = LoggerFactory.getLogger("PgSQLDatabaseService"); - public static final String DB_NAME = "postgresql"; - public static final String DB_DRIVER = "org.postgresql.Driver"; - private static PgSQLDatabaseService instance; - - private PgSQLDatabaseService() { - } - - public static PgSQLDatabaseService getInstance() { - if (instance == null) { - SQLType.registerSQLDriver(DB_NAME, DB_DRIVER); - instance = new PgSQLDatabaseService(); - if(logger.isDebugEnabled()) { - logger.debug("PgSQLDatabaseService Instance: {}", instance); - } - } - return instance; - } - - @Override - public boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException{ - return PgSQLConnectionManager.getInstance().testConnection(dbConfig); - - } - - @Override - public DatabaseInfo connect(DatabaseConfiguration dbConfig) throws DatabaseServiceException{ - return getMetadata(dbConfig); - } - - @Override - public DatabaseInfo executeQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException{ - try { - Connection connection = PgSQLConnectionManager.getInstance().getConnection(dbConfig, false); - Statement statement = connection.createStatement(); - ResultSet queryResult = statement.executeQuery(query); - PgResultSetMetaData metadata = (PgResultSetMetaData)queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - ArrayList columns = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - DatabaseColumn dc = new DatabaseColumn( - metadata.getColumnName(i), - metadata.getColumnLabel(i), - DatabaseUtils.getDbColumnType(metadata.getColumnType(i)), - metadata.getColumnDisplaySize(i)); - columns.add(dc); - } - int index = 0; - List rows = new ArrayList(); - while (queryResult.next()) { - DatabaseRow row = new DatabaseRow(); - row.setIndex(index); - List values = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - values.add(queryResult.getString(i)); - } - row.setValues(values); - rows.add(row); - index++; - } - DatabaseInfo dbInfo = new DatabaseInfo(); - dbInfo.setColumns(columns); - dbInfo.setRows(rows); - return dbInfo; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - }finally { - PgSQLConnectionManager.getInstance().shutdown(); - } - } - - /** - * @param connectionInfo - * @return - * @throws DatabaseServiceException - */ - private DatabaseInfo getMetadata(DatabaseConfiguration connectionInfo) throws DatabaseServiceException { - try { - Connection connection = PgSQLConnectionManager.getInstance().getConnection(connectionInfo, true); - if(connection != null) { - java.sql.DatabaseMetaData metadata = connection.getMetaData(); - int dbMajorVersion = metadata.getDatabaseMajorVersion(); - int dbMinorVersion = metadata.getDatabaseMinorVersion(); - String dbProductVersion = metadata.getDatabaseProductVersion(); - String dbProductName = metadata.getDatabaseProductName(); - DatabaseInfo dbInfo = new DatabaseInfo(); - dbInfo.setDatabaseMajorVersion(dbMajorVersion); - dbInfo.setDatabaseMinorVersion(dbMinorVersion); - dbInfo.setDatabaseProductVersion(dbProductVersion); - dbInfo.setDatabaseProductName(dbProductName); - return dbInfo; - } - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - return null; - } - - @Override - public ArrayList getColumns(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException{ - try { - Connection connection = PgSQLConnectionManager.getInstance().getConnection(dbConfig, true); - Statement statement = connection.createStatement(); - ResultSet queryResult = statement.executeQuery(query); - PgResultSetMetaData metadata = (PgResultSetMetaData) queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - ArrayList columns = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - DatabaseColumn dc = new DatabaseColumn(metadata.getColumnName(i), metadata.getColumnLabel(i), - DatabaseUtils.getDbColumnType(metadata.getColumnType(i)), metadata.getColumnDisplaySize(i)); - columns.add(dc); - } - return columns; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - @Override - public List getRows(DatabaseConfiguration dbConfig, String query) - throws DatabaseServiceException { - try { - Connection connection = PgSQLConnectionManager.getInstance().getConnection(dbConfig, false); - Statement statement = connection.createStatement(); - statement.setFetchSize(10); - ResultSet queryResult = statement.executeQuery(query); - PgResultSetMetaData metadata = (PgResultSetMetaData)queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - int index = 0; - List rows = new ArrayList(); - while (queryResult.next()) { - DatabaseRow row = new DatabaseRow(); - row.setIndex(index); - List values = new ArrayList(columnCount); - for (int i = 1; i <= columnCount; i++) { - values.add(queryResult.getString(i)); - } - row.setValues(values); - rows.add(row); - index++; - } - return rows; - } catch (SQLException e) { - logger.error("SQLException::{}::{}", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - @Override - protected String getDatabaseUrl(DatabaseConfiguration dbConfig) { - int port = dbConfig.getDatabasePort(); - return "jdbc:" + dbConfig.getDatabaseType() + "://" + dbConfig.getDatabaseHost() - + ((port == 0) ? "" : (":" + port)) + "/" + dbConfig.getDatabaseName() + "?useSSL=" + dbConfig.isUseSSL(); - } - - @Override - public Connection getConnection(DatabaseConfiguration dbConfig) - throws DatabaseServiceException { - return PgSQLConnectionManager.getInstance().getConnection(dbConfig, true); - } - - @Override - public DatabaseInfo testQuery(DatabaseConfiguration dbConfig, String query) - throws DatabaseServiceException { - Statement statement = null; - ResultSet queryResult = null; - try { - Connection connection = PgSQLConnectionManager.getInstance().getConnection(dbConfig, true); - statement = connection.createStatement(); - queryResult = statement.executeQuery(query); - DatabaseInfo dbInfo = new DatabaseInfo(); - return dbInfo; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } finally { - try { - if (queryResult != null) { - queryResult.close(); - } - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - PgSQLConnectionManager.getInstance().shutdown(); - } - } - -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManager.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManager.java deleted file mode 100644 index 5b9b4cfcd..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManager.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2020, Chris Parker - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package com.google.refine.extension.database.sqlite; - -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.SQLType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -public class SQLiteConnectionManager { - - private static final Logger logger = LoggerFactory.getLogger("SQLiteConnectionManager"); - private static SQLiteConnectionManager instance; - private final SQLType type; - private Connection connection; - - private SQLiteConnectionManager() { - type = SQLType.forName(SQLiteDatabaseService.DB_NAME); - } - - /** - * Create a new instance of this connection manager. - * - * @return an instance of the manager - */ - public static SQLiteConnectionManager getInstance() { - if (instance == null) { - if (logger.isDebugEnabled()) { - logger.debug("::Creating new SQLite ConnectionManager ::"); - } - instance = new SQLiteConnectionManager(); - } - return instance; - } - - public static String getDatabaseUrl(DatabaseConfiguration dbConfig) { - return "jdbc:" + dbConfig.getDatabaseType().toLowerCase() + ":" + dbConfig.getDatabaseName(); - } - - /** - * Get the SQL Database type. - * - * @return the type - */ - public SQLType getType() { - return this.type; - } - - /** - * testConnection - * - * @param dbConfig - * @return boolean - */ - public boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException { - try { - boolean connResult = false; - - Connection conn = getConnection(dbConfig); - if (conn != null) { - connResult = true; - conn.close(); - } - - return connResult; - } catch (SQLException e) { - logger.error("Test connection Failed!", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - /** - * Get a connection form the connection pool. - * - * @return connection from the pool - */ - public Connection getConnection(DatabaseConfiguration databaseConfiguration) throws DatabaseServiceException { - try { - if (connection != null) { - connection.close(); - } - - Class.forName(type.getClassPath()); - String dbURL = getDatabaseUrl(databaseConfiguration); - connection = DriverManager.getConnection(dbURL); - - logger.debug("*** Acquired New connection for ::{} **** ", dbURL); - - return connection; - } catch (ClassNotFoundException e) { - logger.error("Jdbc Driver not found", e); - throw new DatabaseServiceException(e.getMessage()); - } catch (SQLException e) { - logger.error("SQLException::Couldn't get a Connection!", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - public void shutdown() { - if (connection != null) { - try { - connection.close(); - } catch (SQLException e) { - logger.warn("Non-Managed connection could not be closed. Whoops!", e); - } - } - } -} diff --git a/OpenRefine/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseService.java b/OpenRefine/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseService.java deleted file mode 100644 index 5971a5e3f..000000000 --- a/OpenRefine/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseService.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright (c) 2020, Chris Parker - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package com.google.refine.extension.database.sqlite; - -import com.google.refine.extension.database.SQLType; -import com.google.refine.extension.database.*; -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.extension.database.model.DatabaseRow; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.*; -import java.util.ArrayList; -import java.util.List; - -public class SQLiteDatabaseService extends DatabaseService { - - public static final String DB_NAME = "sqlite"; - public static final String DB_DRIVER = "org.sqlite.JDBC"; - private static final Logger logger = LoggerFactory.getLogger("SQLiteDatabaseService"); - private static SQLiteDatabaseService instance; - - public static SQLiteDatabaseService getInstance() { - if (instance == null) { - SQLType.registerSQLDriver(DB_NAME, DB_DRIVER, false); - instance = new SQLiteDatabaseService(); - if (logger.isDebugEnabled()) { - logger.debug("SQLiteDatabaseService Instance: {}", instance); - } - } - return instance; - } - - @Override - protected String getDatabaseUrl(DatabaseConfiguration dbConfig) { - return SQLiteConnectionManager.getDatabaseUrl(dbConfig); - } - - @Override - public Connection getConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException { - return SQLiteConnectionManager.getInstance().getConnection(dbConfig); - } - - @Override - public boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException { - return SQLiteConnectionManager.getInstance().testConnection(dbConfig); - } - - @Override - public DatabaseInfo connect(DatabaseConfiguration dbConfig) throws DatabaseServiceException { - return getMetadata(dbConfig); - } - - /** - * @param dbConfig - * @return - * @throws DatabaseServiceException - */ - private DatabaseInfo getMetadata(DatabaseConfiguration connectionInfo) throws DatabaseServiceException { - try { - Connection connection = SQLiteConnectionManager.getInstance().getConnection(connectionInfo); - if (connection != null) { - java.sql.DatabaseMetaData metadata = connection.getMetaData(); - int dbMajorVersion = metadata.getDatabaseMajorVersion(); - int dbMinorVersion = metadata.getDatabaseMinorVersion(); - String dbProductVersion = metadata.getDatabaseProductVersion(); - String dbProductName = metadata.getDatabaseProductName(); - DatabaseInfo dbInfo = new DatabaseInfo(); - dbInfo.setDatabaseMajorVersion(dbMajorVersion); - dbInfo.setDatabaseMinorVersion(dbMinorVersion); - dbInfo.setDatabaseProductVersion(dbProductVersion); - dbInfo.setDatabaseProductName(dbProductName); - return dbInfo; - } - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - return null; - } - - @Override - public DatabaseInfo executeQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException { - try { - Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig); - Statement statement = connection.createStatement(); - ResultSet queryResult = statement.executeQuery(query); - ResultSetMetaData metadata = queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - ArrayList columns = new ArrayList<>(columnCount); - for (int i = 1; i <= columnCount; i++) { - DatabaseColumn dc = new DatabaseColumn(metadata.getColumnName(i), metadata.getColumnLabel(i), - DatabaseUtils.getDbColumnType(metadata.getColumnType(i)), - metadata.getColumnDisplaySize(i)); - columns.add(dc); - } - int index = 0; - List rows = new ArrayList<>(); - while (queryResult.next()) { - DatabaseRow row = new DatabaseRow(); - row.setIndex(index); - List values = new ArrayList<>(columnCount); - for (int i = 1; i <= columnCount; i++) { - values.add(queryResult.getString(i)); - } - row.setValues(values); - rows.add(row); - index++; - } - DatabaseInfo dbInfo = new DatabaseInfo(); - dbInfo.setColumns(columns); - dbInfo.setRows(rows); - return dbInfo; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } finally { - SQLiteConnectionManager.getInstance().shutdown(); - } - } - - @Override - public DatabaseInfo testQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException { - Statement statement = null; - ResultSet queryResult = null; - try { - Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig); - statement = connection.createStatement(); - queryResult = statement.executeQuery(query); - return new DatabaseInfo(); - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } finally { - try { - if (queryResult != null) { - queryResult.close(); - } - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - SQLiteConnectionManager.getInstance().shutdown(); - } - } - - @Override - public List getColumns(DatabaseConfiguration dbConfig, String query) - throws DatabaseServiceException { - try { - Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig); - Statement statement = connection.createStatement(); - ResultSet queryResult = statement.executeQuery(query); - ResultSetMetaData metadata = queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - ArrayList columns = new ArrayList<>(columnCount); - for (int i = 1; i <= columnCount; i++) { - DatabaseColumn dc = new DatabaseColumn(metadata.getColumnName(i), metadata.getColumnLabel(i), - DatabaseUtils.getDbColumnType(metadata.getColumnType(i)), - metadata.getColumnDisplaySize(i)); - columns.add(dc); - } - return columns; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } - - @Override - public List getRows(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException { - try { - Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig); - Statement statement = connection.createStatement(); - statement.setFetchSize(10); - ResultSet queryResult = statement.executeQuery(query); - ResultSetMetaData metadata = queryResult.getMetaData(); - int columnCount = metadata.getColumnCount(); - int index = 0; - List rows = new ArrayList<>(); - while (queryResult.next()) { - DatabaseRow row = new DatabaseRow(); - row.setIndex(index); - List values = new ArrayList<>(columnCount); - for (int i = 1; i <= columnCount; i++) { - values.add(queryResult.getString(i)); - } - row.setValues(values); - rows.add(row); - index++; - } - return rows; - } catch (SQLException e) { - logger.error("SQLException::", e); - throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); - } - } -} diff --git a/OpenRefine/extensions/database/tests/conf/appveyor_tests.xml b/OpenRefine/extensions/database/tests/conf/appveyor_tests.xml deleted file mode 100644 index ecb767fca..000000000 --- a/OpenRefine/extensions/database/tests/conf/appveyor_tests.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/OpenRefine/extensions/database/tests/conf/github_actions_tests.xml b/OpenRefine/extensions/database/tests/conf/github_actions_tests.xml deleted file mode 100644 index 3e508e567..000000000 --- a/OpenRefine/extensions/database/tests/conf/github_actions_tests.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/OpenRefine/extensions/database/tests/conf/test-mariadb.sql b/OpenRefine/extensions/database/tests/conf/test-mariadb.sql deleted file mode 100644 index c5716c5a8..000000000 --- a/OpenRefine/extensions/database/tests/conf/test-mariadb.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `test_db` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; -USE test_db; - -CREATE TABLE IF NOT EXISTS `test_table` ( - `id` int(11) NOT NULL, - `ue_id` char(8) NOT NULL, - `start_time` timestamp NOT NULL, - `end_date` date DEFAULT NULL, - `bytes_upload` int(11) NOT NULL, - `bytes_download` int(11) NOT NULL, - `mcc` char(3) DEFAULT NULL, - `mnc` char(3) NOT NULL, - `lac` varchar(11) DEFAULT NULL, - `imei` char(16) NOT NULL -); - -INSERT INTO test_table(id, ue_id, start_time, end_date, bytes_upload, bytes_download, mcc, mnc, lac, imei) - VALUES (1, '11100022', now(), now(), 1024, 2048, 321, 543, 12209823498, 1344498988877487); \ No newline at end of file diff --git a/OpenRefine/extensions/database/tests/conf/test-mysql.sql b/OpenRefine/extensions/database/tests/conf/test-mysql.sql deleted file mode 100644 index d2b53c80e..000000000 --- a/OpenRefine/extensions/database/tests/conf/test-mysql.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE test_db; - -CREATE TABLE IF NOT EXISTS `test_table` ( - `id` int(11) NOT NULL, - `ue_id` char(8) NOT NULL, - `start_time` timestamp NOT NULL, - `end_date` date DEFAULT NULL, - `bytes_upload` int(11) NOT NULL, - `bytes_download` int(11) NOT NULL, - `mcc` char(3) DEFAULT NULL, - `mnc` char(3) NOT NULL, - `lac` varchar(11) DEFAULT NULL, - `imei` char(16) NOT NULL -); - -INSERT INTO test_table(id, ue_id, start_time, end_date, bytes_upload, bytes_download, mcc, mnc, lac, imei) - VALUES (1, '11100022', now(), now(), 1024, 2048, 321, 543, 12209823498, 1344498988877487); \ No newline at end of file diff --git a/OpenRefine/extensions/database/tests/conf/test-pgsql.sql b/OpenRefine/extensions/database/tests/conf/test-pgsql.sql deleted file mode 100644 index f8299a964..000000000 --- a/OpenRefine/extensions/database/tests/conf/test-pgsql.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE test_db; - -CREATE TABLE IF NOT EXISTS test_table ( - id integer NOT NULL, - ue_id char(8) NOT NULL, - start_time timestamp NOT NULL, - end_date date DEFAULT NULL, - bytes_upload integer NOT NULL, - bytes_download integer NOT NULL, - mcc char(3) DEFAULT NULL, - mnc char(3) NOT NULL, - lac varchar(11) DEFAULT NULL, - imei char(16) NOT NULL -); - -INSERT INTO test_table(id, ue_id, start_time, end_date, bytes_upload, bytes_download, mcc, mnc, lac, imei) - VALUES (1, '11100022', now(), now(), 1024, 2048, 321, 543, 12209823498, 1344498988877487); \ No newline at end of file diff --git a/OpenRefine/extensions/database/tests/conf/test-sqlite.sql b/OpenRefine/extensions/database/tests/conf/test-sqlite.sql deleted file mode 100644 index ae48f3db1..000000000 --- a/OpenRefine/extensions/database/tests/conf/test-sqlite.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE TABLE IF NOT EXISTS test_table ( - id integer NOT NULL, - ue_id char(8) NOT NULL, - start_time timestamp NOT NULL, - end_date date DEFAULT NULL, - bytes_upload integer NOT NULL, - bytes_download integer NOT NULL, - mcc char(3) DEFAULT NULL, - mnc char(3) NOT NULL, - lac varchar(11) DEFAULT NULL, - imei char(16) NOT NULL -); - -INSERT INTO test_table(id, ue_id, start_time, end_date, bytes_upload, bytes_download, mcc, mnc, lac, imei) - VALUES (1, '11100022', now(), now(), 1024, 2048, 321, 543, 12209823498, 1344498988877487); diff --git a/OpenRefine/extensions/database/tests/conf/tests.xml b/OpenRefine/extensions/database/tests/conf/tests.xml deleted file mode 100644 index 7885174bf..000000000 --- a/OpenRefine/extensions/database/tests/conf/tests.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/OpenRefine/extensions/database/tests/log4j-test.properties b/OpenRefine/extensions/database/tests/log4j-test.properties deleted file mode 100644 index 0ed0d9f64..000000000 --- a/OpenRefine/extensions/database/tests/log4j-test.properties +++ /dev/null @@ -1,5 +0,0 @@ -log4j.rootLogger=ERROR, console -log4j.logger.com.google.refine.extension.database=DEBUG - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.layout=com.google.refine.logging.IndentingLayout diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTestUtils.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTestUtils.java deleted file mode 100644 index d91ed4f08..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTestUtils.java +++ /dev/null @@ -1,439 +0,0 @@ - -package com.google.refine.extension.database; - -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.Date; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -import org.apache.commons.text.StringSubstitutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DBExtensionTestUtils { - - private static final Logger logger = LoggerFactory.getLogger("DBExtensionTestUtils"); - - private static final String MYSQL_DB_NAME = "mysql"; - private static final String DEFAULT_MYSQL_HOST = "127.0.0.1"; - private static final int DEFAULT_MYSQL_PORT = 3306; - private static final String DEFAULT_MYSQL_USER = "root"; - private static final String DEFAULT_MYSQL_PASSWORD = "secret"; - private static final String DEFAULT_MYSQL_DB_NAME = "testdb"; - - private static final String PGSQL_DB_NAME = "postgresql"; - private static final String DEFAULT_PGSQL_HOST = "127.0.0.1"; - private static final int DEFAULT_PGSQL_PORT = 5432; - private static final String DEFAULT_PGSQL_USER = "postgres"; - private static final String DEFAULT_PGSQL_PASSWORD = ""; - private static final String DEFAULT_PGSQL_DB_NAME = "openrefine"; - - private static final String DEFAULT_TEST_TABLE = "test_data"; - - private static final int SAMPLE_SIZE = 500000; - private static final int BATCH_SIZE = 1000; - - private static Random rand = new Random(); - - private Map mncMap; - private Map mccMap; - - /** - * Create Test Table with one row of Data - * - * @param dbConfig - * DatabaseConfiguration to test - * @param tableName - * @throws DatabaseServiceException - * @throws SQLException - */ - public static void initTestData(DatabaseConfiguration dbConfig, String tableName) - throws DatabaseServiceException, SQLException { - - Statement stmt = null; - Connection conn = null; - try { - DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType()); - conn = dbService.getConnection(dbConfig); - stmt = conn.createStatement(); - - DatabaseMetaData dbm = conn.getMetaData(); - // check if "employee" table is there - ResultSet tables = dbm.getTables(null, null, tableName, null); - boolean dropTable = false; - if (tables.next()) { - dropTable = true; - // System.out.println("Drop Table Result::" + dropResult); - } - tables.close(); - if (dropTable) { - stmt.executeUpdate("DROP TABLE " + tableName); - } - - String createSQL = " CREATE TABLE " + tableName + " ( " - + " ID INT NOT NULL, " - + " NAME VARCHAR (20) NOT NULL, " - + " CITY VARCHAR (20) NOT NULL," - + " PRIMARY KEY (ID) );"; - - stmt.executeUpdate(createSQL); - // System.out.println("Create Table Result::" + createResult); - - String insertTableSQL = "INSERT INTO " + tableName - + "(ID, NAME, CITY) " + "VALUES" - + "(1,'frank lens','Dallas')"; - - stmt.executeUpdate(insertTableSQL); - // System.out.println("Insert Data Result::" + insertResult); - - logger.info("Database Test Init Data Created!!!"); - - } finally { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - } - - } - - public static void initTestData(DatabaseConfiguration dbConfig) - throws DatabaseServiceException, SQLException { - - Statement stmt = null; - Connection conn = null; - try { - DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType()); - conn = dbService.getConnection(dbConfig); - stmt = conn.createStatement(); - - DatabaseMetaData dbm = conn.getMetaData(); - // check if "employee" table is there - ResultSet tables = dbm.getTables(null, null, DEFAULT_TEST_TABLE, null); - boolean dropTable = false; - if (tables.next()) { - dropTable = true; - // System.out.println("Drop Table Result::" + dropResult); - } - tables.close(); - if (dropTable) { - stmt.executeUpdate("DROP TABLE " + DEFAULT_TEST_TABLE); - // System.out.println("Drop Table Result::" + dropResult); - } - - String createSQL = " CREATE TABLE TEST_DATA( " - + " ID INT NOT NULL, " - + " NAME VARCHAR (20) NOT NULL, " - + " CITY VARCHAR (20) NOT NULL," - + " PRIMARY KEY (ID) );"; - - stmt.executeUpdate(createSQL); - // System.out.println("Create Table Result::" + createResult); - - String insertTableSQL = "INSERT INTO TEST_DATA" - + "(ID, NAME, CITY) " + "VALUES" - + "(1,'frank lens','Dallas')"; - - stmt.executeUpdate(insertTableSQL); - // System.out.println("Insert Data Result::" + insertResult); - - logger.info("Database Test Init Data Created!!!"); - - } finally { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } - - } - - /** - * CREATE test data in MySQL Table name: test_data - * - * @param sampleSize - * @param batchSize - * @throws DatabaseServiceException - * @throws SQLException - */ - public void generateMySQLTestData(int sampleSize, int batchSize) - throws DatabaseServiceException, SQLException { - mncMap = new HashMap(); - mccMap = new HashMap(); - mccMap.put(0, 302); - mccMap.put(1, 311); - mccMap.put(2, 730); - mccMap.put(1, 622); - - mncMap.put(0, 006); - mncMap.put(1, 140); - mncMap.put(2, 380); - mncMap.put(3, 710); - - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost(DEFAULT_MYSQL_HOST); - dc.setDatabaseName(DEFAULT_MYSQL_DB_NAME); - dc.setDatabasePassword(DEFAULT_MYSQL_PASSWORD); - dc.setDatabasePort(DEFAULT_MYSQL_PORT); - dc.setDatabaseType(MYSQL_DB_NAME); - dc.setDatabaseUser(DEFAULT_MYSQL_USER); - dc.setUseSSL(false); - - String truncateTableSQL = "TRUNCATE test_data"; - - String insertTableSQL = "INSERT INTO test_data(" - + "id, ue_id, start_time, end_date, bytes_upload, bytes_download, cell_id, mcc, mnc, lac, imei)" - + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; - - Connection conn = DatabaseService.get(MYSQL_DB_NAME).getConnection(dc); - - Statement truncateStmt = conn.createStatement(); - int result = truncateStmt.executeUpdate(truncateTableSQL); - System.out.println("Truncate Table Result::" + result); - truncateStmt.close(); - - conn.setAutoCommit(false); - - PreparedStatement stmt = conn.prepareStatement(insertTableSQL); - - int counter = 1; - for (int i = 0; i < sampleSize; i++) { - stmt.setLong(1, i); - stmt.setString(2, getNextUeId()); - stmt.setDate(3, getNextStartDate()); - stmt.setDate(4, getNextEndDate()); - stmt.setInt(5, rand.nextInt()); - stmt.setInt(6, rand.nextInt()); - stmt.setInt(7, rand.nextInt(10)); - stmt.setInt(8, getMCC()); - stmt.setInt(9, getMNC()); - stmt.setInt(10, rand.nextInt(100)); - stmt.setString(11, getNextIMEI()); - - stmt.addBatch(); - - // Execute batch of 1000 records - if (i % batchSize == 0) { - stmt.executeBatch(); - conn.commit(); - System.out.println("Batch " + (counter++) + " executed successfully"); - } - } - // execute final batch - stmt.executeBatch(); - System.out.println("Final Batch Executed " + (counter++) + " executed successfully"); - conn.commit(); - conn.close(); - } - - /** - * - * @param sampleSize - * @param batchSize - * @throws DatabaseServiceException - * @throws SQLException - */ - public void generatePgSQLTestData(int sampleSize, int batchSize) throws DatabaseServiceException, SQLException { - mncMap = new HashMap(); - mccMap = new HashMap(); - mccMap.put(0, 302); - mccMap.put(1, 311); - mccMap.put(2, 730); - mccMap.put(1, 622); - - mncMap.put(0, 006); - mncMap.put(1, 140); - mncMap.put(2, 380); - mncMap.put(3, 710); - - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost(DEFAULT_PGSQL_HOST); - dc.setDatabaseName(DEFAULT_PGSQL_DB_NAME); - dc.setDatabasePassword(DEFAULT_PGSQL_PASSWORD); - dc.setDatabasePort(DEFAULT_PGSQL_PORT); - dc.setDatabaseType(PGSQL_DB_NAME); - dc.setDatabaseUser(DEFAULT_PGSQL_USER); - dc.setUseSSL(false); - - String truncateTableSQL = "TRUNCATE public.test_data"; - - String insertTableSQL = "INSERT INTO public.test_data(" - + "id, ue_id, start_time, end_date, bytes_upload, bytes_download, cell_id, mcc, mnc, lac, imei)" - + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; - - Connection conn = DatabaseService.get(PGSQL_DB_NAME).getConnection(dc); - - Statement truncateStmt = conn.createStatement(); - int result = truncateStmt.executeUpdate(truncateTableSQL); - System.out.println("Truncate Table Result::" + result); - truncateStmt.close(); - - conn.setAutoCommit(false); - - PreparedStatement stmt = conn.prepareStatement(insertTableSQL); - - int counter = 1; - for (int i = 0; i < sampleSize; i++) { - stmt.setLong(1, i); - stmt.setString(2, getNextUeId()); - stmt.setDate(3, getNextStartDate()); - stmt.setDate(4, getNextEndDate()); - stmt.setInt(5, rand.nextInt()); - stmt.setInt(6, rand.nextInt()); - stmt.setInt(7, rand.nextInt(10)); - stmt.setInt(8, getMCC()); - stmt.setInt(9, getMNC()); - stmt.setInt(10, rand.nextInt(100)); - stmt.setString(11, getNextIMEI()); - - stmt.addBatch(); - - // Execute batch of 1000 records - if (i % batchSize == 0) { - stmt.executeBatch(); - conn.commit(); - System.out.println("Batch " + (counter++) + " executed successfully"); - } - } - // execute final batch - stmt.executeBatch(); - System.out.println("Final Batch Executed " + (counter++) + " executed successfully"); - conn.commit(); - conn.close(); - - } - - private String getNextIMEI() { - int n = 1000000000 + rand.nextInt(900000000); - return "" + n; - } - - private int getMNC() { - - return mncMap.get(rand.nextInt(3)); - } - - private int getMCC() { - - return mccMap.get(rand.nextInt(3)); - } - - private Date getNextEndDate() { - - return new Date(System.currentTimeMillis() + 1); - } - - private Date getNextStartDate() { - - return new Date(System.currentTimeMillis()); - } - - private String getNextUeId() { - - int n = 300000000 + rand.nextInt(900000000); - - return "" + n; - } - - public static void main(String[] args) throws DatabaseServiceException, SQLException { - DBExtensionTestUtils testUtil = new DBExtensionTestUtils(); - testUtil.generatePgSQLTestData(SAMPLE_SIZE, BATCH_SIZE); - // testUtil.generateMySQLTestData(); - } - - public static void cleanUpTestData(DatabaseConfiguration dbConfig) { - Statement stmt = null; - Connection conn = null; - try { - DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType()); - conn = dbService.getConnection(dbConfig); - stmt = conn.createStatement(); - - DatabaseMetaData dbm = conn.getMetaData(); - // check if "employee" table is there - ResultSet tables = dbm.getTables(null, null, DEFAULT_TEST_TABLE, null); - if (tables.next()) { - stmt.executeUpdate("DROP TABLE " + DEFAULT_TEST_TABLE); - // System.out.println("Drop Table Result::" + dropResult); - } - - logger.info("Database Test Cleanup Done"); - - } catch (DatabaseServiceException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (SQLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } finally { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } - - } - - public static File createTempDirectory(String name) - throws IOException { - File dir = File.createTempFile(name, ""); - dir.delete(); - dir.mkdir(); - return dir; - } - - public static String getJDBCUrl(DatabaseConfiguration dbConfig) { - Map substitutes = new HashMap(); - substitutes.put("dbType", dbConfig.getDatabaseType()); - substitutes.put("host", dbConfig.getDatabaseHost()); - substitutes.put("port", "" + dbConfig.getDatabasePort()); - substitutes.put("dbName", dbConfig.getDatabaseName()); - substitutes.put("useSSL", dbConfig.isUseSSL()); - String urlTemplate = "jdbc:${dbType}://${host}:${port}/${dbName}?useSSL=${useSSL}"; - StringSubstitutor strSub = new StringSubstitutor(substitutes); - return strSub.replace(urlTemplate); - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTests.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTests.java deleted file mode 100644 index bd1d0bc11..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTests.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - -Copyright 2010,2011 Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -package com.google.refine.extension.database; - -import java.util.Properties; - -import org.slf4j.Logger; - -public class DBExtensionTests { - - protected final String MYSQL_DB_NAME = "mysql"; - protected final String DEFAULT_MYSQL_HOST = "127.0.0.1"; - protected final String DEFAULT_MYSQL_PORT = "3306"; - protected final String DEFAULT_MYSQL_USER = "root"; - protected final String DEFAULT_MYSQL_PASSWORD = "secret"; - protected final String DEFAULT_MYSQL_DB_NAME = "testdb"; - - protected final String PGSQL_DB_NAME = "postgresql"; - protected final String DEFAULT_PGSQL_HOST = "127.0.0.1"; - protected final String DEFAULT_PGSQL_PORT = "5432"; - protected final String DEFAULT_PGSQL_USER = "postgres"; - protected final String DEFAULT_PGSQL_PASSWORD = ""; - protected final String DEFAULT_PGSQL_DB_NAME = "testdb"; - - protected final String MARIA_DB_NAME = "mariadb"; - protected final String DEFAULT_MARIADB_HOST = "127.0.0.1"; - protected final String DEFAULT_MARIADB_PORT = "3306"; - protected final String DEFAULT_MARIADB_USER = "root"; - protected final String DEFAULT_MARIADB_PASSWORD = "secret"; - protected final String DEFAULT_MARIADB_NAME = "testdb"; - - protected final String SQLITE_DB_NAME = "sqlite"; - protected final String DEFAULT_SQLITE_DB_NAME = "extension_test_db.sqlite"; - - protected final String DEFAULT_TEST_TABLE = "test_data"; - - protected Properties properties; - - protected Logger logger; - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DatabaseImportControllerTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DatabaseImportControllerTest.java deleted file mode 100644 index 561994872..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DatabaseImportControllerTest.java +++ /dev/null @@ -1,247 +0,0 @@ - -package com.google.refine.extension.database; - -import static org.mockito.Mockito.when; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.RefineServlet; -import com.google.refine.extension.database.mysql.MySQLDatabaseService; -import com.google.refine.extension.database.stub.RefineDbServletStub; -import com.google.refine.importing.ImportingJob; -import com.google.refine.importing.ImportingManager; -import com.google.refine.io.FileProjectManager; -import com.google.refine.model.Project; -import com.google.refine.util.ParsingUtilities; - -@Test(groups = { "requiresMySQL" }) -public class DatabaseImportControllerTest extends DBExtensionTests { - - @Mock - private HttpServletRequest request; - - @Mock - private HttpServletResponse response; - - private Project project; - private ProjectMetadata metadata; - private ImportingJob job; - private RefineServlet servlet; - - private String JSON_OPTION = "{\"mode\":\"row-based\"}}"; - - private DatabaseConfiguration testDbConfig; - - private String query; - - // System under test - private DatabaseImportController SUT = null; - - @BeforeMethod - public void setUp() throws IOException { - MockitoAnnotations.initMocks(this); - - File dir = DBExtensionTestUtils.createTempDirectory("OR_DBExtension_Test_WorkspaceDir"); - FileProjectManager.initialize(dir); - - servlet = new RefineDbServletStub(); - ImportingManager.initialize(servlet); - project = new Project(); - metadata = new ProjectMetadata(); - job = ImportingManager.createJob(); - - metadata.setName("Database Import Test Project"); - ProjectManager.singleton.registerProject(project, metadata); - SUT = new DatabaseImportController(); - - } - - @AfterMethod - public void tearDown() { - SUT = null; - request = null; - response = null; - project = null; - metadata = null; - ImportingManager.disposeJob(job.id); - job = null; - // options = null; - } - - @Test - public void testDoGet() { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - try { - when(response.getWriter()).thenReturn(pw); - - SUT.doGet(request, response); - - String result = sw.getBuffer().toString().trim(); - ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); - String code = json.get("status").asText(); - String message = json.get("message").asText(); - Assert.assertNotNull(code); - Assert.assertNotNull(message); - Assert.assertEquals(code, "error"); - Assert.assertEquals(message, "GET not implemented"); - - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - @Test - public void testDoPostInvalidSubCommand() throws IOException, ServletException { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - when(request.getQueryString()).thenReturn( - "http://127.0.0.1:3333/command/core/importing-controller?controller=database/database-import-controller&subCommand=invalid-sub-command"); - - when(response.getWriter()).thenReturn(pw); - // test - SUT.doPost(request, response); - - String result = sw.getBuffer().toString().trim(); - ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); - - String code = json.get("status").asText(); - String message = json.get("message").asText(); - Assert.assertNotNull(code); - Assert.assertNotNull(message); - Assert.assertEquals(code, "error"); - Assert.assertEquals(message, "No such sub command"); - } - - @Test - public void testDoPostInitializeParser() throws ServletException, IOException { - - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - when(request.getQueryString()).thenReturn( - "http://127.0.0.1:3333/command/core/importing-controller?controller=database/database-import-controller&subCommand=initialize-parser-ui"); - when(response.getWriter()).thenReturn(pw); - - SUT.doPost(request, response); - - String result = sw.getBuffer().toString().trim(); - ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); - - String status = json.get("status").asText(); - // System.out.println("json::" + json); - Assert.assertEquals(status, "ok"); - } - - @Test - public void testDoPostParsePreview() throws IOException, ServletException { - - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - long jobId = job.id; - - when(request.getQueryString()).thenReturn( - "http://127.0.0.1:3333/command/core/importing-controller?controller=database%2Fdatabase-import-controller&jobID=" - + jobId + "&subCommand=parse-preview"); - when(response.getWriter()).thenReturn(pw); - - when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType()); - when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); - when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); - when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); - when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); - when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); - when(request.getParameter("query")).thenReturn(query); - when(request.getParameter("options")).thenReturn(JSON_OPTION); - - SUT.doPost(request, response); - - String result = sw.getBuffer().toString().trim(); - ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); - - String status = json.get("status").asText(); - // System.out.println("json::" + json); - Assert.assertEquals(status, "ok"); - } - - @Test - public void testDoPostCreateProject() throws IOException, ServletException { - - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - long jobId = job.id; - - when(request.getQueryString()).thenReturn( - "http://127.0.0.1:3333/command/core/importing-controller?controller=database%2Fdatabase-import-controller&jobID=" - + jobId + "&subCommand=create-project"); - when(response.getWriter()).thenReturn(pw); - - when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType()); - when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); - when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); - when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); - when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); - when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); - when(request.getParameter("query")).thenReturn(query); - when(request.getParameter("options")).thenReturn(JSON_OPTION); - - SUT.doPost(request, response); - - String result = sw.getBuffer().toString().trim(); - ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); - - String status = json.get("status").asText(); - // System.out.println("json::" + json); - Assert.assertEquals(status, "ok"); - } - - @BeforeTest - @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" }) - public void beforeTest( - @Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, - @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, - @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { - - MockitoAnnotations.initMocks(this); - - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(mySqlDbHost); - testDbConfig.setDatabaseName(mySqlDbName); - testDbConfig.setDatabasePassword(mySqlDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); - testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(mySqlDbUser); - testDbConfig.setUseSSL(false); - query = "SELECT count(*) FROM " + mySqlTestTable; - - // testTable = mySqlTestTable; - - DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); - - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DatabaseServiceTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DatabaseServiceTest.java deleted file mode 100644 index c43cf7a51..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DatabaseServiceTest.java +++ /dev/null @@ -1,149 +0,0 @@ - -package com.google.refine.extension.database; - -import java.sql.Connection; -import java.util.List; - -import com.google.refine.extension.database.sqlite.SQLiteDatabaseService; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.google.refine.extension.database.mariadb.MariaDBDatabaseService; -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.extension.database.model.DatabaseRow; -import com.google.refine.extension.database.mysql.MySQLDatabaseService; -import com.google.refine.extension.database.pgsql.PgSQLDatabaseService; - -public class DatabaseServiceTest extends DBExtensionTests { - - private DatabaseConfiguration testDbConfig; - private String testTable; - - @BeforeTest - @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" }) - public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, - @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, - @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { - - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(mySqlDbHost); - testDbConfig.setDatabaseName(mySqlDbName); - testDbConfig.setDatabasePassword(mySqlDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); - testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(mySqlDbUser); - testDbConfig.setUseSSL(false); - - testTable = mySqlTestTable; - // DBExtensionTestUtils.initTestData(testDbConfig); - - DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); - DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); - DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance()); - - } - - @Test - public void testGetDatabaseUrl() { - DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); - String dbUrl = dbService.getDatabaseUrl(testDbConfig); - Assert.assertNotNull(dbUrl); - Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); - } - - @Test(groups = { "requiresPgSQL" }) - public void testGetPgSQLDBService() { - DatabaseService dbService = DatabaseService.get(PgSQLDatabaseService.DB_NAME); - Assert.assertNotNull(dbService); - Assert.assertEquals(dbService.getClass(), PgSQLDatabaseService.class); - } - - @Test(groups = { "requiresMySQL" }) - public void testGetMySQLDBService() { - - DatabaseService dbService = DatabaseService.get(MySQLDatabaseService.DB_NAME); - Assert.assertNotNull(dbService); - Assert.assertEquals(dbService.getClass(), MySQLDatabaseService.class); - } - - @Test(groups = { "requiresMariaDB" }) - public void testGetMariaDBSQLDBService() { - - DatabaseService dbService = DatabaseService.get(MariaDBDatabaseService.DB_NAME); - Assert.assertNotNull(dbService); - Assert.assertEquals(dbService.getClass(), MariaDBDatabaseService.class); - } - - @Test(groups = { "requiresSQLite" }) - public void testGetSQLiteDBService() { - - DatabaseService dbService = DatabaseService.get(SQLiteDatabaseService.DB_NAME); - Assert.assertNotNull(dbService); - Assert.assertEquals(dbService.getClass(), SQLiteDatabaseService.class); - } - - @Test(groups = { "requiresMySQL" }) - public void testGetConnection() throws DatabaseServiceException { - DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); - Connection conn = dbService.getConnection(testDbConfig); - Assert.assertNotNull(conn); - } - - @Test(groups = { "requiresMySQL" }) - public void testTestConnection() throws DatabaseServiceException { - DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); - boolean result = dbService.testConnection(testDbConfig); - Assert.assertEquals(result, true); - } - - @Test(groups = { "requiresMySQL" }) - public void testConnect() throws DatabaseServiceException { - DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); - DatabaseInfo databaseInfo = dbService.connect(testDbConfig); - Assert.assertNotNull(databaseInfo); - } - - @Test(groups = { "requiresMySQL" }) - public void testExecuteQuery() throws DatabaseServiceException { - DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); - DatabaseInfo databaseInfo = dbService.testQuery(testDbConfig, - "SELECT * FROM " + testTable); - - Assert.assertNotNull(databaseInfo); - } - - @Test - public void testBuildLimitQuery() { - DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); - String limitQuery = dbService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable); - Assert.assertNotNull(limitQuery); - Assert.assertEquals(limitQuery, "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";"); - } - - @Test(groups = { "requiresMySQL" }) - public void testGetColumns() throws DatabaseServiceException { - List dbColumns; - - DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); - dbColumns = dbService.getColumns(testDbConfig, "SELECT * FROM " + testTable); - Assert.assertNotNull(dbColumns); - - int cols = dbColumns.size(); - Assert.assertEquals(cols, 10); - } - - @Test(groups = { "requiresMySQL" }) - public void testGetRows() throws DatabaseServiceException { - DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); - List dbRows = dbService.getRows(testDbConfig, - "SELECT * FROM " + testTable); - - Assert.assertNotNull(dbRows); - Assert.assertEquals(dbRows.size(), 1); - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitMariaDBTestDatabase.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitMariaDBTestDatabase.java deleted file mode 100644 index 9a25dc399..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitMariaDBTestDatabase.java +++ /dev/null @@ -1,44 +0,0 @@ - -package com.google.refine.extension.database; - -import java.sql.SQLException; - -import org.testng.annotations.AfterSuite; -import org.testng.annotations.BeforeSuite; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.google.refine.extension.database.mariadb.MariaDBDatabaseService; - -@Test(groups = { "requiresMariaDB" }) -public class InitMariaDBTestDatabase extends DBExtensionTests { - - private DatabaseConfiguration mariadbDbConfig; - - @BeforeSuite - @Parameters({ "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariadbTestTable" }) - public void beforeSuite( - @Optional(DEFAULT_MARIADB_NAME) String mariadbDbName, @Optional(DEFAULT_MARIADB_HOST) String mariadbDbHost, - @Optional(DEFAULT_MARIADB_PORT) String mariadbDbPort, @Optional(DEFAULT_MARIADB_USER) String mariadbDbUser, - @Optional(DEFAULT_MARIADB_PASSWORD) String mariadbDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariadbTestTable) - throws DatabaseServiceException, SQLException { - - mariadbDbConfig = new DatabaseConfiguration(); - mariadbDbConfig.setDatabaseHost(mariadbDbHost); - mariadbDbConfig.setDatabaseName(mariadbDbName); - mariadbDbConfig.setDatabasePassword(mariadbDbPassword); - mariadbDbConfig.setDatabasePort(Integer.parseInt(mariadbDbPort)); - mariadbDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME); - mariadbDbConfig.setDatabaseUser(mariadbDbUser); - mariadbDbConfig.setUseSSL(false); - - DBExtensionTestUtils.initTestData(mariadbDbConfig); - } - - @AfterSuite - public void afterSuite() { - DBExtensionTestUtils.cleanUpTestData(mariadbDbConfig); - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitMySQLTestDatabase.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitMySQLTestDatabase.java deleted file mode 100644 index 3125fa30b..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitMySQLTestDatabase.java +++ /dev/null @@ -1,44 +0,0 @@ - -package com.google.refine.extension.database; - -import java.sql.SQLException; - -import org.testng.annotations.AfterSuite; -import org.testng.annotations.BeforeSuite; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.google.refine.extension.database.mysql.MySQLDatabaseService; - -@Test(groups = { "requiresMySQL" }) -public class InitMySQLTestDatabase extends DBExtensionTests { - - private DatabaseConfiguration mysqlDbConfig; - - @BeforeSuite - @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" }) - public void beforeSuite( - @Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, - @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, - @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) - throws DatabaseServiceException, SQLException { - - // System.out.println("@BeforeSuite\n"); - mysqlDbConfig = new DatabaseConfiguration(); - mysqlDbConfig.setDatabaseHost(mySqlDbHost); - mysqlDbConfig.setDatabaseName(mySqlDbName); - mysqlDbConfig.setDatabasePassword(mySqlDbPassword); - mysqlDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); - mysqlDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); - mysqlDbConfig.setDatabaseUser(mySqlDbUser); - mysqlDbConfig.setUseSSL(false); - - DBExtensionTestUtils.initTestData(mysqlDbConfig); - } - - @AfterSuite - public void afterSuite() { - DBExtensionTestUtils.cleanUpTestData(mysqlDbConfig); - } -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitPostgresTestDatabase.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitPostgresTestDatabase.java deleted file mode 100644 index 435f608f0..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitPostgresTestDatabase.java +++ /dev/null @@ -1,43 +0,0 @@ - -package com.google.refine.extension.database; - -import java.sql.SQLException; - -import org.testng.annotations.AfterSuite; -import org.testng.annotations.BeforeSuite; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.google.refine.extension.database.pgsql.PgSQLDatabaseService; - -@Test(groups = { "requiresPgSQL" }) -public class InitPostgresTestDatabase extends DBExtensionTests { - - private DatabaseConfiguration pgsqlDbConfig; - - @BeforeSuite - @Parameters({ "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable" }) - public void beforeSuite( - @Optional(DEFAULT_PGSQL_DB_NAME) String pgSqlDbName, @Optional(DEFAULT_PGSQL_HOST) String pgSqlDbHost, - @Optional(DEFAULT_PGSQL_PORT) String pgSqlDbPort, @Optional(DEFAULT_PGSQL_USER) String pgSqlDbUser, - @Optional(DEFAULT_PGSQL_PASSWORD) String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String pgSqlTestTable) - throws DatabaseServiceException, SQLException { - - pgsqlDbConfig = new DatabaseConfiguration(); - pgsqlDbConfig.setDatabaseHost(pgSqlDbHost); - pgsqlDbConfig.setDatabaseName(pgSqlDbName); - pgsqlDbConfig.setDatabasePassword(pgSqlDbPassword); - pgsqlDbConfig.setDatabasePort(Integer.parseInt(pgSqlDbPort)); - pgsqlDbConfig.setDatabaseType(PgSQLDatabaseService.DB_NAME); - pgsqlDbConfig.setDatabaseUser(pgSqlDbUser); - pgsqlDbConfig.setUseSSL(false); - - DBExtensionTestUtils.initTestData(pgsqlDbConfig); - } - - @AfterSuite - public void afterSuite() { - DBExtensionTestUtils.cleanUpTestData(pgsqlDbConfig); - } -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitSQLiteTestDatabase.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitSQLiteTestDatabase.java deleted file mode 100644 index aadd432f4..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitSQLiteTestDatabase.java +++ /dev/null @@ -1,33 +0,0 @@ - -package com.google.refine.extension.database; - -import java.sql.SQLException; - -import org.testng.annotations.AfterSuite; -import org.testng.annotations.BeforeSuite; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -@Test(groups = { "requiresMariaDB" }) -public class InitSQLiteTestDatabase extends DBExtensionTests { - - private DatabaseConfiguration sqliteDbConfig; - - @BeforeSuite - @Parameters({ "sqliteDbName", "sqliteTestTable" }) - public void beforeSuite( - @Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable) - throws DatabaseServiceException, SQLException { - - sqliteDbConfig = new DatabaseConfiguration(); - sqliteDbConfig.setDatabaseName(sqliteDbName); - - DBExtensionTestUtils.initTestData(sqliteDbConfig); - } - - @AfterSuite - public void afterSuite() { - DBExtensionTestUtils.cleanUpTestData(sqliteDbConfig); - } -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/SimpleTextEncryptorTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/SimpleTextEncryptorTest.java deleted file mode 100644 index db4d15646..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/SimpleTextEncryptorTest.java +++ /dev/null @@ -1,29 +0,0 @@ - -package com.google.refine.extension.database; - -import org.testng.Assert; -import org.testng.annotations.Test; - -public class SimpleTextEncryptorTest { - - @Test - public void encrypt() { - SimpleTextEncryptor textEncryptor = new SimpleTextEncryptor("WEWssa!@d445d"); - String password = "testpass"; - String encPass = textEncryptor.encrypt(password); - Assert.assertNotNull(encPass); - Assert.assertNotEquals(encPass, password); - - } - - @Test - public void decrypt() { - SimpleTextEncryptor textEncryptor = new SimpleTextEncryptor("OOEWssa!@d445d"); - String password = "testpass"; - String encPass = textEncryptor.encrypt(password); - Assert.assertNotNull(encPass); - Assert.assertNotEquals(encPass, password); - String decPass = textEncryptor.decrypt(encPass); - Assert.assertEquals(decPass, password); - } -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/ConnectCommandTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/ConnectCommandTest.java deleted file mode 100644 index ca6c1038b..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/ConnectCommandTest.java +++ /dev/null @@ -1,110 +0,0 @@ - -package com.google.refine.extension.database.cmd; - -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.refine.commands.Command; -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.mysql.MySQLDatabaseService; -import com.google.refine.util.ParsingUtilities; - -@Test(groups = { "requiresMySQL" }) -public class ConnectCommandTest extends DBExtensionTests { - - @Mock - private HttpServletRequest request; - - @Mock - private HttpServletResponse response; - - private DatabaseConfiguration testDbConfig; - // private String testTable; - - @BeforeTest - @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" }) - public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, - @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, - @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { - - MockitoAnnotations.initMocks(this); - - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(mySqlDbHost); - testDbConfig.setDatabaseName(mySqlDbName); - testDbConfig.setDatabasePassword(mySqlDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); - testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(mySqlDbUser); - testDbConfig.setUseSSL(false); - - // testTable = mySqlTestTable; - // DBExtensionTestUtils.initTestData(testDbConfig); - - DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); - - } - - @Test - public void testDoPost() throws IOException, ServletException { - - when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); - when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); - when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); - when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); - when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); - when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); - when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); - - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - ConnectCommand connectCommand = new ConnectCommand(); - - connectCommand.doPost(request, response); - - String result = sw.getBuffer().toString().trim(); - ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); - - String code = json.get("code").asText(); - Assert.assertEquals(code, "ok"); - - String databaseInfo = json.get("databaseInfo").asText(); - Assert.assertNotNull(databaseInfo); - } - - @Test - public void testCsrfProtection() throws ServletException, IOException { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - ConnectCommand connectCommand = new ConnectCommand(); - - connectCommand.doPost(request, response); - Assert.assertEquals( - ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", - ObjectNode.class), - ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class)); - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/ExecuteQueryCommandTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/ExecuteQueryCommandTest.java deleted file mode 100644 index 95fbb3d74..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/ExecuteQueryCommandTest.java +++ /dev/null @@ -1,110 +0,0 @@ - -package com.google.refine.extension.database.cmd; - -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.refine.commands.Command; -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.mysql.MySQLDatabaseService; -import com.google.refine.util.ParsingUtilities; - -@Test(groups = { "requiresMySQL" }) -public class ExecuteQueryCommandTest extends DBExtensionTests { - - @Mock - private HttpServletRequest request; - - @Mock - private HttpServletResponse response; - - private DatabaseConfiguration testDbConfig; - private String testTable; - - @BeforeTest - @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" }) - public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, - @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, - @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { - - MockitoAnnotations.initMocks(this); - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(mySqlDbHost); - testDbConfig.setDatabaseName(mySqlDbName); - testDbConfig.setDatabasePassword(mySqlDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); - testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(mySqlDbUser); - testDbConfig.setUseSSL(false); - - testTable = mySqlTestTable; - // DBExtensionTestUtils.initTestData(testDbConfig); - - DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); - - } - - @Test - public void testDoPost() throws IOException, ServletException { - - when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType()); - when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); - when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); - when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); - when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); - when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); - when(request.getParameter("queryString")).thenReturn("SELECT count(*) FROM " + testTable); - when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); - - StringWriter sw = new StringWriter(); - - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - ExecuteQueryCommand executeQueryCommand = new ExecuteQueryCommand(); - - executeQueryCommand.doPost(request, response); - - String result = sw.getBuffer().toString().trim(); - ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); - - String code = json.get("code").asText(); - Assert.assertEquals(code, "ok"); - - String queryResult = json.get("QueryResult").asText(); - Assert.assertNotNull(queryResult); - } - - @Test - public void testCsrfProtection() throws ServletException, IOException { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - ConnectCommand connectCommand = new ConnectCommand(); - - connectCommand.doPost(request, response); - Assert.assertEquals( - ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", - ObjectNode.class), - ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class)); - } -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/SavedConnectionCommandTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/SavedConnectionCommandTest.java deleted file mode 100644 index 40f709693..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/SavedConnectionCommandTest.java +++ /dev/null @@ -1,337 +0,0 @@ - -package com.google.refine.extension.database.cmd; - -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.times; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.http.HttpStatus; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.RefineServlet; -import com.google.refine.commands.Command; -import com.google.refine.extension.database.DBExtensionTestUtils; -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.mysql.MySQLDatabaseService; -import com.google.refine.extension.database.stub.RefineDbServletStub; -import com.google.refine.importing.ImportingManager; -import com.google.refine.io.FileProjectManager; -import com.google.refine.model.Project; -import com.google.refine.util.ParsingUtilities; - -public class SavedConnectionCommandTest extends DBExtensionTests { - - @Mock - private HttpServletRequest request; - - @Mock - private HttpServletResponse response; - - private DatabaseConfiguration testDbConfig; - - private Project project; - private ProjectMetadata metadata; - // private ImportingJob job; - private RefineServlet servlet; - - // private String JSON_OPTION = "{\"mode\":\"row-based\"}}"; - - // System under test - private SavedConnectionCommand SUT = null; - - @BeforeMethod - public void setUp() throws IOException { - MockitoAnnotations.initMocks(this); - - File dir = DBExtensionTestUtils.createTempDirectory("OR_DBExtension_Test_WorkspaceDir"); - FileProjectManager.initialize(dir); - - servlet = new RefineDbServletStub(); - ImportingManager.initialize(servlet); - project = new Project(); - metadata = new ProjectMetadata(); - // job = ImportingManager.createJob(); - - metadata.setName("Save DB Config Test Project"); - ProjectManager.singleton.registerProject(project, metadata); - SUT = new SavedConnectionCommand(); - - } - - @AfterMethod - public void tearDown() { - SUT = null; - request = null; - response = null; - project = null; - metadata = null; - // ImportingManager.disposeJob(job.id); - // job = null; - // options = null; - } - - @BeforeTest - @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" }) - public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, - @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, - @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { - - // MockitoAnnotations.initMocks(this); - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(mySqlDbHost); - testDbConfig.setDatabaseName(mySqlDbName); - testDbConfig.setDatabasePassword(mySqlDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); - testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(mySqlDbUser); - testDbConfig.setUseSSL(false); - - DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); - - } - - private void saveDatabaseConfiguration(String savedDbName) { - - when(request.getParameter("connectionName")).thenReturn(savedDbName); - when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); - when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); - when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); - when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); - when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); - when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); - when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - try { - when(response.getWriter()).thenReturn(pw); - - SUT.doPost(request, response); - - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - } - - @Test - public void testDoPost() throws IOException, ServletException { - - when(request.getParameter("connectionName")).thenReturn("test-db-name"); - when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); - when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); - when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); - when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); - when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); - when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); - when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); - - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - - SUT.doPost(request, response); - - String result = sw.getBuffer().toString().trim(); - assertNotNull(result); - assertFalse(result.isEmpty(), "Valid response Message expected!"); - - ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); - // System.out.println("json:" + json); - - ArrayNode savedConnections = (ArrayNode) json.get("savedConnections"); - Assert.assertNotNull(savedConnections); - - int len = savedConnections.size(); - - Assert.assertEquals(len, 1); - } - - @Test - public void testDoGet() throws IOException, ServletException { - String testDbName = "testLocalDb"; - // add saved connection - saveDatabaseConfiguration(testDbName); - - when(request.getParameter("connectionName")).thenReturn(testDbName); - when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); - when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); - when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); - when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); - when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); - when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); - when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); - - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - - SUT.doGet(request, response); - - ObjectNode json = ParsingUtilities.mapper.readValue(sw.getBuffer().toString().trim(), ObjectNode.class); - - ArrayNode savedConnections = (ArrayNode) json.get("savedConnections"); - Assert.assertNotNull(savedConnections); - - Assert.assertEquals(savedConnections.size(), 1); - - ObjectNode sc = (ObjectNode) savedConnections.get(0); - String connName = sc.get("connectionName").asText(); - Assert.assertEquals(connName, testDbName); - } - - @Test - public void testDoPut() throws IOException, ServletException { - String testDbName = "testLocalDb"; - saveDatabaseConfiguration(testDbName); - - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - - // modify database config - String newHost = "localhost"; - when(request.getParameter("connectionName")).thenReturn(testDbName); - when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); - when(request.getParameter("databaseServer")).thenReturn(newHost); - when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); - when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); - when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); - when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); - when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); - - SUT.doPut(request, response); - - ObjectNode json = ParsingUtilities.mapper.readValue(sw.getBuffer().toString().trim(), ObjectNode.class); - ArrayNode savedConnections = (ArrayNode) json.get("savedConnections"); - Assert.assertNotNull(savedConnections); - - Assert.assertEquals(savedConnections.size(), 1); - - ObjectNode sc = (ObjectNode) savedConnections.get(0); - String newDbHost = sc.get("databaseHost").asText(); - Assert.assertEquals(newDbHost, newHost); - } - - @Test - public void testDoDeleteValidConnectionName() { - String testDbName = "testLocalDb"; - saveDatabaseConfiguration(testDbName); - - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - try { - when(response.getWriter()).thenReturn(pw); - when(request.getParameter("connectionName")).thenReturn(testDbName); - SUT.doDelete(request, response); - - ObjectNode json = ParsingUtilities.mapper.readValue(sw.getBuffer().toString().trim(), ObjectNode.class); - ArrayNode savedConnections = (ArrayNode) json.get("savedConnections"); - Assert.assertNotNull(savedConnections); - - Assert.assertEquals(savedConnections.size(), 0); - - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - @Test - public void testDoDeleteInValidConnectionName() { - String testDbName = "testLocalDb"; - saveDatabaseConfiguration(testDbName); - - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - try { - when(response.getWriter()).thenReturn(pw); - - when(request.getParameter("connectionName")).thenReturn("noDbName"); - - SUT.doDelete(request, response); - - ObjectNode json = ParsingUtilities.mapper.createObjectNode(); - - Assert.assertNotNull(json); - - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - /** - * Added to check XSS invalid tokens - * - * @throws IOException - * @throws ServletException - */ - @Test - public void testDoPostInvalidConnectionName() throws IOException, ServletException { - - when(request.getParameter("connectionName")).thenReturn(""); - when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); - when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); - when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); - when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); - when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); - when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); - when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); - - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - - SUT.doPost(request, response); - - verify(response, times(1)).sendError(HttpStatus.SC_BAD_REQUEST, "Connection Name is Invalid. Expecting [a-zA-Z0-9._-]"); - } - - @Test - public void testCsrfProtection() throws ServletException, IOException { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - - SUT.doPost(request, response); - Assert.assertEquals( - ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", - ObjectNode.class), - ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class)); - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/TestConnectCommandTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/TestConnectCommandTest.java deleted file mode 100644 index 5221dfe8f..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/TestConnectCommandTest.java +++ /dev/null @@ -1,109 +0,0 @@ - -package com.google.refine.extension.database.cmd; - -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.refine.commands.Command; -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.mysql.MySQLDatabaseService; -import com.google.refine.util.ParsingUtilities; - -@Test(groups = { "requiresMySQL" }) -public class TestConnectCommandTest extends DBExtensionTests { - - @Mock - private HttpServletRequest request; - - @Mock - private HttpServletResponse response; - - private DatabaseConfiguration testDbConfig; - // private String testTable; - - @BeforeTest - @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" }) - public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, - @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, - @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { - - MockitoAnnotations.initMocks(this); - - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(mySqlDbHost); - testDbConfig.setDatabaseName(mySqlDbName); - testDbConfig.setDatabasePassword(mySqlDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); - testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(mySqlDbUser); - testDbConfig.setUseSSL(false); - - // testTable = mySqlTestTable; - // DBExtensionTestUtils.initTestData(testDbConfig); - - DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); - - } - - @Test - public void testDoPost() throws IOException, ServletException { - - when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); - when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); - when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); - when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); - when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); - when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); - when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); - - StringWriter sw = new StringWriter(); - - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - TestConnectCommand connectCommand = new TestConnectCommand(); - - connectCommand.doPost(request, response); - - String result = sw.getBuffer().toString().trim(); - ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); - - String code = json.get("code").asText(); - Assert.assertEquals(code, "ok"); - - } - - @Test - public void testCsrfProtection() throws ServletException, IOException { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - ConnectCommand connectCommand = new ConnectCommand(); - - connectCommand.doPost(request, response); - Assert.assertEquals( - ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", - ObjectNode.class), - ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class)); - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/TestQueryCommandTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/TestQueryCommandTest.java deleted file mode 100644 index 1351095ba..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/TestQueryCommandTest.java +++ /dev/null @@ -1,112 +0,0 @@ - -package com.google.refine.extension.database.cmd; - -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.refine.commands.Command; -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.mysql.MySQLDatabaseService; -import com.google.refine.util.ParsingUtilities; - -@Test(groups = { "requiresMySQL" }) -public class TestQueryCommandTest extends DBExtensionTests { - - @Mock - private HttpServletRequest request; - - @Mock - private HttpServletResponse response; - - private DatabaseConfiguration testDbConfig; - private String testTable; - - @BeforeTest - @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" }) - public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, - @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, - @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { - - MockitoAnnotations.initMocks(this); - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(mySqlDbHost); - testDbConfig.setDatabaseName(mySqlDbName); - testDbConfig.setDatabasePassword(mySqlDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); - testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(mySqlDbUser); - testDbConfig.setUseSSL(false); - - testTable = mySqlTestTable; - // DBExtensionTestUtils.initTestData(testDbConfig); - - DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); - - } - - @Test - public void testDoPost() throws IOException, ServletException { - - when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType()); - when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); - when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); - when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); - when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); - when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); - when(request.getParameter("query")).thenReturn("SELECT count(*) FROM " + testTable); - when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); - - StringWriter sw = new StringWriter(); - - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - TestQueryCommand executeQueryCommand = new TestQueryCommand(); - - executeQueryCommand.doPost(request, response); - - String result = sw.getBuffer().toString().trim(); - ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); - - String code = json.get("code").asText(); - Assert.assertEquals(code, "ok"); - - String queryResult = json.get("QueryResult").asText(); - Assert.assertNotNull(queryResult); - - } - - @Test - public void testCsrfProtection() throws ServletException, IOException { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - - when(response.getWriter()).thenReturn(pw); - TestQueryCommand connectCommand = new TestQueryCommand(); - - connectCommand.doPost(request, response); - Assert.assertEquals( - ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", - ObjectNode.class), - ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class)); - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManagerTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManagerTest.java deleted file mode 100644 index 18824776c..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManagerTest.java +++ /dev/null @@ -1,73 +0,0 @@ - -package com.google.refine.extension.database.mariadb; - -import java.sql.Connection; -import java.sql.SQLException; - -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; - -@Test(groups = { "requiresMariaDB" }) -public class MariaDBConnectionManagerTest extends DBExtensionTests { - - private DatabaseConfiguration testDbConfig; - - @BeforeTest - @Parameters({ "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariaTestTable" }) - public void beforeTest(@Optional(DEFAULT_MARIADB_NAME) String mariaDbName, @Optional(DEFAULT_MARIADB_HOST) String mariaDbHost, - @Optional(DEFAULT_MARIADB_PORT) String mariaDbPort, @Optional(DEFAULT_MARIADB_USER) String mariaDbUser, - @Optional(DEFAULT_MARIADB_PASSWORD) String mariaDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariaDbTestTable) { - - MockitoAnnotations.initMocks(this); - - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(mariaDbHost); - testDbConfig.setDatabaseName(mariaDbName); - testDbConfig.setDatabasePassword(mariaDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(mariaDbPort)); - testDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(mariaDbUser); - testDbConfig.setUseSSL(false); - -// testTable = mariaDbTestTable; - // DBExtensionTestUtils.initTestData(testDbConfig); - - DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); - - } - - @Test - public void testTestConnection() throws DatabaseServiceException { - boolean conn = MariaDBConnectionManager.getInstance().testConnection(testDbConfig); - Assert.assertEquals(conn, true); - } - - @Test - public void testGetConnection() throws DatabaseServiceException { - Connection conn = MariaDBConnectionManager.getInstance().getConnection(testDbConfig, true); - Assert.assertNotNull(conn); - } - - @Test - public void testShutdown() throws DatabaseServiceException, SQLException { - Connection conn = MariaDBConnectionManager.getInstance().getConnection(testDbConfig, true); - Assert.assertNotNull(conn); - - MariaDBConnectionManager.getInstance().shutdown(); - - if (conn != null) { - Assert.assertEquals(conn.isClosed(), true); - } - - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseServiceTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseServiceTest.java deleted file mode 100644 index b0844b3cf..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseServiceTest.java +++ /dev/null @@ -1,130 +0,0 @@ - -package com.google.refine.extension.database.mariadb; - -import java.sql.Connection; -import java.util.List; - -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.google.refine.extension.database.DBExtensionTestUtils; -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.extension.database.model.DatabaseRow; - -@Test(groups = { "requiresMariaDB" }) -public class MariaDBDatabaseServiceTest extends DBExtensionTests { - - private DatabaseConfiguration testDbConfig; - - private String testTable; - - @BeforeTest - @Parameters({ "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariadbTestTable" }) - public void beforeTest(@Optional(DEFAULT_MARIADB_NAME) String mariaDbName, @Optional(DEFAULT_MARIADB_HOST) String mariaDbHost, - @Optional(DEFAULT_MARIADB_PORT) String mariaDbPort, @Optional(DEFAULT_MARIADB_USER) String mariaDbUser, - @Optional(DEFAULT_MARIADB_PASSWORD) String mariaDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariaDbTestTable) { - - MockitoAnnotations.initMocks(this); - - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(mariaDbHost); - testDbConfig.setDatabaseName(mariaDbName); - testDbConfig.setDatabasePassword(mariaDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(mariaDbPort)); - testDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(mariaDbUser); - testDbConfig.setUseSSL(false); - - testTable = mariaDbTestTable; - // DBExtensionTestUtils.initTestData(testDbConfig); - - DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); - - } - - @Test - public void testGetDatabaseUrl() { - MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME); - String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig); - // System.out.println("dbUrl:" + dbUrl); - Assert.assertNotNull(dbUrl); - Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); - } - - @Test - public void testGetConnection() throws DatabaseServiceException { - MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME); - Connection conn = pgSqlService.getConnection(testDbConfig); - - Assert.assertNotNull(conn); - } - - @Test - public void testTestConnection() throws DatabaseServiceException { - MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME); - - boolean result = pgSqlService.testConnection(testDbConfig); - Assert.assertEquals(result, true); - } - - @Test - public void testConnect() throws DatabaseServiceException { - MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME); - DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig); - Assert.assertNotNull(databaseInfo); - } - - @Test - public void testExecuteQuery() throws DatabaseServiceException { - MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService - .get(MariaDBDatabaseService.DB_NAME); - DatabaseInfo databaseInfo = pgSqlService.testQuery(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(databaseInfo); - } - - @Test - public void testBuildLimitQuery() { - MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME); - String limitQuery = pgSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable); - Assert.assertNotNull(limitQuery); - Assert.assertEquals(limitQuery, "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";"); - } - - @Test - public void testGetRows() throws DatabaseServiceException { - MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService - .get(MariaDBDatabaseService.DB_NAME); - List dbRows = pgSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(dbRows); - } - - @Test - public void testGetInstance() { - MariaDBDatabaseService instance = MariaDBDatabaseService.getInstance(); - Assert.assertNotNull(instance); - } - - @Test - public void testGetColumns() throws DatabaseServiceException { - List dbColumns; - - MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService - .get(MariaDBDatabaseService.DB_NAME); - - dbColumns = pgSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(dbColumns); - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mysql/MySQLConnectionManagerTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mysql/MySQLConnectionManagerTest.java deleted file mode 100644 index f896a69fc..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mysql/MySQLConnectionManagerTest.java +++ /dev/null @@ -1,75 +0,0 @@ - -package com.google.refine.extension.database.mysql; - -import java.sql.Connection; -import java.sql.SQLException; - -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; - -@Test(groups = { "requiresMySQL" }) -public class MySQLConnectionManagerTest extends DBExtensionTests { - - private DatabaseConfiguration testDbConfig; - - @BeforeTest - @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" }) - public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, - @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, - @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { - - MockitoAnnotations.initMocks(this); - - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(mySqlDbHost); - testDbConfig.setDatabaseName(mySqlDbName); - testDbConfig.setDatabasePassword(mySqlDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); - testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(mySqlDbUser); - testDbConfig.setUseSSL(false); - - // testTable = mySqlTestTable; - // DBExtensionTestUtils.initTestData(testDbConfig); - - DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); - - } - - @Test - public void testTestConnection() throws DatabaseServiceException { - - boolean conn = MySQLConnectionManager.getInstance().testConnection(testDbConfig); - Assert.assertEquals(conn, true); - } - - @Test - public void testGetConnection() throws DatabaseServiceException { - - Connection conn = MySQLConnectionManager.getInstance().getConnection(testDbConfig, true); - Assert.assertNotNull(conn); - } - - @Test - public void testShutdown() throws DatabaseServiceException, SQLException { - - Connection conn = MySQLConnectionManager.getInstance().getConnection(testDbConfig, true); - Assert.assertNotNull(conn); - - MySQLConnectionManager.getInstance().shutdown(); - - if (conn != null) { - Assert.assertEquals(conn.isClosed(), true); - } - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mysql/MySQLDatabaseServiceTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mysql/MySQLDatabaseServiceTest.java deleted file mode 100644 index 6c1ba3784..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mysql/MySQLDatabaseServiceTest.java +++ /dev/null @@ -1,129 +0,0 @@ - -package com.google.refine.extension.database.mysql; - -import java.sql.Connection; -import java.util.List; - -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.google.refine.extension.database.DBExtensionTestUtils; -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.extension.database.model.DatabaseRow; - -@Test(groups = { "requiresMySQL" }) -public class MySQLDatabaseServiceTest extends DBExtensionTests { - - private DatabaseConfiguration testDbConfig; - private String testTable; - - @BeforeTest - @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" }) - public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, - @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, - @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { - - MockitoAnnotations.initMocks(this); - - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(mySqlDbHost); - testDbConfig.setDatabaseName(mySqlDbName); - testDbConfig.setDatabasePassword(mySqlDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); - testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(mySqlDbUser); - testDbConfig.setUseSSL(false); - - testTable = mySqlTestTable; - // DBExtensionTestUtils.initTestData(testDbConfig); - - DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); - - } - - @Test - public void testGetDatabaseUrl() { - MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME); - String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig); - // System.out.println("dbUrl:" + dbUrl); - Assert.assertNotNull(dbUrl); - Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); - } - - @Test - public void testGetConnection() throws DatabaseServiceException { - MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME); - Connection conn = pgSqlService.getConnection(testDbConfig); - - Assert.assertNotNull(conn); - } - - @Test - public void testTestConnection() throws DatabaseServiceException { - MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME); - - boolean result = pgSqlService.testConnection(testDbConfig); - Assert.assertEquals(result, true); - } - - @Test - public void testConnect() throws DatabaseServiceException { - MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME); - DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig); - Assert.assertNotNull(databaseInfo); - } - - @Test - public void testExecuteQuery() throws DatabaseServiceException { - - MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService - .get(MySQLDatabaseService.DB_NAME); - DatabaseInfo databaseInfo = pgSqlService.testQuery(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(databaseInfo); - } - - @Test - public void testBuildLimitQuery() { - MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME); - String limitQuery = pgSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable); - Assert.assertNotNull(limitQuery); - Assert.assertEquals(limitQuery, "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";"); - } - - @Test - public void testGetRows() throws DatabaseServiceException { - MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService - .get(MySQLDatabaseService.DB_NAME); - List dbRows = pgSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(dbRows); - } - - @Test - public void testGetInstance() { - MySQLDatabaseService instance = MySQLDatabaseService.getInstance(); - Assert.assertNotNull(instance); - } - - @Test - public void testGetColumns() throws DatabaseServiceException { - List dbColumns; - MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService - .get(MySQLDatabaseService.DB_NAME); - - dbColumns = pgSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(dbColumns); - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/pgsql/PgSQLConnectionManagerTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/pgsql/PgSQLConnectionManagerTest.java deleted file mode 100644 index 9f3a85b85..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/pgsql/PgSQLConnectionManagerTest.java +++ /dev/null @@ -1,76 +0,0 @@ - -package com.google.refine.extension.database.pgsql; - -import java.sql.Connection; -import java.sql.SQLException; - -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; - -@Test(groups = { "requiresPgSQL" }) -public class PgSQLConnectionManagerTest extends DBExtensionTests { - - private DatabaseConfiguration testDbConfig; - - @BeforeTest - @Parameters({ "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable" }) - public void beforeTest(@Optional(DEFAULT_PGSQL_DB_NAME) String pgSqlDbName, @Optional(DEFAULT_PGSQL_HOST) String pgSqlDbHost, - @Optional(DEFAULT_PGSQL_PORT) String pgSqlDbPort, @Optional(DEFAULT_PGSQL_USER) String pgSqlDbUser, - @Optional(DEFAULT_PGSQL_PASSWORD) String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String pgSqlTestTable) { - - MockitoAnnotations.initMocks(this); - - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(pgSqlDbHost); - testDbConfig.setDatabaseName(pgSqlDbName); - testDbConfig.setDatabasePassword(pgSqlDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(pgSqlDbPort)); - testDbConfig.setDatabaseType(PgSQLDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(pgSqlDbUser); - testDbConfig.setUseSSL(false); - - // testTable = mySqlTestTable; - // DBExtensionTestUtils.initTestData(testDbConfig); - - DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance()); - - } - - @Test - public void testTestConnection() throws DatabaseServiceException { - - boolean isConnected = PgSQLConnectionManager.getInstance().testConnection(testDbConfig); - Assert.assertEquals(isConnected, true); - } - - @Test - public void testGetConnection() throws DatabaseServiceException { - - Connection conn = PgSQLConnectionManager.getInstance().getConnection(testDbConfig, true); - Assert.assertNotNull(conn); - } - - @Test - public void testShutdown() throws DatabaseServiceException, SQLException { - - Connection conn = PgSQLConnectionManager.getInstance().getConnection(testDbConfig, true); - Assert.assertNotNull(conn); - - PgSQLConnectionManager.getInstance().shutdown(); - - if (conn != null) { - Assert.assertEquals(conn.isClosed(), true); - } - - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/pgsql/PgSQLDatabaseServiceTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/pgsql/PgSQLDatabaseServiceTest.java deleted file mode 100644 index fc5a0306c..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/pgsql/PgSQLDatabaseServiceTest.java +++ /dev/null @@ -1,131 +0,0 @@ - -package com.google.refine.extension.database.pgsql; - -import java.sql.Connection; -import java.util.List; - -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Optional; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import com.google.refine.extension.database.DBExtensionTestUtils; -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.extension.database.model.DatabaseRow; - -@Test(groups = { "requiresPgSQL" }) -public class PgSQLDatabaseServiceTest extends DBExtensionTests { - - private DatabaseConfiguration testDbConfig; - private String testTable; - - @BeforeTest - @Parameters({ "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable" }) - public void beforeTest(@Optional(DEFAULT_PGSQL_DB_NAME) String pgSqlDbName, @Optional(DEFAULT_PGSQL_HOST) String pgSqlDbHost, - @Optional(DEFAULT_PGSQL_PORT) String pgSqlDbPort, @Optional(DEFAULT_PGSQL_USER) String pgSqlDbUser, - @Optional(DEFAULT_PGSQL_PASSWORD) String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String pgSqlTestTable) { - - MockitoAnnotations.initMocks(this); - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseHost(pgSqlDbHost); - testDbConfig.setDatabaseName(pgSqlDbName); - testDbConfig.setDatabasePassword(pgSqlDbPassword); - testDbConfig.setDatabasePort(Integer.parseInt(pgSqlDbPort)); - testDbConfig.setDatabaseType(PgSQLDatabaseService.DB_NAME); - testDbConfig.setDatabaseUser(pgSqlDbUser); - testDbConfig.setUseSSL(false); - - testTable = pgSqlTestTable; - // DBExtensionTestUtils.initTestData(testDbConfig); - - DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance()); - } - - @Test - public void testGetDatabaseUrl() { - PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService.get(PgSQLDatabaseService.DB_NAME); - String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig); - - Assert.assertNotNull(dbUrl); - Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); - } - - @Test - public void testGetConnection() throws DatabaseServiceException { - - PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService.get(PgSQLDatabaseService.DB_NAME); - Connection conn = pgSqlService.getConnection(testDbConfig); - - Assert.assertNotNull(conn); - } - - @Test - public void testTestConnection() throws DatabaseServiceException { - PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService.get(PgSQLDatabaseService.DB_NAME); - - boolean result = pgSqlService.testConnection(testDbConfig); - Assert.assertEquals(result, true); - } - - @Test - public void testConnect() throws DatabaseServiceException { - - PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService.get(PgSQLDatabaseService.DB_NAME); - DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig); - Assert.assertNotNull(databaseInfo); - } - - @Test - public void testExecuteQuery() throws DatabaseServiceException { - - PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService - .get(PgSQLDatabaseService.DB_NAME); - DatabaseInfo databaseInfo = pgSqlService.testQuery(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(databaseInfo); - } - - @Test - public void testBuildLimitQuery() { - PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService.get(PgSQLDatabaseService.DB_NAME); - String limitQuery = pgSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable); - Assert.assertNotNull(limitQuery); - Assert.assertEquals(limitQuery, "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";"); - } - - @Test - public void testGetRows() throws DatabaseServiceException { - PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService - .get(PgSQLDatabaseService.DB_NAME); - List dbRows = pgSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(dbRows); - } - - @Test - public void testGetInstance() { - PgSQLDatabaseService instance = PgSQLDatabaseService.getInstance(); - Assert.assertNotNull(instance); - } - - @Test - public void testGetColumns() throws DatabaseServiceException { - List dbColumns; - - PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService - .get(PgSQLDatabaseService.DB_NAME); - - dbColumns = pgSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(dbColumns); - - } - -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManagerTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManagerTest.java deleted file mode 100644 index 2daf059c5..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManagerTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2020, Chris Parker - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package com.google.refine.extension.database.sqlite; - -import com.google.refine.extension.database.DBExtensionTests; -import com.google.refine.extension.database.DatabaseConfiguration; -import com.google.refine.extension.database.DatabaseService; -import com.google.refine.extension.database.DatabaseServiceException; -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.*; - -import java.io.File; -import java.sql.Connection; -import java.sql.SQLException; - -@Test(groups = { "requiresSQLite" }) -public class SQLiteConnectionManagerTest extends DBExtensionTests { - - private DatabaseConfiguration testDbConfig; - - @BeforeTest - @Parameters({ "sqliteDbName", "sqliteTestTable" }) - public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, - @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable) - throws DatabaseServiceException, SQLException { - - MockitoAnnotations.initMocks(this); - - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseName(sqliteDbName); - testDbConfig.setDatabaseType(SQLiteDatabaseService.DB_NAME); - - DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance()); - } - - @AfterTest - @Parameters({ "sqliteDbName" }) - public void afterTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName) { - File f = new File(sqliteDbName); - if (f.exists()) { - f.delete(); - } - } - - @Test - public void testTestConnection() throws DatabaseServiceException { - boolean isConnected = SQLiteConnectionManager.getInstance().testConnection(testDbConfig); - Assert.assertTrue(isConnected); - } - - @Test - public void testGetConnection() throws DatabaseServiceException { - - Connection conn = SQLiteConnectionManager.getInstance().getConnection(testDbConfig); - Assert.assertNotNull(conn); - } - - @Test - public void testShutdown() throws DatabaseServiceException, SQLException { - - Connection conn = SQLiteConnectionManager.getInstance().getConnection(testDbConfig); - Assert.assertNotNull(conn); - - SQLiteConnectionManager.getInstance().shutdown(); - - if (conn != null) { - Assert.assertTrue(conn.isClosed()); - } - - } -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseServiceTest.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseServiceTest.java deleted file mode 100644 index c29cc82c2..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseServiceTest.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright (c) 2020, Chris Parker - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package com.google.refine.extension.database.sqlite; - -import com.google.refine.extension.database.*; -import com.google.refine.extension.database.model.DatabaseColumn; -import com.google.refine.extension.database.model.DatabaseInfo; -import com.google.refine.extension.database.model.DatabaseRow; -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.*; - -import java.io.File; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.List; - -@Test(groups = { "requiresSQLite" }) -public class SQLiteDatabaseServiceTest extends DBExtensionTests { - - private DatabaseConfiguration testDbConfig; - private String testTable; - - @BeforeTest - @Parameters({ "sqliteDbName", "sqliteTestTable" }) - public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, - @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable) - throws DatabaseServiceException, SQLException { - - MockitoAnnotations.initMocks(this); - testDbConfig = new DatabaseConfiguration(); - testDbConfig.setDatabaseName(sqliteDbName); - testDbConfig.setDatabaseType(SQLiteDatabaseService.DB_NAME); - - testTable = sqliteTestTable; - DBExtensionTestUtils.initTestData(testDbConfig, sqliteTestTable); - - DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance()); - } - - @AfterTest - @Parameters({ "sqliteDbName" }) - public void afterTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName) { - File f = new File(sqliteDbName); - if (f.exists()) { - f.delete(); - } - } - - @Test - public void testGetDatabaseUrl() { - SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService - .get(SQLiteDatabaseService.DB_NAME); - String dbUrl = sqLiteDatabaseService.getDatabaseUrl(testDbConfig); - - Assert.assertNotNull(dbUrl); - Assert.assertEquals(dbUrl, "jdbc:sqlite:extension_test_db.sqlite"); - } - - @Test - public void testGetConnection() throws DatabaseServiceException { - - SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService - .get(SQLiteDatabaseService.DB_NAME); - Connection conn = sqLiteDatabaseService.getConnection(testDbConfig); - - Assert.assertNotNull(conn); - } - - @Test - public void testTestConnection() throws DatabaseServiceException { - SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService - .get(SQLiteDatabaseService.DB_NAME); - - boolean result = sqLiteDatabaseService.testConnection(testDbConfig); - Assert.assertTrue(result); - } - - @Test - public void testConnect() throws DatabaseServiceException { - - SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService - .get(SQLiteDatabaseService.DB_NAME); - DatabaseInfo databaseInfo = sqLiteDatabaseService.connect(testDbConfig); - Assert.assertNotNull(databaseInfo); - } - - @Test - public void testExecuteQuery() throws DatabaseServiceException { - SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService - .get(SQLiteDatabaseService.DB_NAME); - DatabaseInfo databaseInfo = sqLiteDatabaseService.testQuery(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(databaseInfo); - } - - @Test - public void testBuildLimitQuery() { - SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService - .get(SQLiteDatabaseService.DB_NAME); - String limitQuery = sqliteSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable); - Assert.assertNotNull(limitQuery); - Assert.assertEquals(limitQuery, - "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";"); - } - - @Test - public void testGetRows() throws DatabaseServiceException { - SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService - .get(SQLiteDatabaseService.DB_NAME); - List dbRows = sqliteSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(dbRows); - } - - @Test - public void testGetInstance() { - SQLiteDatabaseService instance = SQLiteDatabaseService.getInstance(); - Assert.assertNotNull(instance); - } - - @Test - public void testGetColumns() throws DatabaseServiceException { - List dbColumns; - - SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService - .get(SQLiteDatabaseService.DB_NAME); - - dbColumns = sqliteSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable); - - Assert.assertNotNull(dbColumns); - } -} diff --git a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/stub/RefineDbServletStub.java b/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/stub/RefineDbServletStub.java deleted file mode 100644 index fd5f5a428..000000000 --- a/OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/stub/RefineDbServletStub.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - -Copyright 2010, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -package com.google.refine.extension.database.stub; - -import java.io.File; -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.google.refine.RefineServlet; -import com.google.refine.commands.Command; - -/** - * Exposes protected methods of com.google.refine.RefineServlet as public for unit testing - * - */ -public class RefineDbServletStub extends RefineServlet { - - private static File tempDir = null; - - // requirement of extending HttpServlet, not required for testing - private static final long serialVersionUID = 1L; - - public void wrapService(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - super.service(request, response); - } - - public String wrapGetCommandName(HttpServletRequest request) { - return super.getCommandKey(request); - } - - @Override - public File getTempDir() { - if (tempDir == null) { - try { - tempDir = File.createTempFile("refine-test-dir", ""); - tempDir.deleteOnExit(); - } catch (IOException e) { - throw new RuntimeException("Failed to create temp directory", e); - } - } - return tempDir; - } - - // -------------------helper methods-------------- - /** - * Helper method for inserting a mock object - * - * @param commandName - * @param command - */ - public void insertCommand(String commandName, Command command) { - registerOneCommand("core/" + commandName, command); - } - - /** - * Helper method for clearing up after testing - * - * @param commandName - */ - public void removeCommand(String commandName) { - unregisterCommand("core/" + commandName); - } -} diff --git a/OpenRefine/extensions/gdata/LICENSE.txt b/OpenRefine/extensions/gdata/LICENSE.txt deleted file mode 100644 index f6982420d..000000000 --- a/OpenRefine/extensions/gdata/LICENSE.txt +++ /dev/null @@ -1,75 +0,0 @@ -Copyright 2010 Thomas F. Morris - -Licensed under the terms of the Eclipse Public License v1.0 -http://www.eclipse.org/legal/epl-v10.html - -Eclipse Public License - v 1.0 - -THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - -1. DEFINITIONS - -"Contribution" means: - -a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and -b) in the case of each subsequent Contributor: -i) changes to the Program, and -ii) additions to the Program; -where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program. -"Contributor" means any person or entity that distributes the Program. - -"Licensed Patents" mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. - -"Program" means the Contributions distributed in accordance with this Agreement. - -"Recipient" means anyone who receives the Program under this Agreement, including all Contributors. - -2. GRANT OF RIGHTS - -a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form. -b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder. -c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient's responsibility to acquire that license before distributing the Program. -d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. -3. REQUIREMENTS - -A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that: - -a) it complies with the terms and conditions of this Agreement; and -b) its license agreement: -i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose; -ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits; -iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and -iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange. -When the Program is made available in source code form: - -a) it must be made available under this Agreement; and -b) a copy of this Agreement must be included with each copy of the Program. -Contributors may not remove or alter any copyright notices contained within the Program. - -Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution. - -4. COMMERCIAL DISTRIBUTION - -Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. - -For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor's responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. - -5. NO WARRANTY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. - -6. DISCLAIMER OF LIABILITY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. GENERAL - -If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. - -If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights granted under Section 2(b) shall terminate as of the date such litigation is filed. - -All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient's obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. - -Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. - -This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. \ No newline at end of file diff --git a/OpenRefine/extensions/gdata/README.txt b/OpenRefine/extensions/gdata/README.txt deleted file mode 100644 index c3ea8e9d1..000000000 --- a/OpenRefine/extensions/gdata/README.txt +++ /dev/null @@ -1,23 +0,0 @@ -OpenRefine extension for Google Spreadsheets -by Tom Morris - -This extension provides Refine with the ability to read Google spreadsheets -directly so you don't have to export them as CSV files first. - -Unpack the contents of this archive (jar, zip, etc) into the extensions folder -of your Refine installation and restart Refine. - -Enter the *public* share URL of a Google Spreadsheet in the "Data File URL" -field of the "Create a New Project" pane of the Refine home page. - -If you end up with a project full of HTML tags and an RSS feed, the extension -isn't installed properly. A properly installed extension should have an About -page available at http://127.0.0.1:3333/extension/gdata/ (the trailing -slash is significant). - - -Known limitations: - -- No write support -- No authentication support. You must use the public share URL for the - the Google spreadsheet you want to load. \ No newline at end of file diff --git a/OpenRefine/extensions/gdata/module/MOD-INF/controller.js b/OpenRefine/extensions/gdata/module/MOD-INF/controller.js deleted file mode 100644 index fc6ff960f..000000000 --- a/OpenRefine/extensions/gdata/module/MOD-INF/controller.js +++ /dev/null @@ -1,130 +0,0 @@ -/* - -Copyright 2010, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - */ - -/* - * Controller for GData extension. - * - * This is run in the Butterfly (ie Refine) server context using the Rhino - * Javascript interpreter. - */ - -var html = "text/html"; -var encoding = "UTF-8"; -var version = "0.3"; -var ClientSideResourceManager = Packages.com.google.refine.ClientSideResourceManager; - -/* - * Function invoked to initialize the extension. - */ -function init() { - var RS = Packages.com.google.refine.RefineServlet; - RS.registerCommand(module, "deauthorize", Packages.com.google.refine.extension.gdata.DeAuthorizeCommand()); - RS.registerCommand(module, "upload", Packages.com.google.refine.extension.gdata.UploadCommand()); - - // Register importer and exporter - var IM = Packages.com.google.refine.importing.ImportingManager; - - IM.registerController( - module, - "gdata-importing-controller", - new Packages.com.google.refine.extension.gdata.GDataImportingController() - ); - - // Script files to inject into /index page - ClientSideResourceManager.addPaths( - "index/scripts", - module, - [ - "scripts/gdata-extension.js", - "scripts/index/importing-controller.js", - "scripts/index/gdata-source-ui.js" - ] - ); - // Style files to inject into /index page - ClientSideResourceManager.addPaths( - "index/styles", - module, - [ - "styles/importing-controller.less" - ] - ); - - // Script files to inject into /project page - ClientSideResourceManager.addPaths( - "project/scripts", - module, - [ - "scripts/gdata-extension.js", - "scripts/project/exporters.js" - ] - ); -} - -/* - * Function invoked to handle each request in a custom way. - */ -function process(path, request, response) { - // Analyze path and handle this request yourself. - if (path == "authorize") { - var context = {}; - context.authorizationUrl = Packages.com.google.refine.extension.gdata.GoogleAPIExtension.getAuthorizationUrl(module, request); - - send(request, response, "authorize.vt", context); - } else if (path == "authorized") { - var context = {}; - context.state = request.getParameter("state"); - - (function() { - if (Packages.com.google.refine.extension.gdata.TokenCookie.getToken(request) !== null) { - return; - } - var tokenAndExpiresInSeconds = Packages.com.google.refine.extension.gdata.GoogleAPIExtension.getTokenFromCode(module,request); - if (tokenAndExpiresInSeconds) { - var tokenInfo = tokenAndExpiresInSeconds.split(","); - Packages.com.google.refine.extension.gdata.TokenCookie.setToken(request, response, tokenInfo[0], tokenInfo[1]); - return; - } - Packages.com.google.refine.extension.gdata.TokenCookie.deleteToken(request, response); - })(); - - send(request, response, "authorized.vt", context); - } else if (path == "/" || path == "") { - var context = {}; - context.version = version; - send(request, response, "index.vt", context); - } -} - -function send(request, response, template, context) { - butterfly.sendTextFromTemplate(request, response, context, template, encoding, html); -} diff --git a/OpenRefine/extensions/gdata/module/MOD-INF/module.properties b/OpenRefine/extensions/gdata/module/MOD-INF/module.properties deleted file mode 100644 index 49c1bf4a8..000000000 --- a/OpenRefine/extensions/gdata/module/MOD-INF/module.properties +++ /dev/null @@ -1,4 +0,0 @@ -name = gdata -description = Google Data (gData) importer/exporter for OpenRefine -templating.macros = macros.vm -requires = core diff --git a/OpenRefine/extensions/gdata/module/authorize.vt b/OpenRefine/extensions/gdata/module/authorize.vt deleted file mode 100644 index dbb26a585..000000000 --- a/OpenRefine/extensions/gdata/module/authorize.vt +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - - diff --git a/OpenRefine/extensions/gdata/module/authorized.vt b/OpenRefine/extensions/gdata/module/authorized.vt deleted file mode 100644 index 67ed3f571..000000000 --- a/OpenRefine/extensions/gdata/module/authorized.vt +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - OpenRefine - Authorized - - - - - - - diff --git a/OpenRefine/extensions/gdata/module/index.vt b/OpenRefine/extensions/gdata/module/index.vt deleted file mode 100644 index 3b0559794..000000000 --- a/OpenRefine/extensions/gdata/module/index.vt +++ /dev/null @@ -1,22 +0,0 @@ -#* - * Access this page at the URL /extension/gdata/ - *# - - - Google Data (gData) Extension v$version - - -

      Google Data Extension v$version

      -

      by Tom Morris

      -

      The Google Spreadsheet/Google Data extension allows OpenRefine to read - (and write, soon) Google Spreadsheets. It should be easily adaptable to - other data sources which implement the gData protocol such as Google Base. -

      -

      Known Limitations

      -
        -
      • No write support
      • -
      • No authentication support. You must use the public share URL for the - the Google spreadsheet you want to load.
      • -
      - - \ No newline at end of file diff --git a/OpenRefine/extensions/gdata/module/langs/translation-bn.json b/OpenRefine/extensions/gdata/module/langs/translation-bn.json deleted file mode 100644 index d7e23aa6a..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-bn.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "gdata-import/next->": "পরবর্তী »", - "gdata-exporter/uploading": "আপলোড চলছে…", - "gdata-exporter/upload-error": "আপলোড ত্রুটি: ", - "gdata-exporter/upload-success": "প্রকল্প সফলভাবে গুগল ড্রাইভে আপলোড করা হয়েছে ", - "gdata-exporter/new-spreadsheet": "একটি নতুন গুগল স্প্রেডশীট", - "gdata-exporter/enter-spreadsheet": "নতুন গুগল স্প্রেডশীটের নাম দিন", - "gdata-parsing/option": "অপশনস", - "gdata-parsing/worksheet": "ওয়ার্কশিটসমূহ", - "gdata-parsing/updating-preview": "প্রিভিউ হালনাগাদ করা হচ্ছে …", - "gdata-import/import-by-url": "একটি পাবলিক গুগল স্প্রেডশিট ইম্পোর্ট করুন ইউ আর এল :", - "gdata-parsing/create-proj": "প্রজেক্ট তৈরি করুন »", - "gdata-parsing/proj-name": "প্রজেক্ট নাম", - "gdata-parsing/conf-pars": "পার্সিং অপশন কনফিগার করুন", - "gdata-parsing/start-over": "« নতুন করে শুরু করুন", - "gdata-import/re-sign-in-another": "অন্য অ্যাকাউন্ট দিয়ে সাইন ইন করুন", - "gdata-import/retrieving": "গুগল ডকুমেন্টস পুনরুদ্ধার করা হচ্ছে …", - "gdata-import/sign-out": "সাইন আউট", - "gdata-import/sign-in": "সাইন ইন", - "gdata-import/please-signin": "দয়া করে সাইন ইন করুন এবং অথরাইজ করুন আপনার গুগল ডেটা অ্যাক্সেস দেয়ার জন্য", - "gdata-import/auth-doc": "অনুমোদিত ডকুমেন্টস", - "gdata-import/title": "পাবলিক ডকুমেন্টস", - "gdata-import/creating": "প্রজেক্ট তৈরি করা হচ্ছে …", - "gdata-import/preparing": "প্রস্তুত করা হচ্ছে …" -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-bn_IN.json b/OpenRefine/extensions/gdata/module/langs/translation-bn_IN.json deleted file mode 100644 index 0967ef424..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-bn_IN.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-cs.json b/OpenRefine/extensions/gdata/module/langs/translation-cs.json deleted file mode 100644 index e06c9e151..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-cs.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "gdata-auth/authorized-label": "Proces autorizace dokončen. Zavřete toto okno pro návrat do OpenRefine.", - "gdata-auth/authorize-label": "OpenRefine – autorizace", - "gdata-exporter/enter-filename": "Zadejte název souboru pro export projektu na Disk Google", - "gdata-exporter/upload-google-sheets-success": "Projekt se nahrál na Disk Google ", - "gdata-exporter/enter-spreadsheet": "Zadejte název pro tabulku Google", - "gdata-exporter/new-spreadsheet": "Nová tabulka Google", - "gdata-exporter/upload-google-drive-success": "Projekt se nahrál na Disk Google ", - "gdata-exporter/upload-error": "Chyba nahrávání: ", - "gdata-exporter/uploading": "Nahrávání…", - "gdata-exporter/google-sheets": "Tabulky Google", - "gdata-exporter/export-to-google-drive": "Zálohovat projekt OpenRefine na Disk Google…", - "gdata-source/updated": "Aktualizováno", - "gdata-source/authors": "Autoři", - "gdata-source/title": "Titul", - "gdata-source/type": "Typ", - "gdata-source/alert-url": "Musíte určit webovou adresu (URL) pro import.", - "gdata-parsing/store-cell": "Uložit prádné buňky jako null", - "gdata-parsing/store-row": "Uložit prázdné řádky", - "gdata-parsing/limit": "řádek/řádky/řádků dat", - "gdata-parsing/limit-next": "Nahrát nejvýše", - "gdata-parsing/discard": "řádek/řádky/řádků dat", - "gdata-parsing/discard-next": "Vynechat počáteční(ch)", - "gdata-parsing/parse": "řádek/řádky jako hlavičky sloupců", - "gdata-parsing/parse-next": "Analyzuj další(ch)", - "gdata-parsing/ignore": "řádek/řádky/řádků na začátku souboru", - "gdata-parsing/ignore-first": "Vynechat počáteční(ch)", - "gdata-parsing/preview-button": "Aktualizovat náhled", - "gdata-parsing/option": "Možnosti", - "gdata-parsing/worksheet": "Pracovní listy", - "gdata-parsing/updating-preview": "Aktualizace náhledu…", - "gdata-parsing/create-proj": "Vytvořit projekt »", - "gdata-parsing/proj-name": "Název projektu", - "gdata-parsing/conf-pars": "Nastavit možnosti parsování", - "gdata-parsing/start-over": "« Začít znovu", - "gdata-import/re-sign-in-another": "Přihlásit se pod jiným účtem", - "gdata-import/retrieving": "Načítání dokumentů z Google Docs", - "gdata-import/sign-out": "Odhlásit se", - "gdata-import/sign-in": "Přihlášit se", - "gdata-import/please-signin": "Přihlaste se a autorizuje přístup k vašim datům na Googlu", - "gdata-import/auth-doc": "Zabezpečené dokumenty", - "gdata-import/next->": "Další »", - "gdata-import/import-by-url": "Importovat veřejný sešit Googlu pomocí URL:", - "gdata-import/title": "Veřejné dokumenty", - "gdata-import/creating": "Vytváření projektu…", - "gdata-import/preparing": "Příprava…" -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-en.json b/OpenRefine/extensions/gdata/module/langs/translation-en.json deleted file mode 100644 index 9b98bdc4d..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-en.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "gdata-import/preparing": "Preparing …", - "gdata-import/creating": "Creating project …", - "gdata-import/title": "Public Documents", - "gdata-import/import-by-url": "Import a public Google Spreadsheet by its URL:", - "gdata-import/next->": "Next »", - "gdata-import/auth-doc": "Authorized Documents", - "gdata-import/please-signin": "Please sign in and authorize to access your Google data", - "gdata-import/sign-in": "Sign in", - "gdata-import/sign-out": "Sign out", - "gdata-import/retrieving": "Retrieving Google Docs documents …", - "gdata-import/re-sign-in-another": "Sign in with different account", - "gdata-parsing/start-over": "« Start Over", - "gdata-parsing/conf-pars": "Configure Parsing Options", - "gdata-parsing/proj-name": "Project name", - "gdata-parsing/create-proj": "Create Project »", - "gdata-parsing/updating-preview": "Updating preview …", - "gdata-parsing/worksheet": "Worksheets", - "gdata-parsing/option": "Options", - "gdata-parsing/preview-button": "Update preview", - "gdata-parsing/disable-auto-preview": "Disable auto preview", - "gdata-parsing/ignore-first": "Ignore first", - "gdata-parsing/ignore": "line(s) at beginning of file", - "gdata-parsing/parse-next": "Parse next", - "gdata-parsing/parse": "line(s) as column headers", - "gdata-parsing/discard-next": "Discard initial", - "gdata-parsing/discard": "row(s) of data", - "gdata-parsing/limit-next": "Load at most", - "gdata-parsing/limit": "row(s) of data", - "gdata-parsing/store-row": "Store blank rows", - "gdata-parsing/store-cell": "Store blank cells as nulls", - "gdata-source/alert-url": "You must specify a web address (URL) to import.", - "gdata-source/type": "Type", - "gdata-source/title": "Title", - "gdata-source/authors": "Authors", - "gdata-source/updated": "Updated", - "gdata-exporter/export-to-google-drive": "OpenRefine project archive to Google Drive…", - "gdata-exporter/google-sheets": "Google Sheets", - "gdata-exporter/uploading": "Uploading…", - "gdata-exporter/upload-error": "Upload error: ", - "gdata-exporter/upload-google-drive-success": "Project uploaded to Google Drive ", - "gdata-exporter/new-spreadsheet": "A new Google spreadsheet", - "gdata-exporter/enter-spreadsheet": "Enter a name for the new Google spreadsheet", - "gdata-exporter/upload-google-sheets-success": "Project uploaded to Google Drive ", - "gdata-exporter/enter-filename": "Enter a filename for the project export in Google Drive", - "gdata-auth/authorize-label": "OpenRefine - Authorization", - "gdata-auth/authorized-label": "Authorization process completed. Close this window and return to OpenRefine." -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-en_GB.json b/OpenRefine/extensions/gdata/module/langs/translation-en_GB.json deleted file mode 100644 index f43a17e8f..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-en_GB.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "gdata-import/auth-doc": "Authorised Documents", - "gdata-import/sign-in": "sign in and authorise", - "gdata-auth/authorize-label": "OpenRefine - Authorisation", - "gdata-auth/authorized-label": "Authorisation process completed. Close this window and return to OpenRefine." -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-es.json b/OpenRefine/extensions/gdata/module/langs/translation-es.json deleted file mode 100644 index bbbdcbd7f..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-es.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "gdata-auth/authorized-label": "Se completó el proceso de autorización. Cierre esta ventana y vuelva a OpenRefine.", - "gdata-auth/authorize-label": "OpenRefine • Autorización", - "gdata-exporter/enter-filename": "Escriba un nombre de archivo para la exportación del proyecto en Google Drive", - "gdata-exporter/upload-google-sheets-success": "Se cargó el proyecto en Google Drive ", - "gdata-exporter/enter-spreadsheet": "Escriba un nombre para la hoja de Google nueva", - "gdata-exporter/new-spreadsheet": "Una hoja de cálculo de Google nueva", - "gdata-exporter/upload-google-drive-success": "Se cargó el proyecto en Google Drive ", - "gdata-exporter/upload-error": "Error de carga: ", - "gdata-exporter/uploading": "Cargando…", - "gdata-exporter/google-sheets": "Google Sheets", - "gdata-exporter/export-to-google-drive": "Archivador de proyecto de OpenRefine a Google Drive…", - "gdata-source/updated": "Actualización el", - "gdata-source/authors": "Autores", - "gdata-source/title": "Título", - "gdata-source/type": "Tipo", - "gdata-source/alert-url": "Debe especificar una dirección web (URL) que importar.", - "gdata-parsing/store-cell": "Almacenar celdas vacías como nulos", - "gdata-parsing/store-row": "Almacenar filas vacías", - "gdata-parsing/limit": "filas de datos", - "gdata-parsing/limit-next": "Cargar como máximo", - "gdata-parsing/discard": "filas de datos iniciales", - "gdata-parsing/discard-next": "Descartar", - "gdata-parsing/parse": "renglones como cabeceras de columnas", - "gdata-parsing/parse-next": "Analizar próximos", - "gdata-parsing/ignore": "renglones al principio del archivo", - "gdata-parsing/ignore-first": "Ignorar primeros", - "gdata-parsing/preview-button": "Actualizar previsualización", - "gdata-parsing/option": "Opciones", - "gdata-parsing/worksheet": "Hojas de trabajo", - "gdata-parsing/updating-preview": "Actualizando previsualización…", - "gdata-parsing/create-proj": "Crear proyecto →", - "gdata-parsing/proj-name": "Nombre del proyecto", - "gdata-parsing/conf-pars": "Configurar opciones de análisis", - "gdata-parsing/start-over": "← Comenzar de cero", - "gdata-import/another-account": "con otra cuenta", - "gdata-import/re-sign-in": "acceder de nuevo", - "gdata-import/retrieving": "Recuperando los documentos de Google Docs…", - "gdata-import/access-data": "el acceso a sus datos de Google.", - "gdata-import/sign-out": "Finalizar sesión", - "gdata-import/sign-in": "Ingrese", - "gdata-import/please": "Rogamos que", - "gdata-import/auth-doc": "Documentos autorizados", - "gdata-import/next->": "Siguiente →", - "gdata-import/import-by-url": "Importe una hoja de cálculo pública de Google mediante su URL:", - "gdata-import/title": "Documentos públicos", - "gdata-import/creating": "Creando proyecto…", - "gdata-import/preparing": "Preparando…", - "gdata-import/re-sign-in-another": "Iniciar sesión con una cuenta diferente", - "gdata-import/please-signin": "Inicie sesión y autorice el acceso a sus datos de Google" -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-fr.json b/OpenRefine/extensions/gdata/module/langs/translation-fr.json deleted file mode 100644 index bffb8f794..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-fr.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "gdata-import/preparing": "Préparation…", - "gdata-import/creating": "Création du projet…", - "gdata-import/title": "Documents publics", - "gdata-import/import-by-url": "Importer un tableau public Google Spreadsheet par son URL :", - "gdata-import/next->": "Suivant »", - "gdata-import/auth-doc": "Documents autorisés", - "gdata-import/please": "Merci", - "gdata-import/sign-in": "Se connecter", - "gdata-import/sign-out": "Se déconnecter", - "gdata-import/access-data": "l’accès à vos données Google.", - "gdata-import/retrieving": "Récupération des documents de Google Docs…", - "gdata-import/re-sign-in": "se reconnecter", - "gdata-import/another-account": "avec un autre compte", - "gdata-parsing/start-over": "« Recommencer", - "gdata-parsing/conf-pars": "Configurer les options d’analyse", - "gdata-parsing/proj-name": "Nom du projet", - "gdata-parsing/create-proj": "Créer un projet »", - "gdata-parsing/updating-preview": "Mise à jour de l’aperçu…", - "gdata-parsing/worksheet": "Feuilles", - "gdata-parsing/option": "Options", - "gdata-parsing/preview-button": "Mettre à jour l’aperçu", - "gdata-parsing/ignore-first": "Ignorer la ou les premières", - "gdata-parsing/ignore": "ligne(s) du début du fichier", - "gdata-parsing/parse-next": "Analyser la ou les", - "gdata-parsing/parse": "ligne(s) suivante(s) comme entêtes de colonnes", - "gdata-parsing/discard-next": "Ignorer la ou les", - "gdata-parsing/discard": "première(s) ligne(s) de données", - "gdata-parsing/limit-next": "Charger au plus", - "gdata-parsing/limit": "ligne(s) de données", - "gdata-parsing/store-row": "Conserver les lignes vides", - "gdata-parsing/store-cell": "Enregistrer les cellules vides comme des valeurs nulles", - "gdata-source/alert-url": "Vous devez indiquer une adresse web (URL) à importer.", - "gdata-source/type": "Type", - "gdata-source/title": "Titre", - "gdata-source/authors": "Auteurs", - "gdata-source/updated": "Mis à jour", - "gdata-exporter/uploading": "Téléversement…", - "gdata-exporter/upload-error": "Erreur de téléversement : ", - "gdata-exporter/new-spreadsheet": "Un nouveau tableau Google Spreadsheet", - "gdata-exporter/enter-spreadsheet": "Indiquer un nom pour le nouveau tableau Google Spreadsheet", - "gdata-exporter/upload-success": "Le projet a été chargé avec succès dans Google Drive avec l'identifiant ", - "gdata-auth/authorize-label": "OpenRefine - Autorisation", - "gdata-auth/authorized-label": "Procédure d’autorisation terminée. Fermer cette fenêtre et revenir à OpenRefine.", - "gdata-exporter/enter-filename": "Entrez un nom de fichier pour l'export du projet dans Google Drive", - "gdata-exporter/upload-google-sheets-success": "Projet téléversé sur Google Drive ", - "gdata-exporter/upload-google-drive-success": "Projet téléversé sur Google Drive ", - "gdata-exporter/google-sheets": "Google Sheets", - "gdata-import/re-sign-in-another": "Connectez-vous avec un autre compte", - "gdata-import/please-signin": "Veuillez vous connecter et autoriser l'accès à vos données Google", - "gdata-exporter/export-to-google-drive": "Projet OpenRefine archivé dans Google Drive…", - "gdata-parsing/disable-auto-preview": "Désactiver l'aperçu automatique" -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-he.json b/OpenRefine/extensions/gdata/module/langs/translation-he.json deleted file mode 100644 index 670117a08..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-he.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "gdata-import/preparing": "בהכנות…", - "gdata-import/creating": "המיזם נוצר…", - "gdata-import/title": "מסמכים ציבוריים", - "gdata-import/import-by-url": "ייבוא גיליון נתונים ציבורי מ־Google לפי הכתובת שלו:", - "gdata-import/next->": "הבא «", - "gdata-import/auth-doc": "מסמכים מאושרים", - "gdata-import/please": "בבקשה", - "gdata-import/sign-in": "כניסה", - "gdata-import/sign-out": "יציאה", - "gdata-import/access-data": "גישה לנתונים שלך ב־Google.", - "gdata-import/retrieving": "מתקבלים מסמכים מ־Google Docs…", - "gdata-import/re-sign-in": "להיכנס מחדש", - "gdata-import/another-account": "עם חשבון אחר", - "gdata-parsing/start-over": "» להתחיל מההתחלה", - "gdata-parsing/conf-pars": "הגדרת אפשרויות פענוח", - "gdata-parsing/proj-name": "שם המיזם", - "gdata-parsing/create-proj": "יצירת מיזם «", - "gdata-parsing/updating-preview": "התצוגה המקדימה מתעדכנת…", - "gdata-parsing/worksheet": "גיליונות נתונים", - "gdata-parsing/option": "אפשרויות", - "gdata-parsing/preview-button": "עדכון תצוגה מקדימה", - "gdata-parsing/ignore-first": "להתעלם מ־", - "gdata-parsing/ignore": "השורות הראשונות בתחילת הקובץ", - "gdata-parsing/parse-next": "לפענח את", - "gdata-parsing/parse": "השורות הבאות ככותרות לעמודות", - "gdata-parsing/discard-next": "התעלמות מ־", - "gdata-parsing/discard": "שורות הנתונים הראשונות", - "gdata-parsing/limit-next": "לטעון", - "gdata-parsing/limit": "שורות נתונים לכל היותר", - "gdata-parsing/store-row": "לאחסן שורות ריקות", - "gdata-parsing/store-cell": "לאחסן תאים ריקים כ־null (ריק)", - "gdata-source/alert-url": "עליך לציין כתובת (URL) לייבוא.", - "gdata-source/type": "סוג", - "gdata-source/title": "כותרת", - "gdata-source/authors": "מחברים", - "gdata-source/updated": "מעודכן", - "gdata-exporter/uploading": "מתבצעת העלאה…", - "gdata-exporter/upload-error": "שגיאת העלאה: ", - "gdata-exporter/new-spreadsheet": "גיליון נתונים חדש ב־Google", - "gdata-exporter/enter-spreadsheet": "נא למלא שם לגיליון הנתונים החדש ב־Google", - "gdata-auth/authorize-label": "OpenRefine - אישור", - "gdata-auth/authorized-label": "תהליך האישור הסתיים. יש לסגור את החלון ולחזור ל־OpenRefine.", - "gdata-exporter/enter-filename": "נא למלא שם קובץ לייצוא המיזם ב־Google Drive", - "gdata-exporter/upload-google-sheets-success": "המיזם הועלה ל־Google Drive ", - "gdata-exporter/upload-google-drive-success": "המיזם הועלה ל־Google Drive ", - "gdata-exporter/google-sheets": "Google Sheets", - "gdata-exporter/export-to-google-drive": "מיזם OpenRefine שומר בארכיון ל־Google Drive…", - "gdata-import/re-sign-in-another": "כניסה עם חשבון אחר", - "gdata-import/please-signin": "נא להיכנס כדי לאשר את הגישה לנתונים שלך ב־Google", - "gdata-parsing/disable-auto-preview": "השבתת תצוגה מקדימה אוטומטית" -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-it.json b/OpenRefine/extensions/gdata/module/langs/translation-it.json deleted file mode 100644 index 6a57fdd8e..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-it.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "gdata-import/preparing": "In preparazione…", - "gdata-import/creating": "Sto creando il progetto…", - "gdata-import/title": "Documenti Pubblici", - "gdata-import/import-by-url": "Importa un Google Spreadsheet pubblico inserendo l'URL:", - "gdata-import/next->": "Avanti »", - "gdata-import/auth-doc": "Documenti Autorizzati", - "gdata-import/please": "Per piacere", - "gdata-import/sign-in": "autenticati ed autorizza", - "gdata-import/sign-out": "esci", - "gdata-import/access-data": "accedi ai tuoi dati Google.", - "gdata-import/retrieving": "Sto recuperando i documenti da Google Docs…", - "gdata-import/re-sign-in": "rieffettua l'autenticazione", - "gdata-import/another-account": "con un altro account", - "gdata-parsing/start-over": "« Ricomincia", - "gdata-parsing/conf-pars": "Configura le opzioni per il parsing", - "gdata-parsing/proj-name": "Nome del progetto", - "gdata-parsing/create-proj": "Crea un progetto »", - "gdata-parsing/updating-preview": "Aggiornamento anteprima…", - "gdata-parsing/worksheet": "Fogli di lavoro", - "gdata-parsing/option": "Opzioni", - "gdata-parsing/preview-button": "Aggiorna la Preview", - "gdata-parsing/ignore-first": "Ignora le prime", - "gdata-parsing/ignore": "linee all'inizio del file", - "gdata-parsing/parse-next": "Parsa le prossime", - "gdata-parsing/parse": "linee come nomi delle colonne", - "gdata-parsing/discard-next": "Scarta le prime", - "gdata-parsing/discard": "righe di dati", - "gdata-parsing/limit-next": "Carica al massimo", - "gdata-parsing/limit": "righe di dati", - "gdata-parsing/store-row": "Salva righe vuote", - "gdata-parsing/store-cell": "Salva le celle vuote come 'null'", - "gdata-source/alert-url": "Devi specificare un indirizzo web (URL) per l'import.", - "gdata-source/type": "Tipo", - "gdata-source/title": "Titolo", - "gdata-source/authors": "Autori", - "gdata-source/updated": "Aggiornato", - "gdata-exporter/uploading": "Caricando…", - "gdata-exporter/upload-error": "Errore durante il caricamento: ", - "gdata-exporter/new-spreadsheet": "Un nuovo Google spreadsheet", - "gdata-exporter/enter-spreadsheet": "Inserisci un nome per il nuovo Google spreadsheet", - "gdata-auth/authorize-label": "OpenRefine - Autorizzazione", - "gdata-auth/authorized-label": "Processo di autorizzazione completato. Chiudi questa finestra e torna ad OpenRefine.", - "gdata-exporter/upload-success": "Il progetto è stato caricato con successo su Google Drive ", - "gdata-exporter/enter-filename": "Inserisci un nome del file per esportare il progetto su Google Drive", - "gdata-exporter/upload-google-sheets-success": "Progetto caricato su Google Drive ", - "gdata-exporter/export-to-google-drive": "Archiviando progetto di OpenRefine su Google Drive…", - "gdata-exporter/upload-google-drive-success": "Progetto caricato su Google Drive ", - "gdata-exporter/google-sheets": "Google Sheets" -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-jp.json b/OpenRefine/extensions/gdata/module/langs/translation-jp.json deleted file mode 100644 index 4c4afcfcb..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-jp.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "gdata-import/preparing": "準備中…", - "gdata-import/creating": "プロジェクトを作る…", - "gdata-import/title": "一般公開文書(public document)", - "gdata-import/import-by-url": "一般公開している(public)Google SpreadsheetのURLを入力してください:", - "gdata-import/next->": "次へ »", - "gdata-import/auth-doc": "制限付き文書(authorized document)", - "gdata-import/please": "まず", - "gdata-import/sign-in": "サインイン", - "gdata-import/sign-out": "サインアウト", - "gdata-import/access-data": "してから、あなたのGoogle文書にアクセスします.", - "gdata-import/retrieving": "Google Docsを取り込んでいます…", - "gdata-import/re-sign-in": "サインし直す", - "gdata-import/another-account": "別のアカウント", - "gdata-parsing/start-over": "« やり直し", - "gdata-parsing/conf-pars": "解析オプションの設定", - "gdata-parsing/proj-name": "プロジェクト名", - "gdata-parsing/create-proj": "プロジェクトを作る »", - "gdata-parsing/updating-preview": "プレビューを更新中…", - "gdata-parsing/worksheet": "シート", - "gdata-parsing/option": "オプション", - "gdata-parsing/preview-button": "プレビューを更新", - "gdata-parsing/ignore-first": "先頭を無視", - "gdata-parsing/ignore": "行分", - "gdata-parsing/parse-next": "カラム名として解釈", - "gdata-parsing/parse": "行分", - "gdata-parsing/discard-next": "先頭データを破棄", - "gdata-parsing/discard": "行分", - "gdata-parsing/limit-next": "最大読み込み行数", - "gdata-parsing/limit": "行分", - "gdata-parsing/store-row": "空白行も保存", - "gdata-parsing/store-cell": "nullとして空白セルを保存", - "gdata-source/alert-url": "インポートするアドレス(URL)を入力してください.", - "gdata-source/type": "形式", - "gdata-source/title": "題名", - "gdata-source/authors": "著者", - "gdata-source/updated": "更新済み", - "gdata-exporter/uploading": "アップロード中…", - "gdata-exporter/upload-error": "アップロードでエラー : ", - "gdata-exporter/new-spreadsheet": "新しいGoogle spreadsheet", - "gdata-exporter/enter-spreadsheet": "新しいGoogle spreadsheetの題名を入力してください", - "gdata-exporter/upload-success": "プロジェクトがGoogle Driveにアップロードされました ", - "gdata-auth/authorize-label": "OpenRefine - 認証", - "gdata-auth/authorized-label": "認証が完了しました。このウインドウを閉じてOpenRefineで作業できます.", - "gdata-exporter/enter-filename": "Google Driveにエクスポートするファイル名を入力", - "gdata-exporter/google-sheets": "Googleシート", - "gdata-exporter/upload-google-sheets-success": "Google Driveにアップロードされたプロジェクト ", - "gdata-exporter/upload-google-drive-success": "Google Driveにアップロードされたプロジェクト ", - "gdata-exporter/export-to-google-drive": "プロジェクトをGoogle Driveへ…", - "gdata-import/re-sign-in-another": "別のアカウントでサインイン", - "gdata-import/please-signin": "サインしてGoogleのアクセス権限を認証してください" -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-ko.json b/OpenRefine/extensions/gdata/module/langs/translation-ko.json deleted file mode 100644 index fec6dc8fb..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-ko.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "gdata-import/preparing": "준비중 ...", - "gdata-import/creating": "프로젝트 생성중 ...", - "gdata-import/title": "공개 문서", - "gdata-import/import-by-url": "다음 URL을 가지고 public Google Spreadsheet 또는 Fusion Table 을 임포트하세요:", - "gdata-import/next->": "다음 »", - "gdata-import/auth-doc": "검증된 문서", - "gdata-import/please": "해주세요", - "gdata-import/sign-in": "서명 및 검증해주세요", - "gdata-import/sign-out": "로그아웃", - "gdata-import/access-data": "당신의 Google 데이터에 접근하세요.", - "gdata-import/retrieving": "Google Docs 문서를 가져오는중 ...", - "gdata-import/re-sign-in": "재 로그인", - "gdata-import/another-account": "다른 계정으로", - "gdata-parsing/start-over": "« 다시 시작", - "gdata-parsing/conf-pars": "파싱 옵션 설정", - "gdata-parsing/proj-name": "프로젝트 이름", - "gdata-parsing/create-proj": "프로젝트 생성 »", - "gdata-parsing/updating-preview": "미리보기 업데이트중 ...", - "gdata-parsing/worksheet": "워크시트", - "gdata-parsing/option": "옵션", - "gdata-parsing/preview-button": "업데이트 미리보기", - "gdata-parsing/ignore-first": "처음을 무시", - "gdata-parsing/ignore": "파일의 첫번째 줄(들)", - "gdata-parsing/parse-next": "다음 파싱", - "gdata-parsing/parse": "컬럼 헤더로 라인(들)을", - "gdata-parsing/discard-next": "초기값을 무시", - "gdata-parsing/discard": "데이터 줄수", - "gdata-parsing/limit-next": "최대한 로드하세요", - "gdata-parsing/limit": "데이터 줄수", - "gdata-parsing/store-row": "빈 줄을 저장하세요", - "gdata-parsing/store-cell": "빈 셀을 null로 저장하세요", - "gdata-source/alert-url": "임포트할 웹주소(URL)을 지정해야 합니다.", - "gdata-source/type": "타입", - "gdata-source/title": "타이틀", - "gdata-source/authors": "저자", - "gdata-source/updated": "업데이트", - "gdata-exporter/uploading": "업데이트중...", - "gdata-exporter/upload-error": "업로드 에러: ", - "gdata-exporter/upload-success": "다음 Id로 Google Drive에 프로젝트가 성공적으로 업로드 되었습니다 ", - "gdata-exporter/new-spreadsheet": "새로운 Google speadsheet", - "gdata-exporter/enter-spreadsheet": "새로운 Google Spreadsheet 이름을 입력하세요", - "gdata-auth/authorize-label": "OpenRefine - 검증", - "gdata-auth/authorized-label": "검증 프로세스 완료. 이 창을 닫고 OpenRefine으로 돌아가세요." -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-nb_NO.json b/OpenRefine/extensions/gdata/module/langs/translation-nb_NO.json deleted file mode 100644 index 1da4463d2..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-nb_NO.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "gdata-import/preparing": "Forbereder …", - "gdata-import/creating": "Oppretter prosjekt …", - "gdata-import/title": "Offentlige dokumenter", - "gdata-import/import-by-url": "Importer et offentlig Google-regneark fra dens nettadresse:", - "gdata-import/next->": "Neste »", - "gdata-import/auth-doc": "Godkjente dokumenter", - "gdata-import/sign-in": "Logg inn og godkjenn", - "gdata-import/sign-out": "Logg ut", - "gdata-import/access-data": "Tilgang til din Google-data.", - "gdata-import/retrieving": "Henter Google Docs-dokumenter …", - "gdata-import/re-sign-in": "Logg inn igjen", - "gdata-import/another-account": "med en annen konto", - "gdata-import/please": "Vær så snill å", - "gdata-parsing/start-over": "« start om", - "gdata-parsing/conf-pars": "Sett opp fortolkningsvalg", - "gdata-parsing/proj-name": "Prosjektnavn", - "gdata-parsing/create-proj": "Opprett prosjekt »", - "gdata-parsing/updating-preview": "Oppdaterer forhåndsvisning …", - "gdata-parsing/worksheet": "Arbeidsdokumenter", - "gdata-parsing/option": "Alternativ", - "gdata-parsing/preview-button": "Oppdater forhåndsvisning", - "gdata-parsing/ignore-first": "Ignorer første", - "gdata-parsing/ignore": "linje(ne) i begynnelsen av filen", - "gdata-parsing/parse-next": "Fortolk neste", - "gdata-parsing/parse": "line(r) som kolonnerubrikker", - "gdata-parsing/discard-next": "Forkast innledende", - "gdata-parsing/discard": "rad(er) med data", - "gdata-parsing/limit-next": "Last inn høyst", - "gdata-parsing/limit": "rad(er) med data", - "gdata-parsing/store-row": "Lagre blanke rader", - "gdata-parsing/store-cell": "Lagre blanke celler som null", - "gdata-source/alert-url": "Du må angi en nettadresse å importere.", - "gdata-source/type": "Type", - "gdata-source/title": "Tittel", - "gdata-source/authors": "Forfattere", - "gdata-source/updated": "Oppdatert", - "gdata-exporter/uploading": "Laster opp …", - "gdata-exporter/upload-error": "Opplastingsfeil: ", - "gdata-exporter/new-spreadsheet": "Nytt Google-regneark", - "gdata-exporter/enter-spreadsheet": "Navngi det nye Google-regnearket", - "gdata-auth/authorize-label": "OpenRefine - Identitetsbekreftelse", - "gdata-auth/authorized-label": "Identitet bekreftet. Lukk dette vinduet og returner til OpenRefine.", - "gdata-exporter/enter-filename": "Skriv inn et filnavn for prosjekteksporten i Google Drive", - "gdata-exporter/export-to-google-drive": "OpenRefine-prosjekt arkivert til Google Drive…", - "gdata-exporter/google-sheets": "Google Sheets", - "gdata-exporter/upload-google-drive-success": "Prosjekt opplastet til Google Drive ", - "gdata-exporter/upload-google-sheets-success": "Prosjekt opplastet til Google Drive " -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-pl.json b/OpenRefine/extensions/gdata/module/langs/translation-pl.json deleted file mode 100644 index 46ceff747..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-pl.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "gdata-parsing/start-over": "«Zacznij od nowa", - "gdata-import/re-sign-in-another": "Zaloguj się za pomocą innego konta", - "gdata-parsing/create-proj": "Utwórz projekt »", - "gdata-parsing/updating-preview": "Aktualizacja podglądu …", - "gdata-import/title": "Publiczne dokumenty", - "gdata-import/next->": "Następny»", - "gdata-import/auth-doc": "Autoryzowane Dokumenty", - "gdata-import/please-signin": "Proszę zaloguj się i autoryzuj, aby otrzymać dostęp do twoich danych Google", - "gdata-import/preparing": "Przygotowanie…", - "gdata-import/creating": "Tworzenie projektu…", - "gdata-import/sign-in": "Zaloguj się", - "gdata-import/sign-out": "Wyloguj się", - "gdata-import/retrieving": "Pobieranie dokumentów Google Docs …", - "gdata-parsing/ignore-first": "Najpierw zignoruj", - "gdata-parsing/conf-pars": "Konfiguracja opcji parsowania", - "gdata-parsing/option": "Opcje" -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-pt.json b/OpenRefine/extensions/gdata/module/langs/translation-pt.json deleted file mode 100644 index 4b998c9d2..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-pt.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "gdata-source/title": "Título", - "gdata-exporter/upload-error": "Erro no carregamento: ", - "gdata-import/sign-out": "Sair", - "gdata-exporter/enter-spreadsheet": "Digite um nome para a nova planilha do Google", - "gdata-import/retrieving": "Recuperando documentos do Google Docs …", - "gdata-auth/authorized-label": "Processo de autorização concluído. Feche esta janela e retorne ao OpenRefine.", - "gdata-exporter/enter-filename": "Digite um nome de ficheiro para exportar o projeto no Google Drive", - "gdata-source/alert-url": "Deve especificar um endereço web (URL) para importar.", - "gdata-parsing/conf-pars": "Configurar opções de análise", - "gdata-import/another-account": "com outra conta", - "gdata-parsing/start-over": "« Recomeçar", - "gdata-parsing/ignore": "linhas(s) no começo do ficheiro", - "gdata-import/please": "Por favor,", - "gdata-parsing/store-cell": "Carregar células em branco como nulas", - "gdata-import/auth-doc": "Documentos autorizados", - "gdata-parsing/option": "Opções", - "gdata-exporter/upload-google-sheets-success": "Projeto enviado para o Google Drive ", - "gdata-parsing/parse-next": "Analisar próximo", - "gdata-parsing/worksheet": "Planilhas", - "gdata-import/title": "Documentos públicos", - "gdata-parsing/limit": "linhas(s) de dados", - "gdata-exporter/export-to-google-drive": "Ficheiro de projeto do OpenRefine para Google Drive…", - "gdata-import/sign-in": "Iniciar sessão", - "gdata-import/import-by-url": "Importar uma Planilha Google pública a partir de sua URL:", - "gdata-parsing/ignore-first": "Ignorar primeira(s)", - "gdata-auth/authorize-label": "OpenRefine - Autorização", - "gdata-import/access-data": "acesso aos seus dados do Google.", - "gdata-exporter/upload-google-drive-success": "Projeto enviado para o Google Drive ", - "gdata-source/authors": "Autores", - "gdata-exporter/uploading": "Carregando…", - "gdata-parsing/create-proj": "Criar projeto »", - "gdata-parsing/parse": "linha(s) como nomes das colunas", - "gdata-parsing/store-row": "Carregar linhas em branco", - "gdata-parsing/proj-name": "Nome do projeto", - "gdata-exporter/new-spreadsheet": "Uma nova planilha do Google", - "gdata-import/preparing": "Preparando …", - "gdata-parsing/preview-button": "Atualizar Pré-visualização", - "gdata-import/next->": "Próximo »", - "gdata-import/creating": "Criando projeto …", - "gdata-parsing/discard": "linhas(s) de dados", - "gdata-source/type": "Tipo", - "gdata-source/updated": "Atualizado", - "gdata-import/re-sign-in": "entrar novamente", - "gdata-exporter/google-sheets": "Planilhas Google", - "gdata-parsing/discard-next": "Descartar primeira(s)", - "gdata-parsing/updating-preview": "Atualizando pré-visualização …", - "gdata-parsing/limit-next": "Carregar ao menos", - "gdata-import/re-sign-in-another": "Faça login com conta diferente", - "gdata-import/please-signin": "Faça login e autorize o acesso aos seus dados do Google" -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-pt_BR.json b/OpenRefine/extensions/gdata/module/langs/translation-pt_BR.json deleted file mode 100644 index da88c685d..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-pt_BR.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "gdata-parsing/start-over": "« Recomeçar", - "gdata-import/please": "Por favor,", - "gdata-import/access-data": "acesso aos seus dados do Google.", - "gdata-auth/authorized-label": "Processo de autorização concluído. Feche esta janela e retorne ao OpenRefine.", - "gdata-auth/authorize-label": "OpenRefine - Autorização", - "gdata-exporter/enter-filename": "Digite um nome de arquivo para exportar o projeto no Google Drive", - "gdata-exporter/upload-google-sheets-success": "Projeto enviado para o Google Drive ", - "gdata-exporter/enter-spreadsheet": "Digite um nome para a nova planilha do Google", - "gdata-exporter/upload-google-drive-success": "Projeto enviado para o Google Drive ", - "gdata-exporter/new-spreadsheet": "Uma nova planilha do Google", - "gdata-exporter/upload-error": "Erro no carregamento: ", - "gdata-exporter/uploading": "Carregando…", - "gdata-exporter/google-sheets": "Planilhas Google", - "gdata-exporter/export-to-google-drive": "Arquivo de projeto do OpenRefine para Google Drive…", - "gdata-source/updated": "Atualizado", - "gdata-source/authors": "Autores", - "gdata-source/title": "Título", - "gdata-source/type": "Tipo", - "gdata-source/alert-url": "Você deve especificar um endereço web (URL) para importar.", - "gdata-parsing/store-cell": "Carregar células em branco como nulas", - "gdata-parsing/store-row": "Carregar linhas em branco", - "gdata-parsing/limit": "linhas(s) de dados", - "gdata-parsing/limit-next": "Carregar ao menos", - "gdata-parsing/discard": "linhas(s) de dados", - "gdata-parsing/discard-next": "Descartar primeira(s)", - "gdata-parsing/parse": "linha(s) como nomes das colunas", - "gdata-parsing/parse-next": "Analisar próximo", - "gdata-parsing/ignore": "linhas(s) no começo do arquivo", - "gdata-parsing/ignore-first": "Ignorar primeira(s)", - "gdata-parsing/preview-button": "Atualizar Pré-visualização", - "gdata-parsing/option": "Opções", - "gdata-parsing/worksheet": "Planilhas", - "gdata-parsing/updating-preview": "Atualizando pré-visualização …", - "gdata-parsing/create-proj": "Criar projeto »", - "gdata-parsing/proj-name": "Nome do projeto", - "gdata-parsing/conf-pars": "Configurar opções de análise", - "gdata-import/another-account": "com outra conta", - "gdata-import/re-sign-in": "entrar novamente", - "gdata-import/retrieving": "Recuperando documentos do Google Docs …", - "gdata-import/sign-out": "sair", - "gdata-import/sign-in": "entrar e autorizar", - "gdata-import/auth-doc": "Documentos autorizados", - "gdata-import/next->": "Próximo »", - "gdata-import/import-by-url": "Importar uma Planilha Google pública a partir de sua URL:", - "gdata-import/title": "Documentos públicos", - "gdata-import/creating": "Criando projeto …", - "gdata-import/preparing": "Preparando …" -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-pt_PT.json b/OpenRefine/extensions/gdata/module/langs/translation-pt_PT.json deleted file mode 100644 index 4b998c9d2..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-pt_PT.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "gdata-source/title": "Título", - "gdata-exporter/upload-error": "Erro no carregamento: ", - "gdata-import/sign-out": "Sair", - "gdata-exporter/enter-spreadsheet": "Digite um nome para a nova planilha do Google", - "gdata-import/retrieving": "Recuperando documentos do Google Docs …", - "gdata-auth/authorized-label": "Processo de autorização concluído. Feche esta janela e retorne ao OpenRefine.", - "gdata-exporter/enter-filename": "Digite um nome de ficheiro para exportar o projeto no Google Drive", - "gdata-source/alert-url": "Deve especificar um endereço web (URL) para importar.", - "gdata-parsing/conf-pars": "Configurar opções de análise", - "gdata-import/another-account": "com outra conta", - "gdata-parsing/start-over": "« Recomeçar", - "gdata-parsing/ignore": "linhas(s) no começo do ficheiro", - "gdata-import/please": "Por favor,", - "gdata-parsing/store-cell": "Carregar células em branco como nulas", - "gdata-import/auth-doc": "Documentos autorizados", - "gdata-parsing/option": "Opções", - "gdata-exporter/upload-google-sheets-success": "Projeto enviado para o Google Drive ", - "gdata-parsing/parse-next": "Analisar próximo", - "gdata-parsing/worksheet": "Planilhas", - "gdata-import/title": "Documentos públicos", - "gdata-parsing/limit": "linhas(s) de dados", - "gdata-exporter/export-to-google-drive": "Ficheiro de projeto do OpenRefine para Google Drive…", - "gdata-import/sign-in": "Iniciar sessão", - "gdata-import/import-by-url": "Importar uma Planilha Google pública a partir de sua URL:", - "gdata-parsing/ignore-first": "Ignorar primeira(s)", - "gdata-auth/authorize-label": "OpenRefine - Autorização", - "gdata-import/access-data": "acesso aos seus dados do Google.", - "gdata-exporter/upload-google-drive-success": "Projeto enviado para o Google Drive ", - "gdata-source/authors": "Autores", - "gdata-exporter/uploading": "Carregando…", - "gdata-parsing/create-proj": "Criar projeto »", - "gdata-parsing/parse": "linha(s) como nomes das colunas", - "gdata-parsing/store-row": "Carregar linhas em branco", - "gdata-parsing/proj-name": "Nome do projeto", - "gdata-exporter/new-spreadsheet": "Uma nova planilha do Google", - "gdata-import/preparing": "Preparando …", - "gdata-parsing/preview-button": "Atualizar Pré-visualização", - "gdata-import/next->": "Próximo »", - "gdata-import/creating": "Criando projeto …", - "gdata-parsing/discard": "linhas(s) de dados", - "gdata-source/type": "Tipo", - "gdata-source/updated": "Atualizado", - "gdata-import/re-sign-in": "entrar novamente", - "gdata-exporter/google-sheets": "Planilhas Google", - "gdata-parsing/discard-next": "Descartar primeira(s)", - "gdata-parsing/updating-preview": "Atualizando pré-visualização …", - "gdata-parsing/limit-next": "Carregar ao menos", - "gdata-import/re-sign-in-another": "Faça login com conta diferente", - "gdata-import/please-signin": "Faça login e autorize o acesso aos seus dados do Google" -} diff --git a/OpenRefine/extensions/gdata/module/langs/translation-sv.json b/OpenRefine/extensions/gdata/module/langs/translation-sv.json deleted file mode 100644 index 5a9945428..000000000 --- a/OpenRefine/extensions/gdata/module/langs/translation-sv.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "gdata-import/preparing": "Förbereder…", - "gdata-import/creating": "Skapar projekt…", - "gdata-import/title": "Offentliga dokument", - "gdata-import/import-by-url": "Importera ett publikt Google-kalkylblad eller fusionstabell med dess URL:", - "gdata-import/next->": "Nästa »", - "gdata-import/auth-doc": "Godkända dokument", - "gdata-import/please": "Var god", - "gdata-import/sign-in": "logga in och godkänn", - "gdata-import/sign-out": "logga ut", - "gdata-import/access-data": "tillgång till dina Google-data.", - "gdata-import/retrieving": "Hämtar Google Docs-dokument…", - "gdata-import/re-sign-in": "logga in igen", - "gdata-import/another-account": "med ett annat konto", - "gdata-parsing/start-over": "« Börja om", - "gdata-parsing/conf-pars": "Konfigurera parsningsalternativ", - "gdata-parsing/proj-name": "Projektnamn", - "gdata-parsing/create-proj": "Skapa projekt »", - "gdata-parsing/updating-preview": "Uppdatera förhandsgranskning…", - "gdata-parsing/worksheet": "kalkylblad", - "gdata-parsing/option": "Alternativ", - "gdata-parsing/preview-button": "Uppdatera förhandsgranskning", - "gdata-parsing/ignore-first": "Ignorera första", - "gdata-parsing/ignore": "raden(-erna) i början på filen", - "gdata-parsing/parse-next": "Parsa nästa", - "gdata-parsing/parse": "rad(er) som kolumnrubriker", - "gdata-parsing/discard-next": "Släng första", - "gdata-parsing/discard": "rad(er) av data", - "gdata-parsing/limit-next": "Ladda högst", - "gdata-parsing/limit": "rad(er) av data", - "gdata-parsing/store-row": "Spara tomma rader", - "gdata-parsing/store-cell": "Spara tomma celler som null", - "gdata-source/alert-url": "Du måste ange en webbadress (URL) att importera.", - "gdata-source/type": "Typ", - "gdata-source/title": "Titel", - "gdata-source/authors": "Författare", - "gdata-source/updated": "Uppdaterad", - "gdata-exporter/uploading": "Laddar upp…", - "gdata-exporter/upload-error": "Uppladdningsfel: ", - "gdata-exporter/upload-success": "Projektet laddades upp till Google Drive med ID ", - "gdata-exporter/new-spreadsheet": "Ett nytt Google-kalkylblad", - "gdata-exporter/enter-spreadsheet": "Ange ett namn för det nya Google-kalkylbladet", - "gdata-auth/authorize-label": "OpenRefine - godkännande", - "gdata-auth/authorized-label": "Godkännandeprocessen avslutad. Stäng det här fönstret och återgå till OpenRefine." -} diff --git a/OpenRefine/extensions/gdata/module/macros.vm b/OpenRefine/extensions/gdata/module/macros.vm deleted file mode 100644 index 18a83bfce..000000000 --- a/OpenRefine/extensions/gdata/module/macros.vm +++ /dev/null @@ -1,14 +0,0 @@ -#* - This file contains common velocity macros used in all .vt files. - For Velocity documentation, see: - - http://velocity.apache.org/engine/releases/velocity-1.5/user-guide.html -*# - -#macro( makeAList $list ) -
        - #foreach($item in $list) -
      • $item
      • - #end -
      -#end \ No newline at end of file diff --git a/OpenRefine/extensions/gdata/module/scripts/gdata-extension.js b/OpenRefine/extensions/gdata/module/scripts/gdata-extension.js deleted file mode 100644 index dd332616d..000000000 --- a/OpenRefine/extensions/gdata/module/scripts/gdata-extension.js +++ /dev/null @@ -1,63 +0,0 @@ -/* - -Copyright 2010, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -var GdataExtension = {}; - -GdataExtension.isAuthorized = function() { - // TODO: Unreliable - just means that we were authorized at one point - return Cookies.get('oauth2_token') != null; -}; - -GdataExtension.showAuthorizationDialog = function(onAuthorized, onNotAuthorized) { - if (window.name) { - var windowName = window.name; - } else { - var windowName = "openrefine" + new Date().getTime(); - window.name = windowName; - } - - var callbackName = "cb" + new Date().getTime(); - var callback = function(evt) { - delete window[callbackName]; - if (GdataExtension.isAuthorized()) { - onAuthorized(); - } else if (onNotAuthorized) { - onNotAuthorized(); - } - window.setTimeout(function() { win.close(); }, 100); - }; - window[callbackName] = callback; - - var url = ModuleWirings['gdata'] + "authorize?winname=" + escape(windowName) + "&cb=" + escape(callbackName); - var win = window.open(url, "openrefinegdataauth", "resizable=1,width=800,height=600"); -}; diff --git a/OpenRefine/extensions/gdata/module/scripts/index/gdata-parsing-panel.html b/OpenRefine/extensions/gdata/module/scripts/index/gdata-parsing-panel.html deleted file mode 100644 index ef5f949f1..000000000 --- a/OpenRefine/extensions/gdata/module/scripts/index/gdata-parsing-panel.html +++ /dev/null @@ -1,56 +0,0 @@ -
      - - - - - -
      - -
      - -
      - -
      - -
      - - - - - - - - - - - - - - -
        -
      - - - - - - - - - - - - - - - - - - - - -
      -
      -
      -
      -
      \ No newline at end of file diff --git a/OpenRefine/extensions/gdata/module/scripts/index/gdata-source-ui.js b/OpenRefine/extensions/gdata/module/scripts/index/gdata-source-ui.js deleted file mode 100644 index 76b9caf1b..000000000 --- a/OpenRefine/extensions/gdata/module/scripts/index/gdata-source-ui.js +++ /dev/null @@ -1,197 +0,0 @@ -/* - -Copyright 2011, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - */ - -Refine.GDataSourceUI = function(controller) { - this._controller = controller; -}; - -Refine.GDataSourceUI.prototype.attachUI = function(body) { - this._body = body; - - this._body.html(DOM.loadHTML("gdata", "scripts/index/import-from-gdata-form.html")); - this._elmts = DOM.bind(this._body); - - $('#gdata-title').text($.i18n('gdata-import/title')); - $('#gdata-import').html($.i18n('gdata-import/import-by-url')); - $('#gdata-next').html($.i18n('gdata-import/next->')); - $('#gdata-auth-doc').text($.i18n('gdata-import/auth-doc')); - $('#gdata-please-signin').text($.i18n('gdata-import/please-signin')); - $('#gdata-signin-btn').text($.i18n('gdata-import/sign-in')); - $('#gdata-retrieving').text($.i18n('gdata-import/retrieving')); - $('#gdata-signout').text($.i18n('gdata-import/sign-out')); - $('#gdata-resignin').text($.i18n('gdata-import/sign-in')); - $('#gdata-re-signin-another').text($.i18n('gdata-import/re-sign-in-another')); - - var self = this; - this._body.find('.gdata-signin.button').click(function() { - GdataExtension.showAuthorizationDialog( - function() { - self._listDocuments(); - }, - function() { - self._body.find('.gdata-page').hide(); - self._elmts.signinPage.show(); - } - ); - }); - this._body.find('.gdata-signout.button').click(function() { - $.get("command/gdata/deauthorize" ); - self._body.find('.gdata-page').hide(); - self._elmts.signinPage.show(); - }); - - this._elmts.urlNextButton.click(function(evt) { - var url = $.trim(self._elmts.urlInput[0].value); - if (url.length === 0) { - window.alert($.i18n('gdata-source/alert-url')); - } else { - var doc = {}; - doc.docSelfLink = url; - if (doc.docSelfLink.contains('spreadsheet')) { // TODO: fragile? - doc.type = 'spreadsheet'; - } else { - doc.type = 'table'; - } - self._controller.startImportingDocument(doc); - } - }); - - this._body.find('.gdata-page').hide(); - this._elmts.signinPage.show(); - - if (GdataExtension.isAuthorized()) { - this._listDocuments(); - } -}; - -Refine.GDataSourceUI.prototype.focus = function() { -}; - -Refine.GDataSourceUI.prototype._listDocuments = function() { - this._body.find('.gdata-page').hide(); - this._elmts.progressPage.show(); - - var self = this; - Refine.wrapCSRF(function(token) { - $.post( - "command/core/importing-controller?" + $.param({ - "controller": "gdata/gdata-importing-controller", - "subCommand": "list-documents", - "csrf_token": token - }), - null, - function(o) { - self._renderDocuments(o); - }, - "json" - ); - }); -}; - -Refine.GDataSourceUI.prototype._renderDocuments = function(o) { - var self = this; - - this._elmts.listingContainer.empty(); - - var table = $( - '' + - '' + // starred - '' + - '' + - '' + - '' + - '
      '+$.i18n('gdata-source/type')+''+$.i18n('gdata-source/title')+''+$.i18n('gdata-source/authors')+''+$.i18n('gdata-source/updated')+'
      ' - ).appendTo(this._elmts.listingContainer)[0]; - - var renderDocument = function(doc) { - var tr = table.insertRow(table.rows.length); - - var td = tr.insertCell(tr.cells.length); - if (doc.isStarred) { - $('').attr('src', 'images/star.png').appendTo(td); - } - - td = tr.insertCell(tr.cells.length); - $('').text(doc.type).appendTo(td); - - td = tr.insertCell(tr.cells.length); - $('') - .addClass('gdata-doc-title') - .attr('href', 'javascript:{}') - .text(doc.title) - .appendTo(td) - .click(function(evt) { - self._controller.startImportingDocument(doc); - }); - - $('') - .addClass('gdata-doc-preview') - .attr('href', doc.docLink) - .attr('target', '_blank') - .text('preview') - .appendTo(td); - - td = tr.insertCell(tr.cells.length); - $('') - .addClass('gdata-doc-authors') - .text((doc.authors) ? doc.authors.join(', ') : '') - .appendTo(td); - - td = tr.insertCell(tr.cells.length); - $('') - .addClass('gdata-doc-date') - .text((doc.updated) ? formatRelativeDate(doc.updated) : '') - .attr('title', (doc.updated) ? doc.updated : '') - .appendTo(td); - }; - - if (o.status === 'error') { - // We're probably not logged in, even though we thought we were. Show signin page - this._body.find('.gdata-page').hide(); - this._elmts.signinPage.show(); - } else { - var docs = o.documents; - $.each(docs, function() { - this.updatedDate = (this.updated) ? new Date(this.updated) : null; - this.updatedDateTime = (this.updatedDate) ? this.updatedDate.getTime() : 0; - }); - docs.sort(function(a, b) { return b.updatedDateTime - a.updatedDateTime; }); - - for (var i = 0; i < docs.length; i++) { - renderDocument(docs[i]); - } - - this._body.find('.gdata-page').hide(); - this._elmts.listingPage.show(); - } -}; diff --git a/OpenRefine/extensions/gdata/module/scripts/index/import-from-gdata-form.html b/OpenRefine/extensions/gdata/module/scripts/index/import-from-gdata-form.html deleted file mode 100644 index 61e9c1ca8..000000000 --- a/OpenRefine/extensions/gdata/module/scripts/index/import-from-gdata-form.html +++ /dev/null @@ -1,30 +0,0 @@ -
      -
      -

      -
      - - - - - -
      -
      - -
      -

      - -
      -

      -
      -
      -

      -
      -
      - - - -
      - -
      -
      -
      \ No newline at end of file diff --git a/OpenRefine/extensions/gdata/module/scripts/index/importing-controller.js b/OpenRefine/extensions/gdata/module/scripts/index/importing-controller.js deleted file mode 100644 index 30644a8e8..000000000 --- a/OpenRefine/extensions/gdata/module/scripts/index/importing-controller.js +++ /dev/null @@ -1,453 +0,0 @@ -/* - -Copyright 2011, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - */ - -//Internationalization init -var lang = navigator.language.split("-")[0] - || navigator.userLanguage.split("-")[0]; -var dictionary = ""; -$.ajax({ - url : "command/core/load-language?", - type : "POST", - async : false, - data : { - module : "gdata", -// lang : lang - }, - success : function(data) { - dictionary = data['dictionary']; - lang = data['lang']; - } -}); -$.i18n().load(dictionary, lang); -// End internationalization - -Refine.GDataImportingController = function(createProjectUI) { - this._createProjectUI = createProjectUI; - - this._parsingPanel = createProjectUI.addCustomPanel(); - - createProjectUI.addSourceSelectionUI({ - label: "Google Data", - id: "gdata-source", - ui: new Refine.GDataSourceUI(this) - }); - - $('#gdata-authorize').text($.i18n('gdata-auth/authorize-label')); - $('#gdata-authorized').text($.i18n('gdata-auth/authorized-label')); -}; -Refine.CreateProjectUI.controllers.push(Refine.GDataImportingController); - -Refine.GDataImportingController.prototype.startImportingDocument = function(doc) { - var dismiss = DialogSystem.showBusy($.i18n('gdata-import/preparing')); - - var self = this; - Refine.postCSRF( - "command/core/create-importing-job", - null, - function(data) { - Refine.wrapCSRF(function(token) { - $.post( - "command/core/importing-controller?" + $.param({ - "controller": "gdata/gdata-importing-controller", - "subCommand": "initialize-parser-ui", - "docUrl": doc.docSelfLink, - "docType": doc.type, - "csrf_token": token - }), - null, - function(data2) { - dismiss(); - - if (data2.status == 'ok') { - self._doc = doc; - self._jobID = data.jobID; - self._options = data2.options; - - self._showParsingPanel(); - } else { - alert(data2.message); - } - }, - "json" - ); - }); - }, - "json" - ); -}; - -Refine.GDataImportingController.prototype.getOptions = function() { - var options = { - docUrl: this._doc.docSelfLink, - docType: this._doc.type, - }; - - var parseIntDefault = function(s, def) { - try { - var n = parseInt(s); - if (!isNaN(n)) { - return n; - } - } catch (e) { - // Ignore - } - return def; - }; - - if (this._doc.type != 'table') { - this._parsingPanelElmts.sheetRecordContainer.find('input').each(function() { - if (this.checked) { - options.sheetUrl = this.getAttribute('sheetUrl'); - options.worksheetIndex = this.getAttribute('worksheetIndex'); - } - }); - - if (this._parsingPanelElmts.ignoreCheckbox[0].checked) { - options.ignoreLines = parseIntDefault(this._parsingPanelElmts.ignoreInput[0].value, -1); - } else { - options.ignoreLines = -1; - } - if (this._parsingPanelElmts.headerLinesCheckbox[0].checked) { - options.headerLines = parseIntDefault(this._parsingPanelElmts.headerLinesInput[0].value, 0); - } else { - options.headerLines = 0; - } - } - - if (this._parsingPanelElmts.skipCheckbox[0].checked) { - options.skipDataLines = parseIntDefault(this._parsingPanelElmts.skipInput[0].value, 0); - } else { - options.skipDataLines = 0; - } - if (this._parsingPanelElmts.limitCheckbox[0].checked) { - options.limit = parseIntDefault(this._parsingPanelElmts.limitInput[0].value, -1); - } else { - options.limit = -1; - } - options.storeBlankRows = this._parsingPanelElmts.storeBlankRowsCheckbox[0].checked; - options.storeBlankCellsAsNulls = this._parsingPanelElmts.storeBlankCellsAsNullsCheckbox[0].checked; - - options.disableAutoPreview = this._parsingPanelElmts.disableAutoPreviewCheckbox[0].checked; - - return options; -}; - -Refine.GDataImportingController.prototype._showParsingPanel = function() { - var self = this; - - this._parsingPanel.unbind().empty().html( - DOM.loadHTML("gdata", 'scripts/index/gdata-parsing-panel.html')); - this._parsingPanelElmts = DOM.bind(this._parsingPanel); - - this._parsingPanelElmts.gdata_worksheet.html($.i18n('gdata-parsing/worksheet')); - this._parsingPanelElmts.gdata_ignore_first.html($.i18n('gdata-parsing/ignore-first')); - this._parsingPanelElmts.gdata_ignore.html($.i18n('gdata-parsing/ignore')); - this._parsingPanelElmts.gdata_parse_next.html($.i18n('gdata-parsing/parse-next')); - this._parsingPanelElmts.gdata_parse.html($.i18n('gdata-parsing/parse')); - this._parsingPanelElmts.startOverButton.html($.i18n('gdata-parsing/start-over')); - this._parsingPanelElmts.gdata_conf_pars.html($.i18n('gdata-parsing/conf-pars')); - this._parsingPanelElmts.gdata_proj_name.html($.i18n('gdata-parsing/proj-name')); - this._parsingPanelElmts.createProjectButton.html($.i18n('gdata-parsing/create-proj')); - this._parsingPanelElmts.gdata_options.html($.i18n('gdata-parsing/option')); - this._parsingPanelElmts.gdata_disable_auto_preview.text($.i18n('gdata-parsing/disable-auto-preview')); - this._parsingPanelElmts.previewButton.html($.i18n('gdata-parsing/preview-button')); - this._parsingPanelElmts.gdata_updating.html($.i18n('gdata-parsing/updating-preview')); - this._parsingPanelElmts.gdata_discard_next.html($.i18n('gdata-parsing/discard-next')); - this._parsingPanelElmts.gdata_discard.html($.i18n('gdata-parsing/discard')); - this._parsingPanelElmts.gdata_limit_next.html($.i18n('gdata-parsing/limit-next')); - this._parsingPanelElmts.gdata_limit.html($.i18n('gdata-parsing/limit')); - this._parsingPanelElmts.gdata_store_row.html($.i18n('gdata-parsing/store-row')); - this._parsingPanelElmts.gdata_store_cell.html($.i18n('gdata-parsing/store-cell')); - - if (this._parsingPanelResizer) { - $(window).unbind('resize', this._parsingPanelResizer); - } - - this._parsingPanelResizer = function() { - var elmts = self._parsingPanelElmts; - var width = self._parsingPanel.width(); - var height = self._parsingPanel.height(); - var headerHeight = elmts.wizardHeader.outerHeight(true); - var controlPanelHeight = 250; - - elmts.dataPanel - .css("left", "0px") - .css("top", headerHeight + "px") - .css("width", (width - DOM.getHPaddings(elmts.dataPanel)) + "px") - .css("height", (height - headerHeight - controlPanelHeight - DOM.getVPaddings(elmts.dataPanel)) + "px"); - elmts.progressPanel - .css("left", "0px") - .css("top", headerHeight + "px") - .css("width", (width - DOM.getHPaddings(elmts.progressPanel)) + "px") - .css("height", (height - headerHeight - controlPanelHeight - DOM.getVPaddings(elmts.progressPanel)) + "px"); - - elmts.controlPanel - .css("left", "0px") - .css("top", (height - controlPanelHeight) + "px") - .css("width", (width - DOM.getHPaddings(elmts.controlPanel)) + "px") - .css("height", (controlPanelHeight - DOM.getVPaddings(elmts.controlPanel)) + "px"); - }; - $(window).resize(this._parsingPanelResizer); - this._parsingPanelResizer(); - - this._parsingPanelElmts.startOverButton.click(function() { - // explicitly cancel the import job - Refine.CreateProjectUI.cancelImportingJob(self._jobID); - - delete self._doc; - delete self._jobID; - delete self._options; - - self._createProjectUI.showSourceSelectionPanel(); - }); - this._parsingPanelElmts.createProjectButton.click(function() { self._createProject(); }); - this._parsingPanelElmts.previewButton.click(function() { self._updatePreview(); }); - - this._parsingPanelElmts.projectNameInput[0].value = this._doc.title; - - if (this._doc.type != 'table') { - var sheetTable = this._parsingPanelElmts.sheetRecordContainer[0]; - $.each(this._options.worksheets, function(i, v) { - var id = 'gdata_worksheet-' + Math.round(Math.random() * 1000000); - - var tr = sheetTable.insertRow(sheetTable.rows.length); - var td0 = $(tr.insertCell(0)).attr('width', '1%'); - var checkbox = $('') - .attr('id', id) - .attr('type', 'radio') - .attr('name', 'gdata-importing-parsing-worksheet') - .attr('sheetUrl', this.link) - .attr('worksheetIndex', this.worksheetIndex) - .appendTo(td0); - if (i === 0) { - checkbox.prop("checked", true); - } - - $('