change OR version
This commit is contained in:
prance
parent
8694374775
commit
2a7e2e66b5
1
OpenRefine/.github/FUNDING.yml
vendored
1
OpenRefine/.github/FUNDING.yml
vendored
@ -1 +0,0 @@
|
|||||||
github: OpenRefine
|
|
||||||
40
OpenRefine/.github/ISSUE_TEMPLATE/bug_report.md
vendored
40
OpenRefine/.github/ISSUE_TEMPLATE/bug_report.md
vendored
@ -1,40 +0,0 @@
|
|||||||
---
|
|
||||||
name: Bug report
|
|
||||||
about: Create a report to help us improve OpenRefine
|
|
||||||
title: ''
|
|
||||||
labels: bug, to be reviewed
|
|
||||||
assignees: ''
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
<!-- Describe the bug - Please add a clear and concise description of the bug above this line. You can delete this line if you want. It will be hidden in the final bug report -->
|
|
||||||
|
|
||||||
### To Reproduce
|
|
||||||
Steps to reproduce the behavior:
|
|
||||||
1. First, do ...
|
|
||||||
2. Then, ...
|
|
||||||
3. Finally, ...
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### Current Results
|
|
||||||
<!-- What results occurred or were shown. -->
|
|
||||||
|
|
||||||
### Expected Behavior
|
|
||||||
<!-- A clear and concise description of what you expected to happen or to show. -->
|
|
||||||
|
|
||||||
### Screenshots
|
|
||||||
<!-- If applicable, add screenshots to help explain your problem. -->
|
|
||||||
|
|
||||||
### Versions<!-- (please complete the following information)-->
|
|
||||||
- Operating System: <!-- e.g. iOS, Windows 10, Linux, Ubuntu 18.04 -->
|
|
||||||
- Browser Version: <!-- e.g. Chrome 19, Firefox 61, Safari, NOTE: OpenRefine does not support IE but works OK in most cases -->
|
|
||||||
- JRE or JDK Version: <!-- output of "java -version" e.g. JRE 1.8.0_181 -->
|
|
||||||
- OpenRefine: <!-- e.g. OpenRefine 3.0 Beta] -->
|
|
||||||
|
|
||||||
### Datasets
|
|
||||||
<!-- If you are allowed and are OK with making your data public, it would be awesome if you can include or attach the data causing the issue or a URL pointing to where the data is.
|
|
||||||
If you are concerned about keeping your data private, ping us on our [mailing list](https://groups.google.com/forum/#!forum/openrefine) -->
|
|
||||||
|
|
||||||
### Additional context
|
|
||||||
<!-- Add any other context about the problem here. -->
|
|
||||||
8
OpenRefine/.github/ISSUE_TEMPLATE/config.yml
vendored
8
OpenRefine/.github/ISSUE_TEMPLATE/config.yml
vendored
@ -1,8 +0,0 @@
|
|||||||
blank_issues_enabled: false
|
|
||||||
contact_links:
|
|
||||||
- name: Ask a question (mailing list)
|
|
||||||
url: https://groups.google.com/d/forum/openrefine
|
|
||||||
about: Please ask and answer questions here.
|
|
||||||
- name: Gitter chat
|
|
||||||
url: https://gitter.im/OpenRefine/OpenRefine
|
|
||||||
about: General and developer discussions
|
|
||||||
@ -1,19 +0,0 @@
|
|||||||
---
|
|
||||||
name: Feature request
|
|
||||||
about: Suggest an idea for OpenRefine
|
|
||||||
title: ''
|
|
||||||
labels: enhancement, to be reviewed
|
|
||||||
assignees: ''
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
<!-- Please provide a clear and concise description of your problem or unsatisfied needs. Ex. I'm always frustrated when [...] or, It would be easier if OpenRefine did [...]. This comment can be deleted, if desired, but it will be hidden in your final submission. Please make sure that your new text is outside the enclosing angle brackets. -->
|
|
||||||
|
|
||||||
### Proposed solution
|
|
||||||
<!-- If you have a proposal for how this need could be met, please provide a clear and concise description of what you want to happen. -->
|
|
||||||
|
|
||||||
### Alternatives considered
|
|
||||||
<!-- If there alternative solutions that you have considered or think should be considered, please list them here -->
|
|
||||||
|
|
||||||
### Additional context
|
|
||||||
<!-- Add any other context or screenshots about the feature request here. -->
|
|
||||||
5
OpenRefine/.github/SUPPORT.md
vendored
5
OpenRefine/.github/SUPPORT.md
vendored
@ -1,5 +0,0 @@
|
|||||||
If you are having trouble with OpenRefine we suggest the following steps:
|
|
||||||
|
|
||||||
1. Read through our [FAQ (frequently Asked Questions)](https://github.com/OpenRefine/OpenRefine/wiki/FAQ)
|
|
||||||
2. Search for similar issues in our community email list archives: http://groups.google.com/group/openrefine/
|
|
||||||
3. Send an email to our community mailing list: openrefine@googlegroups.com
|
|
||||||
3
OpenRefine/.github/autolabeler.yml
vendored
3
OpenRefine/.github/autolabeler.yml
vendored
@ -1,3 +0,0 @@
|
|||||||
documentation: ["/docs"]
|
|
||||||
"current docs": ["/docs/docs"]
|
|
||||||
"historical docs": ["/docs/versioned_docs"]
|
|
||||||
45
OpenRefine/.github/dependabot.yml
vendored
45
OpenRefine/.github/dependabot.yml
vendored
@ -1,45 +0,0 @@
|
|||||||
# Documentation for all configuration options:
|
|
||||||
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
|
||||||
|
|
||||||
version: 2
|
|
||||||
updates:
|
|
||||||
# For openrefine java deps
|
|
||||||
- package-ecosystem: maven
|
|
||||||
directory: "/"
|
|
||||||
schedule:
|
|
||||||
interval: daily
|
|
||||||
open-pull-requests-limit: 10
|
|
||||||
ignore:
|
|
||||||
- dependency-name: com.thoughtworks.xstream:xstream
|
|
||||||
versions:
|
|
||||||
- "> 1.4.12"
|
|
||||||
- "< 2"
|
|
||||||
|
|
||||||
# Same, on 4.0 branch
|
|
||||||
- package-ecosystem: maven
|
|
||||||
directory: "/"
|
|
||||||
schedule:
|
|
||||||
interval: daily
|
|
||||||
open-pull-requests-limit: 10
|
|
||||||
ignore:
|
|
||||||
- dependency-name: com.thoughtworks.xstream:xstream
|
|
||||||
versions:
|
|
||||||
- "> 1.4.12"
|
|
||||||
- "< 2"
|
|
||||||
target-branch: 4.0
|
|
||||||
|
|
||||||
# For documentation website
|
|
||||||
- package-ecosystem: "npm" # For Yarn
|
|
||||||
directory: "docs/"
|
|
||||||
schedule:
|
|
||||||
interval: "monthly"
|
|
||||||
# For github actions
|
|
||||||
- package-ecosystem: "github-actions"
|
|
||||||
directory: "/"
|
|
||||||
schedule:
|
|
||||||
interval: "monthly"
|
|
||||||
# For cypress test_suite
|
|
||||||
- package-ecosystem: "npm"
|
|
||||||
directory: "main/tests/cypress"
|
|
||||||
schedule:
|
|
||||||
interval: "daily"
|
|
||||||
6
OpenRefine/.github/pull_request_template.md
vendored
6
OpenRefine/.github/pull_request_template.md
vendored
@ -1,6 +0,0 @@
|
|||||||
Fixes #{issue number here}
|
|
||||||
|
|
||||||
Changes proposed in this pull request:
|
|
||||||
-
|
|
||||||
-
|
|
||||||
-
|
|
||||||
26
OpenRefine/.github/workflows/label_transfer.yml
vendored
26
OpenRefine/.github/workflows/label_transfer.yml
vendored
@ -1,26 +0,0 @@
|
|||||||
|
|
||||||
name: Copy labels from issue to pull request
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request_target:
|
|
||||||
types: [opened, edited]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
transfer_tags:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2
|
|
||||||
- name: Set up Python 3.9
|
|
||||||
uses: actions/setup-python@v2
|
|
||||||
with:
|
|
||||||
python-version: 3.9
|
|
||||||
- name: Install dependencies
|
|
||||||
run: |
|
|
||||||
python -m pip install --upgrade pip setuptools
|
|
||||||
pip install -r .github/workflows/label_transfer/requirements.txt
|
|
||||||
pip freeze
|
|
||||||
- name: Run Python label transfer script
|
|
||||||
run: python .github/workflows/label_transfer/script.py ${{ github.event.number }}
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
GITHUB_REPO: ${{ github.repository }}
|
|
||||||
@ -1,2 +0,0 @@
|
|||||||
requests
|
|
||||||
lxml
|
|
||||||
@ -1,76 +0,0 @@
|
|||||||
import requests
|
|
||||||
import os
|
|
||||||
from lxml import html
|
|
||||||
import sys
|
|
||||||
import json
|
|
||||||
|
|
||||||
# Config
|
|
||||||
|
|
||||||
# list of labels that should not be transferred to PRs
|
|
||||||
do_not_transfer = [
|
|
||||||
'good first issue',
|
|
||||||
'good second issue',
|
|
||||||
'imported from old code repo',
|
|
||||||
'help wanted',
|
|
||||||
'duplicate',
|
|
||||||
'invalid',
|
|
||||||
'question',
|
|
||||||
'to be reviewed',
|
|
||||||
]
|
|
||||||
|
|
||||||
# Internal
|
|
||||||
|
|
||||||
repo = os.environ.get('GITHUB_REPO')
|
|
||||||
github_token = os.environ.get('GITHUB_TOKEN')
|
|
||||||
|
|
||||||
headers = {
|
|
||||||
'Accept': 'application/vnd.github.v3+json',
|
|
||||||
}
|
|
||||||
|
|
||||||
if github_token:
|
|
||||||
headers['Authorization'] = 'Bearer '+github_token
|
|
||||||
|
|
||||||
def get_linked_issues(pr_number):
|
|
||||||
"""
|
|
||||||
Given a PR number, extract all the linked issue numbers from it.
|
|
||||||
Sadly this is not supported by the API yet, so we just scrape the web UI.
|
|
||||||
"""
|
|
||||||
url = f'https://github.com/{repo}/pull/{pr_number}'
|
|
||||||
page = requests.get(url)
|
|
||||||
page.raise_for_status()
|
|
||||||
parsed = html.document_fromstring(page.text)
|
|
||||||
matches = parsed.xpath('//form/div[@class="css-truncate my-1"]/a')
|
|
||||||
for match in matches:
|
|
||||||
yield int(match.attrib['href'].split('/')[-1])
|
|
||||||
|
|
||||||
def get_issue_labels(issue_number):
|
|
||||||
"""
|
|
||||||
Returns all the labels in a given issue / PR
|
|
||||||
"""
|
|
||||||
url = f'https://api.github.com/repos/{repo}/issues/{issue_number}/labels'
|
|
||||||
response = requests.get(url, headers=headers)
|
|
||||||
response.raise_for_status()
|
|
||||||
return [ tag['name'] for tag in response.json() ]
|
|
||||||
|
|
||||||
def transfer_issue_labels(pr_number):
|
|
||||||
"""
|
|
||||||
Transfers labels from all the linked issues to the PR
|
|
||||||
"""
|
|
||||||
linked_issues = get_linked_issues(pr_number)
|
|
||||||
all_labels = [ label for issue in linked_issues for label in get_issue_labels(issue) ]
|
|
||||||
to_transfer = [ label for label in all_labels if label not in do_not_transfer ]
|
|
||||||
current_labels = get_issue_labels(pr_number)
|
|
||||||
missing_labels = [ label for label in to_transfer if label not in current_labels ]
|
|
||||||
if not missing_labels:
|
|
||||||
return
|
|
||||||
new_labels = current_labels + missing_labels
|
|
||||||
url = f'https://api.github.com/repos/{repo}/issues/{pr_number}/labels'
|
|
||||||
print(f'adding {missing_labels} to PR #{pr_number}')
|
|
||||||
if not github_token:
|
|
||||||
print('no GITHUB_TOKEN, skipping')
|
|
||||||
else:
|
|
||||||
resp = requests.put(url, headers=headers, data=json.dumps({'labels':new_labels}))
|
|
||||||
resp.raise_for_status()
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
transfer_issue_labels(sys.argv[1])
|
|
||||||
171
OpenRefine/.github/workflows/pull_request.yml
vendored
171
OpenRefine/.github/workflows/pull_request.yml
vendored
@ -1,171 +0,0 @@
|
|||||||
name: Continuous Integration
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request_target:
|
|
||||||
paths-ignore:
|
|
||||||
- 'docs/**'
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
- '4.0'
|
|
||||||
|
|
||||||
permissions: read-all
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
server_tests:
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
java: [ 11, 17 ]
|
|
||||||
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
services:
|
|
||||||
postgres:
|
|
||||||
image: postgres
|
|
||||||
ports:
|
|
||||||
- 5432
|
|
||||||
env:
|
|
||||||
POSTGRES_USER: postgres
|
|
||||||
POSTGRES_PASSWORD: 'postgres'
|
|
||||||
POSTGRES_DB: test_db
|
|
||||||
options: >-
|
|
||||||
--health-cmd pg_isready
|
|
||||||
--health-interval 10s
|
|
||||||
--health-timeout 5s
|
|
||||||
--health-retries 5
|
|
||||||
mysql:
|
|
||||||
image: mysql:8
|
|
||||||
ports:
|
|
||||||
- 3306
|
|
||||||
env:
|
|
||||||
MYSQL_ROOT_PASSWORD: root
|
|
||||||
options: >-
|
|
||||||
--health-cmd "mysqladmin ping"
|
|
||||||
--health-interval 5s
|
|
||||||
--health-timeout 2s
|
|
||||||
--health-retries 3
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2.3.4
|
|
||||||
with:
|
|
||||||
ref: ${{ github.event.pull_request.head.ref }}
|
|
||||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
|
||||||
|
|
||||||
- name: Restore dependency cache
|
|
||||||
uses: actions/cache@v2.1.7
|
|
||||||
with:
|
|
||||||
path: ~/.m2/repository
|
|
||||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
|
||||||
restore-keys: |
|
|
||||||
${{ runner.os }}-maven-
|
|
||||||
|
|
||||||
- name: Set up Java ${{ matrix.java }}
|
|
||||||
uses: actions/setup-java@v2
|
|
||||||
with:
|
|
||||||
distribution: 'adopt'
|
|
||||||
java-version: ${{ matrix.java }}
|
|
||||||
|
|
||||||
- name: Check Java linting
|
|
||||||
id: java_linting
|
|
||||||
run: |
|
|
||||||
mvn formatter:validate
|
|
||||||
|
|
||||||
- name: Configure connections to databases
|
|
||||||
id: configure_db_connections
|
|
||||||
run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml
|
|
||||||
|
|
||||||
- name: Populate databases with test data
|
|
||||||
id: populate_databases_with_test_data
|
|
||||||
run: |
|
|
||||||
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;'
|
|
||||||
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql
|
|
||||||
psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql
|
|
||||||
env:
|
|
||||||
PGPASSWORD: postgres
|
|
||||||
|
|
||||||
- name: Build and test with Maven
|
|
||||||
run: mvn jacoco:prepare-agent test
|
|
||||||
|
|
||||||
- name: Submit test coverage to Coveralls
|
|
||||||
run: |
|
|
||||||
mvn prepare-package -DskipTests=true
|
|
||||||
mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }}
|
|
||||||
|
|
||||||
prepare_ui_test_matrix:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
outputs:
|
|
||||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2.3.4
|
|
||||||
with:
|
|
||||||
ref: ${{ github.event.pull_request.head.ref }}
|
|
||||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
|
||||||
- name: Setup Node
|
|
||||||
uses: actions/setup-node@v2
|
|
||||||
with:
|
|
||||||
node-version: '12'
|
|
||||||
- id: set-matrix
|
|
||||||
run: npm install --save glob && node main/tests/cypress/build-test-matrix.js
|
|
||||||
env:
|
|
||||||
browsers: chrome
|
|
||||||
ui_test:
|
|
||||||
needs: prepare_ui_test_matrix
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2.3.4
|
|
||||||
with:
|
|
||||||
ref: ${{ github.event.pull_request.head.ref }}
|
|
||||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
|
||||||
|
|
||||||
- name: Restore dependency cache
|
|
||||||
uses: actions/cache@v2.1.7
|
|
||||||
with:
|
|
||||||
path: ~/.m2/repository
|
|
||||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
|
||||||
restore-keys: |
|
|
||||||
${{ runner.os }}-maven-
|
|
||||||
|
|
||||||
- name: Set up Java 11
|
|
||||||
uses: actions/setup-java@v2
|
|
||||||
with:
|
|
||||||
distribution: 'adopt'
|
|
||||||
java-version: 11
|
|
||||||
|
|
||||||
- name: Build OpenRefine
|
|
||||||
run: ./refine build
|
|
||||||
|
|
||||||
- name: Setup Node
|
|
||||||
uses: actions/setup-node@v2
|
|
||||||
with:
|
|
||||||
node-version: '12'
|
|
||||||
|
|
||||||
- name: Restore Tests dependency cache
|
|
||||||
uses: actions/cache@v2.1.7
|
|
||||||
with:
|
|
||||||
path: |
|
|
||||||
~/cache
|
|
||||||
~/.cache
|
|
||||||
**/node_modules
|
|
||||||
!~/cache/exclude
|
|
||||||
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
|
|
||||||
restore-keys: |
|
|
||||||
${{ runner.os }}-yarn
|
|
||||||
|
|
||||||
- name: Install test dependencies
|
|
||||||
run: |
|
|
||||||
cd ./main/tests/cypress
|
|
||||||
npm i -g yarn
|
|
||||||
yarn install
|
|
||||||
|
|
||||||
- name: Test with Cypress on ${{ matrix.browser }}
|
|
||||||
run: |
|
|
||||||
echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
|
|
||||||
echo REFINE_MEMORY=4096M >> ./refine.ini
|
|
||||||
./refine ui_tests
|
|
||||||
env:
|
|
||||||
CYPRESS_BROWSER: ${{ matrix.browser }}
|
|
||||||
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
|
|
||||||
CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
|
|
||||||
CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
|
|
||||||
CYPRESS_SPECS: ${{ matrix.specs }}
|
|
||||||
177
OpenRefine/.github/workflows/snapshot_release.yml
vendored
177
OpenRefine/.github/workflows/snapshot_release.yml
vendored
@ -1,177 +0,0 @@
|
|||||||
name: Snapshot release
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- 'master'
|
|
||||||
- '4.0'
|
|
||||||
paths-ignore:
|
|
||||||
- 'docs/**'
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
prepare_ui_test_matrix:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
outputs:
|
|
||||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2.3.4
|
|
||||||
with:
|
|
||||||
ref: ${{ github.event.pull_request.head.ref }}
|
|
||||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
|
||||||
- name: Setup Node
|
|
||||||
uses: actions/setup-node@v2
|
|
||||||
with:
|
|
||||||
node-version: '12'
|
|
||||||
- id: set-matrix
|
|
||||||
run: npm install --save glob && node main/tests/cypress/build-test-matrix.js
|
|
||||||
env:
|
|
||||||
browsers: chrome
|
|
||||||
ui_test:
|
|
||||||
needs: prepare_ui_test_matrix
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2.3.4
|
|
||||||
with:
|
|
||||||
ref: ${{ github.event.pull_request.head.ref }}
|
|
||||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
|
||||||
|
|
||||||
- name: Restore dependency cache
|
|
||||||
uses: actions/cache@v2.1.7
|
|
||||||
with:
|
|
||||||
path: ~/.m2/repository
|
|
||||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
|
||||||
restore-keys: |
|
|
||||||
${{ runner.os }}-maven-
|
|
||||||
|
|
||||||
- name: Set up Java 11
|
|
||||||
uses: actions/setup-java@v2
|
|
||||||
with:
|
|
||||||
distribution: 'adopt'
|
|
||||||
java-version: 11
|
|
||||||
|
|
||||||
- name: Build OpenRefine
|
|
||||||
run: ./refine build
|
|
||||||
|
|
||||||
- name: Setup Node
|
|
||||||
uses: actions/setup-node@v2
|
|
||||||
with:
|
|
||||||
node-version: '12'
|
|
||||||
|
|
||||||
- name: Restore Tests dependency cache
|
|
||||||
uses: actions/cache@v2.1.7
|
|
||||||
with:
|
|
||||||
path: |
|
|
||||||
~/cache
|
|
||||||
~/.cache
|
|
||||||
**/node_modules
|
|
||||||
!~/cache/exclude
|
|
||||||
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
|
|
||||||
restore-keys: |
|
|
||||||
${{ runner.os }}-yarn
|
|
||||||
|
|
||||||
- name: Install test dependencies
|
|
||||||
run: |
|
|
||||||
cd ./main/tests/cypress
|
|
||||||
npm i -g yarn
|
|
||||||
yarn install
|
|
||||||
|
|
||||||
- name: Test with Cypress on ${{ matrix.browser }}
|
|
||||||
run: |
|
|
||||||
echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
|
|
||||||
echo REFINE_MEMORY=4096M >> ./refine.ini
|
|
||||||
./refine ui_tests
|
|
||||||
env:
|
|
||||||
CYPRESS_BROWSER: ${{ matrix.browser }}
|
|
||||||
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
|
|
||||||
CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
|
|
||||||
CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
|
|
||||||
CYPRESS_SPECS: ${{ matrix.specs }}
|
|
||||||
|
|
||||||
|
|
||||||
build:
|
|
||||||
|
|
||||||
services:
|
|
||||||
postgres:
|
|
||||||
image: postgres
|
|
||||||
ports:
|
|
||||||
- 5432
|
|
||||||
env:
|
|
||||||
POSTGRES_USER: postgres
|
|
||||||
POSTGRES_PASSWORD: 'postgres'
|
|
||||||
POSTGRES_DB: test_db
|
|
||||||
options: >-
|
|
||||||
--health-cmd pg_isready
|
|
||||||
--health-interval 10s
|
|
||||||
--health-timeout 5s
|
|
||||||
--health-retries 5
|
|
||||||
mysql:
|
|
||||||
image: mysql:8
|
|
||||||
ports:
|
|
||||||
- 3306
|
|
||||||
env:
|
|
||||||
MYSQL_ROOT_PASSWORD: root
|
|
||||||
options: >-
|
|
||||||
--health-cmd "mysqladmin ping"
|
|
||||||
--health-interval 5s
|
|
||||||
--health-timeout 2s
|
|
||||||
--health-retries 3
|
|
||||||
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2.3.4
|
|
||||||
with:
|
|
||||||
fetch-depth: 0 # This is wasteful, but needed for git describe
|
|
||||||
|
|
||||||
- name: Restore dependency cache
|
|
||||||
uses: actions/cache@v2.1.7
|
|
||||||
with:
|
|
||||||
path: ~/.m2/repository
|
|
||||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
|
||||||
restore-keys: |
|
|
||||||
${{ runner.os }}-maven-
|
|
||||||
|
|
||||||
- name: Set up Java 11
|
|
||||||
uses: actions/setup-java@v2
|
|
||||||
with:
|
|
||||||
distribution: 'adopt'
|
|
||||||
java-version: 11
|
|
||||||
server-id: ossrh
|
|
||||||
server-username: OSSRH_USER
|
|
||||||
server-password: OSSRH_PASS
|
|
||||||
|
|
||||||
- name: Install genisoimage and jq
|
|
||||||
run: sudo apt-get install genisoimage jq
|
|
||||||
|
|
||||||
- name: Configure connections to databases
|
|
||||||
id: configure_db_connections
|
|
||||||
run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml
|
|
||||||
|
|
||||||
- name: Populate databases with test data
|
|
||||||
id: populate_databases_with_test_data
|
|
||||||
run: |
|
|
||||||
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;'
|
|
||||||
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql
|
|
||||||
psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql
|
|
||||||
env:
|
|
||||||
PGPASSWORD: postgres
|
|
||||||
|
|
||||||
- name: Build and test with Maven
|
|
||||||
run: mvn jacoco:prepare-agent test
|
|
||||||
|
|
||||||
- name: Submit test coverage to Coveralls
|
|
||||||
run: |
|
|
||||||
mvn prepare-package -DskipTests=true
|
|
||||||
mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }} -DserviceName="GitHub Actions" -DserviceBuildNumber=${{ env.GITHUB_RUN_ID }} -Dbranch=master
|
|
||||||
|
|
||||||
- name: Generate dist files
|
|
||||||
run: mvn package -DskipTests=true
|
|
||||||
|
|
||||||
- name: Upload snapshot releases to OSSRH
|
|
||||||
run: mvn deploy -DskipTests=true
|
|
||||||
env:
|
|
||||||
OSSRH_USER: ${{ secrets.OSSRH_USER }}
|
|
||||||
OSSRH_PASS: ${{ secrets.OSSRH_PASS }}
|
|
||||||
|
|
||||||
19
OpenRefine/Dockerfile
Normal file
19
OpenRefine/Dockerfile
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
FROM maven:3-jdk-11 as build
|
||||||
|
|
||||||
|
WORKDIR /usr/src/app
|
||||||
|
COPY OpenRefine .
|
||||||
|
WORKDIR /usr/src/app/OpenRefine
|
||||||
|
|
||||||
|
RUN ./refine clean
|
||||||
|
RUN ./refine build
|
||||||
|
|
||||||
|
FROM openjdk:11
|
||||||
|
|
||||||
|
WORKDIR /usr/app
|
||||||
|
COPY --from=build /usr/src/app/OpenRefine .
|
||||||
|
|
||||||
|
VOLUME /data
|
||||||
|
EXPOSE 3333
|
||||||
|
|
||||||
|
ENTRYPOINT ["/usr/app/refine"]
|
||||||
|
CMD ["-i", "0.0.0.0", "-d", "/data", "-m", "3G"]
|
||||||
29
OpenRefine/ls1.txt
Normal file
29
OpenRefine/ls1.txt
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
razem 164
|
||||||
|
drwxr-xr-x 13 prance prance 4096 01-30 20:46 .
|
||||||
|
drwxr-xr-x 4 prance prance 4096 01-30 20:41 ..
|
||||||
|
-rw-r--r-- 1 prance prance 1603 01-30 20:42 appveyor.yml
|
||||||
|
drwxr-xr-x 3 prance prance 4096 01-30 20:42 benchmark
|
||||||
|
-rw-r--r-- 1 prance prance 3483 01-30 20:42 CODE_OF_CONDUCT.md
|
||||||
|
drwxr-xr-x 2 prance prance 4096 01-30 20:42 conf
|
||||||
|
-rw-r--r-- 1 prance prance 6438 01-30 20:42 CONTRIBUTING.md
|
||||||
|
drwxr-xr-x 7 prance prance 4096 01-30 20:42 docs
|
||||||
|
drwxr-xr-x 9 prance prance 4096 01-30 20:42 extensions
|
||||||
|
drwxr-xr-x 8 prance prance 4096 01-30 20:42 .git
|
||||||
|
-rw-r--r-- 1 prance prance 482 01-30 20:42 .gitattributes
|
||||||
|
drwxr-xr-x 4 prance prance 4096 01-30 20:42 .github
|
||||||
|
-rw-r--r-- 1 prance prance 1254 01-30 20:42 .gitignore
|
||||||
|
-rw-r--r-- 1 prance prance 8613 01-30 20:42 GOVERNANCE.md
|
||||||
|
drwxr-xr-x 4 prance prance 4096 01-30 20:42 graphics
|
||||||
|
drwxr-xr-x 3 prance prance 4096 01-30 20:42 IDEs
|
||||||
|
-rw-r--r-- 1 prance prance 1478 01-30 20:42 LICENSE.txt
|
||||||
|
-rw-r--r-- 1 prance prance 0 01-30 20:46 ls1.txt
|
||||||
|
drwxr-xr-x 6 prance prance 4096 01-30 20:42 main
|
||||||
|
drwxr-xr-x 2 prance prance 4096 01-30 20:42 packaging
|
||||||
|
-rw-r--r-- 1 prance prance 13518 01-30 20:42 pom.xml
|
||||||
|
-rw-r--r-- 1 prance prance 3523 01-30 20:42 README.md
|
||||||
|
-rwxr-xr-x 1 prance prance 29532 01-30 20:42 refine
|
||||||
|
-rw-r--r-- 1 prance prance 7680 01-30 20:42 refine.bat
|
||||||
|
-rw-r--r-- 1 prance prance 1549 01-30 20:42 refine.ini
|
||||||
|
-rw-r--r-- 1 prance prance 729 01-30 20:42 SECURITY.md
|
||||||
|
drwxr-xr-x 5 prance prance 4096 01-30 20:42 server
|
||||||
|
-rw-r--r-- 1 prance prance 1099 01-30 20:42 settings.xml
|
||||||
31
OpenRefine/ls2.txt
Normal file
31
OpenRefine/ls2.txt
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
razem 172
|
||||||
|
drwxr-xr-x 14 prance prance 4096 01-30 20:47 .
|
||||||
|
drwxr-xr-x 4 prance prance 4096 01-30 20:41 ..
|
||||||
|
-rw-r--r-- 1 prance prance 1603 01-30 20:42 appveyor.yml
|
||||||
|
drwxr-xr-x 3 prance prance 4096 01-30 20:42 benchmark
|
||||||
|
-rw-r--r-- 1 prance prance 3483 01-30 20:42 CODE_OF_CONDUCT.md
|
||||||
|
drwxr-xr-x 2 prance prance 4096 01-30 20:42 conf
|
||||||
|
-rw-r--r-- 1 prance prance 6438 01-30 20:42 CONTRIBUTING.md
|
||||||
|
drwxr-xr-x 7 prance prance 4096 01-30 20:42 docs
|
||||||
|
drwxr-xr-x 9 prance prance 4096 01-30 20:42 extensions
|
||||||
|
drwxr-xr-x 8 prance prance 4096 01-30 20:42 .git
|
||||||
|
-rw-r--r-- 1 prance prance 482 01-30 20:42 .gitattributes
|
||||||
|
drwxr-xr-x 4 prance prance 4096 01-30 20:42 .github
|
||||||
|
-rw-r--r-- 1 prance prance 1254 01-30 20:42 .gitignore
|
||||||
|
-rw-r--r-- 1 prance prance 8613 01-30 20:42 GOVERNANCE.md
|
||||||
|
drwxr-xr-x 4 prance prance 4096 01-30 20:42 graphics
|
||||||
|
drwxr-xr-x 3 prance prance 4096 01-30 20:42 IDEs
|
||||||
|
-rw-r--r-- 1 prance prance 1478 01-30 20:42 LICENSE.txt
|
||||||
|
-rw-r--r-- 1 prance prance 1563 01-30 20:46 ls1.txt
|
||||||
|
-rw-r--r-- 1 prance prance 0 01-30 20:47 ls2.txt
|
||||||
|
drwxr-xr-x 6 prance prance 4096 01-30 20:42 main
|
||||||
|
drwxr-xr-x 2 prance prance 4096 01-30 20:42 packaging
|
||||||
|
-rw-r--r-- 1 prance prance 13518 01-30 20:42 pom.xml
|
||||||
|
-rw-r--r-- 1 prance prance 3523 01-30 20:42 README.md
|
||||||
|
-rwxr-xr-x 1 prance prance 29532 01-30 20:42 refine
|
||||||
|
-rw-r--r-- 1 prance prance 7680 01-30 20:42 refine.bat
|
||||||
|
-rw-r--r-- 1 prance prance 1549 01-30 20:42 refine.ini
|
||||||
|
-rw-r--r-- 1 prance prance 729 01-30 20:42 SECURITY.md
|
||||||
|
drwxr-xr-x 5 prance prance 4096 01-30 20:42 server
|
||||||
|
-rw-r--r-- 1 prance prance 1099 01-30 20:42 settings.xml
|
||||||
|
drwxr-xr-x 2 prance prance 4096 01-30 20:46 tools
|
||||||
2
OpenRefine/ls3.txt
Normal file
2
OpenRefine/ls3.txt
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
drwxr-xr-x 2 prance prance 4096 01-30 20:47 build
|
||||||
|
drwxr-xr-x 2 prance prance 4096 01-30 20:46 tools
|
||||||
@ -1,296 +1,296 @@
|
|||||||
/*
|
/*
|
||||||
|
|
||||||
Copyright 2010, Google Inc.
|
Copyright 2010, Google Inc.
|
||||||
All rights reserved.
|
All rights reserved.
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
Redistribution and use in source and binary forms, with or without
|
||||||
modification, are permitted provided that the following conditions are
|
modification, are permitted provided that the following conditions are
|
||||||
met:
|
met:
|
||||||
|
|
||||||
* Redistributions of source code must retain the above copyright
|
* Redistributions of source code must retain the above copyright
|
||||||
notice, this list of conditions and the following disclaimer.
|
notice, this list of conditions and the following disclaimer.
|
||||||
* Redistributions in binary form must reproduce the above
|
* Redistributions in binary form must reproduce the above
|
||||||
copyright notice, this list of conditions and the following disclaimer
|
copyright notice, this list of conditions and the following disclaimer
|
||||||
in the documentation and/or other materials provided with the
|
in the documentation and/or other materials provided with the
|
||||||
distribution.
|
distribution.
|
||||||
* Neither the name of Google Inc. nor the names of its
|
* Neither the name of Google Inc. nor the names of its
|
||||||
contributors may be used to endorse or promote products derived from
|
contributors may be used to endorse or promote products derived from
|
||||||
this software without specific prior written permission.
|
this software without specific prior written permission.
|
||||||
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package com.google.refine.model;
|
package com.google.refine.model;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.google.refine.expr.ExpressionUtils;
|
import com.google.refine.expr.ExpressionUtils;
|
||||||
|
|
||||||
public class RecordModel {
|
public class RecordModel {
|
||||||
final static Logger logger = LoggerFactory.getLogger("RecordModel");
|
final static Logger logger = LoggerFactory.getLogger("RecordModel");
|
||||||
|
|
||||||
final static public class CellDependency {
|
final static public class CellDependency {
|
||||||
final public int rowIndex;
|
final public int rowIndex;
|
||||||
final public int cellIndex;
|
final public int cellIndex;
|
||||||
|
|
||||||
public CellDependency(int rowIndex, int cellIndex) {
|
public CellDependency(int rowIndex, int cellIndex) {
|
||||||
this.rowIndex = rowIndex;
|
this.rowIndex = rowIndex;
|
||||||
this.cellIndex = cellIndex;
|
this.cellIndex = cellIndex;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return rowIndex+","+cellIndex;
|
return rowIndex+","+cellIndex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
final static public class RowDependency {
|
final static public class RowDependency {
|
||||||
public int recordIndex;
|
public int recordIndex;
|
||||||
public CellDependency[] cellDependencies;
|
public CellDependency[] cellDependencies;
|
||||||
public List<Integer> contextRows;
|
public List<Integer> contextRows;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "Idx: "+recordIndex+" CellDeps: "+Arrays.toString(cellDependencies)+" Rows:"+contextRows;
|
return "Idx: "+recordIndex+" CellDeps: "+Arrays.toString(cellDependencies)+" Rows:"+contextRows;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected List<RowDependency> _rowDependencies;
|
protected List<RowDependency> _rowDependencies;
|
||||||
protected List<Record> _records;
|
protected List<Record> _records;
|
||||||
|
|
||||||
public RowDependency getRowDependency(int rowIndex) {
|
public RowDependency getRowDependency(int rowIndex) {
|
||||||
return _rowDependencies != null && rowIndex >= 0 && rowIndex < _rowDependencies.size() ?
|
return _rowDependencies != null && rowIndex >= 0 && rowIndex < _rowDependencies.size() ?
|
||||||
_rowDependencies.get(rowIndex) : null;
|
_rowDependencies.get(rowIndex) : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonIgnore
|
@JsonIgnore
|
||||||
public int getRecordCount() {
|
public int getRecordCount() {
|
||||||
return _records.size();
|
return _records.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
public Record getRecord(int recordIndex) {
|
public Record getRecord(int recordIndex) {
|
||||||
return _records != null && recordIndex >= 0 && recordIndex < _records.size() ?
|
return _records != null && recordIndex >= 0 && recordIndex < _records.size() ?
|
||||||
_records.get(recordIndex) : null;
|
_records.get(recordIndex) : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Record getRecordOfRow(int rowIndex) {
|
public Record getRecordOfRow(int rowIndex) {
|
||||||
RowDependency rd = getRowDependency(rowIndex);
|
RowDependency rd = getRowDependency(rowIndex);
|
||||||
if (rd != null) {
|
if (rd != null) {
|
||||||
if (rd.recordIndex < 0) {
|
if (rd.recordIndex < 0) {
|
||||||
rd = getRowDependency(rd.contextRows.get(0));
|
rd = getRowDependency(rd.contextRows.get(0));
|
||||||
}
|
}
|
||||||
return getRecord(rd.recordIndex);
|
return getRecord(rd.recordIndex);
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty("hasRecords")
|
@JsonProperty("hasRecords")
|
||||||
public boolean hasRecords() {
|
public boolean hasRecords() {
|
||||||
return _records != null && _rowDependencies != null &&
|
return _records != null && _rowDependencies != null &&
|
||||||
_records.size() < _rowDependencies.size();
|
_records.size() < _rowDependencies.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
static protected class KeyedGroup {
|
static protected class KeyedGroup {
|
||||||
int[] cellIndices;
|
int[] cellIndices;
|
||||||
int keyCellIndex;
|
int keyCellIndex;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
StringBuffer sb = new StringBuffer();
|
StringBuffer sb = new StringBuffer();
|
||||||
for (int i:cellIndices) {
|
for (int i:cellIndices) {
|
||||||
sb.append(i).append(',');
|
sb.append(i).append(',');
|
||||||
}
|
}
|
||||||
return "key: " + keyCellIndex + " cells: " + sb.toString();
|
return "key: " + keyCellIndex + " cells: " + sb.toString();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
synchronized public void update(Project project) {
|
synchronized public void update(Project project) {
|
||||||
synchronized (project) {
|
synchronized (project) {
|
||||||
List<Row> rows = project.rows;
|
List<Row> rows = project.rows;
|
||||||
int rowCount = rows.size();
|
int rowCount = rows.size();
|
||||||
|
|
||||||
ColumnModel columnModel = project.columnModel;
|
ColumnModel columnModel = project.columnModel;
|
||||||
List<KeyedGroup> keyedGroups = computeKeyedGroups(columnModel);
|
List<KeyedGroup> keyedGroups = computeKeyedGroups(columnModel);
|
||||||
int groupCount = keyedGroups.size();
|
int groupCount = keyedGroups.size();
|
||||||
|
|
||||||
int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()];
|
int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()];
|
||||||
for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) {
|
for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) {
|
||||||
lastNonBlankRowsByGroup[i] = -1;
|
lastNonBlankRowsByGroup[i] = -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
_rowDependencies = new ArrayList<RowDependency>(rowCount);
|
_rowDependencies = new ArrayList<RowDependency>(rowCount);
|
||||||
|
|
||||||
int recordIndex = 0;
|
int recordIndex = 0;
|
||||||
for (int r = 0; r < rowCount; r++) {
|
for (int r = 0; r < rowCount; r++) {
|
||||||
Row row = rows.get(r);
|
Row row = rows.get(r);
|
||||||
RowDependency rowDependency = new RowDependency();
|
RowDependency rowDependency = new RowDependency();
|
||||||
|
|
||||||
for (int g = 0; g < groupCount; g++) {
|
for (int g = 0; g < groupCount; g++) {
|
||||||
KeyedGroup group = keyedGroups.get(g);
|
KeyedGroup group = keyedGroups.get(g);
|
||||||
|
|
||||||
if (!ExpressionUtils.isNonBlankData(row.getCellValue(keyedGroups.get(0).keyCellIndex)) &&
|
if (!ExpressionUtils.isNonBlankData(row.getCellValue(keyedGroups.get(0).keyCellIndex)) &&
|
||||||
!ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) {
|
!ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) {
|
||||||
int contextRowIndex = lastNonBlankRowsByGroup[g];
|
int contextRowIndex = lastNonBlankRowsByGroup[g];
|
||||||
if (contextRowIndex >= 0) {
|
if (contextRowIndex >= 0) {
|
||||||
for (int dependentCellIndex : group.cellIndices) {
|
for (int dependentCellIndex : group.cellIndices) {
|
||||||
if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) {
|
if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) {
|
||||||
setRowDependency(
|
setRowDependency(
|
||||||
project,
|
project,
|
||||||
rowDependency,
|
rowDependency,
|
||||||
dependentCellIndex,
|
dependentCellIndex,
|
||||||
contextRowIndex,
|
contextRowIndex,
|
||||||
group.keyCellIndex
|
group.keyCellIndex
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
lastNonBlankRowsByGroup[g] = r;
|
lastNonBlankRowsByGroup[g] = r;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (rowDependency.cellDependencies != null && rowDependency.cellDependencies.length > 0) {
|
if (rowDependency.cellDependencies != null && rowDependency.cellDependencies.length > 0) {
|
||||||
rowDependency.recordIndex = -1;
|
rowDependency.recordIndex = -1;
|
||||||
rowDependency.contextRows = new ArrayList<Integer>();
|
rowDependency.contextRows = new ArrayList<Integer>();
|
||||||
for (CellDependency cd : rowDependency.cellDependencies) {
|
for (CellDependency cd : rowDependency.cellDependencies) {
|
||||||
if (cd != null) {
|
if (cd != null) {
|
||||||
rowDependency.contextRows.add(cd.rowIndex);
|
rowDependency.contextRows.add(cd.rowIndex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Collections.sort(rowDependency.contextRows);
|
Collections.sort(rowDependency.contextRows);
|
||||||
} else {
|
} else {
|
||||||
rowDependency.recordIndex = recordIndex++;
|
rowDependency.recordIndex = recordIndex++;
|
||||||
}
|
}
|
||||||
|
|
||||||
_rowDependencies.add(rowDependency);
|
_rowDependencies.add(rowDependency);
|
||||||
}
|
}
|
||||||
|
|
||||||
_records = new ArrayList<Record>(recordIndex);
|
_records = new ArrayList<Record>(recordIndex);
|
||||||
if (recordIndex > 0) {
|
if (recordIndex > 0) {
|
||||||
recordIndex = 0;
|
recordIndex = 0;
|
||||||
|
|
||||||
int recordRowIndex = 0;
|
int recordRowIndex = 0;
|
||||||
for (int r = 1; r < rowCount; r++) {
|
for (int r = 1; r < rowCount; r++) {
|
||||||
RowDependency rd = _rowDependencies.get(r);
|
RowDependency rd = _rowDependencies.get(r);
|
||||||
if (rd.recordIndex >= 0) {
|
if (rd.recordIndex >= 0) {
|
||||||
_records.add(new Record(recordRowIndex, r, recordIndex++));
|
_records.add(new Record(recordRowIndex, r, recordIndex++));
|
||||||
|
|
||||||
recordIndex = rd.recordIndex;
|
recordIndex = rd.recordIndex;
|
||||||
recordRowIndex = r;
|
recordRowIndex = r;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
_records.add(new Record(recordRowIndex, rowCount, recordIndex++));
|
_records.add(new Record(recordRowIndex, rowCount, recordIndex++));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected List<KeyedGroup> computeKeyedGroups(ColumnModel columnModel) {
|
protected List<KeyedGroup> computeKeyedGroups(ColumnModel columnModel) {
|
||||||
List<KeyedGroup> keyedGroups = new ArrayList<KeyedGroup>();
|
List<KeyedGroup> keyedGroups = new ArrayList<KeyedGroup>();
|
||||||
|
|
||||||
addRootKeyedGroup(columnModel, keyedGroups);
|
addRootKeyedGroup(columnModel, keyedGroups);
|
||||||
|
|
||||||
for (ColumnGroup group : columnModel.columnGroups) {
|
for (ColumnGroup group : columnModel.columnGroups) {
|
||||||
if (group.keyColumnIndex >= 0) {
|
if (group.keyColumnIndex >= 0) {
|
||||||
KeyedGroup keyedGroup = new KeyedGroup();
|
KeyedGroup keyedGroup = new KeyedGroup();
|
||||||
keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex();
|
keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex();
|
||||||
keyedGroup.cellIndices = new int[group.columnSpan - 1];
|
keyedGroup.cellIndices = new int[group.columnSpan - 1];
|
||||||
|
|
||||||
int c = 0;
|
int c = 0;
|
||||||
for (int i = 0; i < group.columnSpan; i++) {
|
for (int i = 0; i < group.columnSpan; i++) {
|
||||||
int columnIndex = group.startColumnIndex + i;
|
int columnIndex = group.startColumnIndex + i;
|
||||||
if (columnIndex != group.keyColumnIndex && columnIndex < columnModel.columns.size()) {
|
if (columnIndex != group.keyColumnIndex && columnIndex < columnModel.columns.size()) {
|
||||||
int cellIndex = columnModel.columns.get(columnIndex).getCellIndex();
|
int cellIndex = columnModel.columns.get(columnIndex).getCellIndex();
|
||||||
keyedGroup.cellIndices[c++] = cellIndex;
|
keyedGroup.cellIndices[c++] = cellIndex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
keyedGroups.add(keyedGroup);
|
keyedGroups.add(keyedGroup);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Collections.sort(keyedGroups, new Comparator<KeyedGroup>() {
|
Collections.sort(keyedGroups, new Comparator<KeyedGroup>() {
|
||||||
@Override
|
@Override
|
||||||
public int compare(KeyedGroup o1, KeyedGroup o2) {
|
public int compare(KeyedGroup o1, KeyedGroup o2) {
|
||||||
return o2.cellIndices.length - o1.cellIndices.length; // larger groups first
|
return o2.cellIndices.length - o1.cellIndices.length; // larger groups first
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
dumpKeyedGroups(keyedGroups, columnModel); // for debug
|
dumpKeyedGroups(keyedGroups, columnModel); // for debug
|
||||||
|
|
||||||
return keyedGroups;
|
return keyedGroups;
|
||||||
}
|
}
|
||||||
|
|
||||||
// debugging helper
|
// debugging helper
|
||||||
private void dumpKeyedGroups(List<KeyedGroup> groups, ColumnModel columnModel) {
|
private void dumpKeyedGroups(List<KeyedGroup> groups, ColumnModel columnModel) {
|
||||||
for (KeyedGroup g : groups) {
|
for (KeyedGroup g : groups) {
|
||||||
String keyColName = columnModel.getColumnByCellIndex(g.keyCellIndex).getName();
|
String keyColName = columnModel.getColumnByCellIndex(g.keyCellIndex).getName();
|
||||||
StringBuffer sb = new StringBuffer();
|
StringBuffer sb = new StringBuffer();
|
||||||
for (int ci : g.cellIndices) {
|
for (int ci : g.cellIndices) {
|
||||||
Column col = columnModel.getColumnByCellIndex(ci);
|
Column col = columnModel.getColumnByCellIndex(ci);
|
||||||
if (col != null) {
|
if (col != null) {
|
||||||
// Old projects have col 0 slot empty
|
// Old projects have col 0 slot empty
|
||||||
sb.append(col.getName()).append(',');
|
sb.append(col.getName()).append(',');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
logger.trace("KeyedGroup " + keyColName + "::" + sb.toString());
|
logger.trace("KeyedGroup " + keyColName + "::" + sb.toString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void addRootKeyedGroup(ColumnModel columnModel, List<KeyedGroup> keyedGroups) {
|
protected void addRootKeyedGroup(ColumnModel columnModel, List<KeyedGroup> keyedGroups) {
|
||||||
int count = columnModel.getMaxCellIndex() + 1;
|
int count = columnModel.getMaxCellIndex() + 1;
|
||||||
if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) {
|
if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) {
|
||||||
KeyedGroup rootKeyedGroup = new KeyedGroup();
|
KeyedGroup rootKeyedGroup = new KeyedGroup();
|
||||||
|
|
||||||
rootKeyedGroup.cellIndices = new int[count - 1];
|
rootKeyedGroup.cellIndices = new int[count - 1];
|
||||||
rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex();
|
rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex();
|
||||||
|
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
if (i < rootKeyedGroup.keyCellIndex) {
|
if (i < rootKeyedGroup.keyCellIndex) {
|
||||||
rootKeyedGroup.cellIndices[i] = i;
|
rootKeyedGroup.cellIndices[i] = i;
|
||||||
} else if (i > rootKeyedGroup.keyCellIndex) {
|
} else if (i > rootKeyedGroup.keyCellIndex) {
|
||||||
rootKeyedGroup.cellIndices[i - 1] = i;
|
rootKeyedGroup.cellIndices[i - 1] = i;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
keyedGroups.add(rootKeyedGroup);
|
keyedGroups.add(rootKeyedGroup);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void setRowDependency(
|
protected void setRowDependency(
|
||||||
Project project,
|
Project project,
|
||||||
RowDependency rowDependency,
|
RowDependency rowDependency,
|
||||||
int cellIndex,
|
int cellIndex,
|
||||||
int contextRowIndex,
|
int contextRowIndex,
|
||||||
int contextCellIndex
|
int contextCellIndex
|
||||||
) {
|
) {
|
||||||
if (rowDependency.cellDependencies == null) {
|
if (rowDependency.cellDependencies == null) {
|
||||||
int count = project.columnModel.getMaxCellIndex() + 1;
|
int count = project.columnModel.getMaxCellIndex() + 1;
|
||||||
|
|
||||||
rowDependency.cellDependencies = new CellDependency[count];
|
rowDependency.cellDependencies = new CellDependency[count];
|
||||||
}
|
}
|
||||||
|
|
||||||
rowDependency.cellDependencies[cellIndex] =
|
rowDependency.cellDependencies[cellIndex] =
|
||||||
new CellDependency(contextRowIndex, contextCellIndex);
|
new CellDependency(contextRowIndex, contextCellIndex);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,84 +1,84 @@
|
|||||||
/*
|
/*
|
||||||
|
|
||||||
Copyright 2010, Google Inc.
|
Copyright 2010, Google Inc.
|
||||||
All rights reserved.
|
All rights reserved.
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
Redistribution and use in source and binary forms, with or without
|
||||||
modification, are permitted provided that the following conditions are
|
modification, are permitted provided that the following conditions are
|
||||||
met:
|
met:
|
||||||
|
|
||||||
* Redistributions of source code must retain the above copyright
|
* Redistributions of source code must retain the above copyright
|
||||||
notice, this list of conditions and the following disclaimer.
|
notice, this list of conditions and the following disclaimer.
|
||||||
* Redistributions in binary form must reproduce the above
|
* Redistributions in binary form must reproduce the above
|
||||||
copyright notice, this list of conditions and the following disclaimer
|
copyright notice, this list of conditions and the following disclaimer
|
||||||
in the documentation and/or other materials provided with the
|
in the documentation and/or other materials provided with the
|
||||||
distribution.
|
distribution.
|
||||||
* Neither the name of Google Inc. nor the names of its
|
* Neither the name of Google Inc. nor the names of its
|
||||||
contributors may be used to endorse or promote products derived from
|
contributors may be used to endorse or promote products derived from
|
||||||
this software without specific prior written permission.
|
this software without specific prior written permission.
|
||||||
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package com.google.refine.model.changes;
|
package com.google.refine.model.changes;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.Writer;
|
import java.io.Writer;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
import com.google.refine.model.Cell;
|
import com.google.refine.model.Cell;
|
||||||
import com.google.refine.util.Pool;
|
import com.google.refine.util.Pool;
|
||||||
|
|
||||||
public class CellAtRowCellIndex {
|
public class CellAtRowCellIndex {
|
||||||
|
|
||||||
final public int row;
|
final public int row;
|
||||||
final public int cellIndex;
|
final public int cellIndex;
|
||||||
final public Cell cell;
|
final public Cell cell;
|
||||||
final static private Pattern semicolonPattern = Pattern.compile(";");
|
final static private Pattern semicolonPattern = Pattern.compile(";");
|
||||||
|
|
||||||
public CellAtRowCellIndex(int row, int cellIndex, Cell cell) {
|
public CellAtRowCellIndex(int row, int cellIndex, Cell cell) {
|
||||||
this.row = row;
|
this.row = row;
|
||||||
this.cell = cell;
|
this.cell = cell;
|
||||||
this.cellIndex = cellIndex;
|
this.cellIndex = cellIndex;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void save(Writer writer, Properties options) throws IOException {
|
public void save(Writer writer, Properties options) throws IOException {
|
||||||
writer.write(Integer.toString(row));
|
writer.write(Integer.toString(row));
|
||||||
writer.write(';');
|
writer.write(';');
|
||||||
writer.write(Integer.toString(cellIndex));
|
writer.write(Integer.toString(cellIndex));
|
||||||
writer.write(';');
|
writer.write(';');
|
||||||
if (cell != null) {
|
if (cell != null) {
|
||||||
cell.save(writer, options);
|
cell.save(writer, options);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static public CellAtRowCellIndex load(String s, Pool pool) throws Exception {
|
static public CellAtRowCellIndex load(String s, Pool pool) throws Exception {
|
||||||
|
|
||||||
Matcher m = semicolonPattern.matcher(s);
|
Matcher m = semicolonPattern.matcher(s);
|
||||||
|
|
||||||
m.find();
|
m.find();
|
||||||
int semicolon = m.start();
|
int semicolon = m.start();
|
||||||
m.find();
|
m.find();
|
||||||
int nextSemicolon = m.start();
|
int nextSemicolon = m.start();
|
||||||
|
|
||||||
int row = Integer.parseInt(s.substring(0, semicolon));
|
int row = Integer.parseInt(s.substring(0, semicolon));
|
||||||
int cellIndex = Integer.parseInt(s.substring(semicolon + 1, nextSemicolon));
|
int cellIndex = Integer.parseInt(s.substring(semicolon + 1, nextSemicolon));
|
||||||
Cell cell = nextSemicolon < s.length() - 1 ? Cell.loadStreaming(s.substring(nextSemicolon + 1), pool)
|
Cell cell = nextSemicolon < s.length() - 1 ? Cell.loadStreaming(s.substring(nextSemicolon + 1), pool)
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
return new CellAtRowCellIndex(row, cellIndex, cell);
|
return new CellAtRowCellIndex(row, cellIndex, cell);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,69 +1,69 @@
|
|||||||
/*
|
/*
|
||||||
|
|
||||||
Copyright 2010, Google Inc.
|
Copyright 2010, Google Inc.
|
||||||
All rights reserved.
|
All rights reserved.
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
Redistribution and use in source and binary forms, with or without
|
||||||
modification, are permitted provided that the following conditions are
|
modification, are permitted provided that the following conditions are
|
||||||
met:
|
met:
|
||||||
|
|
||||||
* Redistributions of source code must retain the above copyright
|
* Redistributions of source code must retain the above copyright
|
||||||
notice, this list of conditions and the following disclaimer.
|
notice, this list of conditions and the following disclaimer.
|
||||||
* Redistributions in binary form must reproduce the above
|
* Redistributions in binary form must reproduce the above
|
||||||
copyright notice, this list of conditions and the following disclaimer
|
copyright notice, this list of conditions and the following disclaimer
|
||||||
in the documentation and/or other materials provided with the
|
in the documentation and/or other materials provided with the
|
||||||
distribution.
|
distribution.
|
||||||
* Neither the name of Google Inc. nor the names of its
|
* Neither the name of Google Inc. nor the names of its
|
||||||
contributors may be used to endorse or promote products derived from
|
contributors may be used to endorse or promote products derived from
|
||||||
this software without specific prior written permission.
|
this software without specific prior written permission.
|
||||||
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
class Facet {
|
class Facet {
|
||||||
constructor(div, config, options) {
|
constructor(div, config, options) {
|
||||||
this._div = div;
|
this._div = div;
|
||||||
this._config = config;
|
this._config = config;
|
||||||
this._options = options || {};
|
this._options = options || {};
|
||||||
this._minimizeState = false;
|
this._minimizeState = false;
|
||||||
|
|
||||||
Refine.showLeftPanel();
|
Refine.showLeftPanel();
|
||||||
};
|
};
|
||||||
|
|
||||||
_minimize() {
|
_minimize() {
|
||||||
if(!this._minimizeState) {
|
if(!this._minimizeState) {
|
||||||
this._div.addClass("facet-state-minimize");
|
this._div.addClass("facet-state-minimize");
|
||||||
} else {
|
} else {
|
||||||
this._div.removeClass("facet-state-minimize");
|
this._div.removeClass("facet-state-minimize");
|
||||||
}
|
}
|
||||||
|
|
||||||
this._minimizeState = !this._minimizeState;
|
this._minimizeState = !this._minimizeState;
|
||||||
};
|
};
|
||||||
|
|
||||||
_remove() {
|
_remove() {
|
||||||
ui.browsingEngine.removeFacet(this);
|
ui.browsingEngine.removeFacet(this);
|
||||||
|
|
||||||
this._div = null;
|
this._div = null;
|
||||||
this._config = null;
|
this._config = null;
|
||||||
|
|
||||||
this._selection = null;
|
this._selection = null;
|
||||||
this._blankChoice = null;
|
this._blankChoice = null;
|
||||||
this._errorChoice = null;
|
this._errorChoice = null;
|
||||||
this._data = null;
|
this._data = null;
|
||||||
this._options = null;
|
this._options = null;
|
||||||
};
|
};
|
||||||
|
|
||||||
dispose() {
|
dispose() {
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|||||||
@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|||||||
font-size: 110%;
|
font-size: 110%;
|
||||||
color: @near_black;
|
color: @near_black;
|
||||||
white-space: nowrap;
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
#left-panel-body {
|
#left-panel-body {
|
||||||
margin-left: @padding_tight;
|
margin-left: @padding_tight;
|
||||||
|
|||||||
@ -1,104 +1,104 @@
|
|||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
* Copyright (C) 2020, OpenRefine contributors
|
* Copyright (C) 2020, OpenRefine contributors
|
||||||
* All rights reserved.
|
* All rights reserved.
|
||||||
*
|
*
|
||||||
* Redistribution and use in source and binary forms, with or without
|
* Redistribution and use in source and binary forms, with or without
|
||||||
* modification, are permitted provided that the following conditions are met:
|
* modification, are permitted provided that the following conditions are met:
|
||||||
*
|
*
|
||||||
* 1. Redistributions of source code must retain the above copyright notice,
|
* 1. Redistributions of source code must retain the above copyright notice,
|
||||||
* this list of conditions and the following disclaimer.
|
* this list of conditions and the following disclaimer.
|
||||||
*
|
*
|
||||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||||
* this list of conditions and the following disclaimer in the documentation
|
* this list of conditions and the following disclaimer in the documentation
|
||||||
* and/or other materials provided with the distribution.
|
* and/or other materials provided with the distribution.
|
||||||
*
|
*
|
||||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||||
* POSSIBILITY OF SUCH DAMAGE.
|
* POSSIBILITY OF SUCH DAMAGE.
|
||||||
******************************************************************************/
|
******************************************************************************/
|
||||||
|
|
||||||
package com.google.refine;
|
package com.google.refine;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
import javax.servlet.ServletException;
|
import javax.servlet.ServletException;
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
import javax.servlet.http.HttpServletResponse;
|
import javax.servlet.http.HttpServletResponse;
|
||||||
import org.eclipse.jetty.server.handler.HandlerWrapper;
|
import org.eclipse.jetty.server.handler.HandlerWrapper;
|
||||||
import org.eclipse.jetty.server.Request;
|
import org.eclipse.jetty.server.Request;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Validate the Host header of the HTTP request to see if it matches either a loopback IP
|
* Validate the Host header of the HTTP request to see if it matches either a loopback IP
|
||||||
* address, localhost or an explicitly specified hostname. This is required to avoid DNS
|
* address, localhost or an explicitly specified hostname. This is required to avoid DNS
|
||||||
* rebinding attacks against users running OpenRefine on their desktop computers.
|
* rebinding attacks against users running OpenRefine on their desktop computers.
|
||||||
*/
|
*/
|
||||||
class ValidateHostHandler extends HandlerWrapper {
|
class ValidateHostHandler extends HandlerWrapper {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Matches:
|
* Matches:
|
||||||
* - addresses in the 127.0.0.0/8 subnet
|
* - addresses in the 127.0.0.0/8 subnet
|
||||||
* - IPv4-mapped addresses in the ::ffff:7f00:00/104 subnet
|
* - IPv4-mapped addresses in the ::ffff:7f00:00/104 subnet
|
||||||
* - different representations of ::1
|
* - different representations of ::1
|
||||||
* - localhost
|
* - localhost
|
||||||
* Matching is a little fuzzy to simplify the regular expression - it expects the Host
|
* Matching is a little fuzzy to simplify the regular expression - it expects the Host
|
||||||
* header to be well-formed. Some invalid addresses would be accepted, for example:
|
* header to be well-formed. Some invalid addresses would be accepted, for example:
|
||||||
* - 127.6..64.245
|
* - 127.6..64.245
|
||||||
* - 0::0:::0:00:1
|
* - 0::0:::0:00:1
|
||||||
* This is not a problem however, as these are not valid DNS names either, and should
|
* This is not a problem however, as these are not valid DNS names either, and should
|
||||||
* never be sent by a well-behaved browser - and validating the host header only ever
|
* never be sent by a well-behaved browser - and validating the host header only ever
|
||||||
* helps if the browser works as expected and cannot be used to fake the Host header.
|
* helps if the browser works as expected and cannot be used to fake the Host header.
|
||||||
*/
|
*/
|
||||||
static private final Pattern LOOPBACK_PATTERN = Pattern
|
static private final Pattern LOOPBACK_PATTERN = Pattern
|
||||||
.compile("^(?:127\\.[0-9\\.]*|\\[[0\\:]*\\:(?:ffff\\:7f[0-9a-f]{2}:[0-9a-f]{1,4}|0{0,3}1)\\]|localhost)(?:\\:[0-9]+)?$", Pattern.CASE_INSENSITIVE);
|
.compile("^(?:127\\.[0-9\\.]*|\\[[0\\:]*\\:(?:ffff\\:7f[0-9a-f]{2}:[0-9a-f]{1,4}|0{0,3}1)\\]|localhost)(?:\\:[0-9]+)?$", Pattern.CASE_INSENSITIVE);
|
||||||
|
|
||||||
private String expectedHost;
|
private String expectedHost;
|
||||||
|
|
||||||
public ValidateHostHandler(String expectedHost) {
|
public ValidateHostHandler(String expectedHost) {
|
||||||
this.expectedHost = expectedHost;
|
this.expectedHost = expectedHost;
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isValidHost(String host) {
|
public boolean isValidHost(String host) {
|
||||||
|
|
||||||
// Allow loopback IPv4 and IPv6 addresses, as well as localhost
|
// Allow loopback IPv4 and IPv6 addresses, as well as localhost
|
||||||
if (LOOPBACK_PATTERN.matcher(host).find()) {
|
if (LOOPBACK_PATTERN.matcher(host).find()) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Strip port from hostname - for IPv6 addresses, if
|
// Strip port from hostname - for IPv6 addresses, if
|
||||||
// they end with a bracket, then there is no port
|
// they end with a bracket, then there is no port
|
||||||
int index = host.lastIndexOf(':');
|
int index = host.lastIndexOf(':');
|
||||||
if (index > 0 && !host.endsWith("]")) {
|
if (index > 0 && !host.endsWith("]")) {
|
||||||
host = host.substring(0, index);
|
host = host.substring(0, index);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Strip brackets from IPv6 addresses
|
// Strip brackets from IPv6 addresses
|
||||||
if (host.startsWith("[") && host.endsWith("]")) {
|
if (host.startsWith("[") && host.endsWith("]")) {
|
||||||
host = host.substring(1, host.length() - 2);
|
host = host.substring(1, host.length() - 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Allow only if stripped hostname matches expected hostname
|
// Allow only if stripped hostname matches expected hostname
|
||||||
return expectedHost.equalsIgnoreCase(host);
|
return expectedHost.equalsIgnoreCase(host);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
|
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
|
||||||
throws IOException, ServletException {
|
throws IOException, ServletException {
|
||||||
String host = request.getHeader("Host");
|
String host = request.getHeader("Host");
|
||||||
if (isValidHost(host)) {
|
if (isValidHost(host)) {
|
||||||
super.handle(target, baseRequest, request, response);
|
super.handle(target, baseRequest, request, response);
|
||||||
} else {
|
} else {
|
||||||
// Return HTTP 404 Not Found, since we are
|
// Return HTTP 404 Not Found, since we are
|
||||||
// not serving content for the requested URL
|
// not serving content for the requested URL
|
||||||
response.sendError(HttpServletResponse.SC_NOT_FOUND, "Invalid hostname");
|
response.sendError(HttpServletResponse.SC_NOT_FOUND, "Invalid hostname");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
Loading…
Reference in New Issue
Block a user