change OR version
This commit is contained in:
parent
8694374775
commit
2a7e2e66b5
1
OpenRefine/.github/FUNDING.yml
vendored
1
OpenRefine/.github/FUNDING.yml
vendored
@ -1 +0,0 @@
|
||||
github: OpenRefine
|
40
OpenRefine/.github/ISSUE_TEMPLATE/bug_report.md
vendored
40
OpenRefine/.github/ISSUE_TEMPLATE/bug_report.md
vendored
@ -1,40 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve OpenRefine
|
||||
title: ''
|
||||
labels: bug, to be reviewed
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
<!-- Describe the bug - Please add a clear and concise description of the bug above this line. You can delete this line if you want. It will be hidden in the final bug report -->
|
||||
|
||||
### To Reproduce
|
||||
Steps to reproduce the behavior:
|
||||
1. First, do ...
|
||||
2. Then, ...
|
||||
3. Finally, ...
|
||||
|
||||
|
||||
|
||||
### Current Results
|
||||
<!-- What results occurred or were shown. -->
|
||||
|
||||
### Expected Behavior
|
||||
<!-- A clear and concise description of what you expected to happen or to show. -->
|
||||
|
||||
### Screenshots
|
||||
<!-- If applicable, add screenshots to help explain your problem. -->
|
||||
|
||||
### Versions<!-- (please complete the following information)-->
|
||||
- Operating System: <!-- e.g. iOS, Windows 10, Linux, Ubuntu 18.04 -->
|
||||
- Browser Version: <!-- e.g. Chrome 19, Firefox 61, Safari, NOTE: OpenRefine does not support IE but works OK in most cases -->
|
||||
- JRE or JDK Version: <!-- output of "java -version" e.g. JRE 1.8.0_181 -->
|
||||
- OpenRefine: <!-- e.g. OpenRefine 3.0 Beta] -->
|
||||
|
||||
### Datasets
|
||||
<!-- If you are allowed and are OK with making your data public, it would be awesome if you can include or attach the data causing the issue or a URL pointing to where the data is.
|
||||
If you are concerned about keeping your data private, ping us on our [mailing list](https://groups.google.com/forum/#!forum/openrefine) -->
|
||||
|
||||
### Additional context
|
||||
<!-- Add any other context about the problem here. -->
|
8
OpenRefine/.github/ISSUE_TEMPLATE/config.yml
vendored
8
OpenRefine/.github/ISSUE_TEMPLATE/config.yml
vendored
@ -1,8 +0,0 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Ask a question (mailing list)
|
||||
url: https://groups.google.com/d/forum/openrefine
|
||||
about: Please ask and answer questions here.
|
||||
- name: Gitter chat
|
||||
url: https://gitter.im/OpenRefine/OpenRefine
|
||||
about: General and developer discussions
|
@ -1,19 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for OpenRefine
|
||||
title: ''
|
||||
labels: enhancement, to be reviewed
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
<!-- Please provide a clear and concise description of your problem or unsatisfied needs. Ex. I'm always frustrated when [...] or, It would be easier if OpenRefine did [...]. This comment can be deleted, if desired, but it will be hidden in your final submission. Please make sure that your new text is outside the enclosing angle brackets. -->
|
||||
|
||||
### Proposed solution
|
||||
<!-- If you have a proposal for how this need could be met, please provide a clear and concise description of what you want to happen. -->
|
||||
|
||||
### Alternatives considered
|
||||
<!-- If there alternative solutions that you have considered or think should be considered, please list them here -->
|
||||
|
||||
### Additional context
|
||||
<!-- Add any other context or screenshots about the feature request here. -->
|
5
OpenRefine/.github/SUPPORT.md
vendored
5
OpenRefine/.github/SUPPORT.md
vendored
@ -1,5 +0,0 @@
|
||||
If you are having trouble with OpenRefine we suggest the following steps:
|
||||
|
||||
1. Read through our [FAQ (frequently Asked Questions)](https://github.com/OpenRefine/OpenRefine/wiki/FAQ)
|
||||
2. Search for similar issues in our community email list archives: http://groups.google.com/group/openrefine/
|
||||
3. Send an email to our community mailing list: openrefine@googlegroups.com
|
3
OpenRefine/.github/autolabeler.yml
vendored
3
OpenRefine/.github/autolabeler.yml
vendored
@ -1,3 +0,0 @@
|
||||
documentation: ["/docs"]
|
||||
"current docs": ["/docs/docs"]
|
||||
"historical docs": ["/docs/versioned_docs"]
|
45
OpenRefine/.github/dependabot.yml
vendored
45
OpenRefine/.github/dependabot.yml
vendored
@ -1,45 +0,0 @@
|
||||
# Documentation for all configuration options:
|
||||
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
# For openrefine java deps
|
||||
- package-ecosystem: maven
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: daily
|
||||
open-pull-requests-limit: 10
|
||||
ignore:
|
||||
- dependency-name: com.thoughtworks.xstream:xstream
|
||||
versions:
|
||||
- "> 1.4.12"
|
||||
- "< 2"
|
||||
|
||||
# Same, on 4.0 branch
|
||||
- package-ecosystem: maven
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: daily
|
||||
open-pull-requests-limit: 10
|
||||
ignore:
|
||||
- dependency-name: com.thoughtworks.xstream:xstream
|
||||
versions:
|
||||
- "> 1.4.12"
|
||||
- "< 2"
|
||||
target-branch: 4.0
|
||||
|
||||
# For documentation website
|
||||
- package-ecosystem: "npm" # For Yarn
|
||||
directory: "docs/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
# For github actions
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
# For cypress test_suite
|
||||
- package-ecosystem: "npm"
|
||||
directory: "main/tests/cypress"
|
||||
schedule:
|
||||
interval: "daily"
|
6
OpenRefine/.github/pull_request_template.md
vendored
6
OpenRefine/.github/pull_request_template.md
vendored
@ -1,6 +0,0 @@
|
||||
Fixes #{issue number here}
|
||||
|
||||
Changes proposed in this pull request:
|
||||
-
|
||||
-
|
||||
-
|
26
OpenRefine/.github/workflows/label_transfer.yml
vendored
26
OpenRefine/.github/workflows/label_transfer.yml
vendored
@ -1,26 +0,0 @@
|
||||
|
||||
name: Copy labels from issue to pull request
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, edited]
|
||||
|
||||
jobs:
|
||||
transfer_tags:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.9
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip setuptools
|
||||
pip install -r .github/workflows/label_transfer/requirements.txt
|
||||
pip freeze
|
||||
- name: Run Python label transfer script
|
||||
run: python .github/workflows/label_transfer/script.py ${{ github.event.number }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_REPO: ${{ github.repository }}
|
@ -1,2 +0,0 @@
|
||||
requests
|
||||
lxml
|
@ -1,76 +0,0 @@
|
||||
import requests
|
||||
import os
|
||||
from lxml import html
|
||||
import sys
|
||||
import json
|
||||
|
||||
# Config
|
||||
|
||||
# list of labels that should not be transferred to PRs
|
||||
do_not_transfer = [
|
||||
'good first issue',
|
||||
'good second issue',
|
||||
'imported from old code repo',
|
||||
'help wanted',
|
||||
'duplicate',
|
||||
'invalid',
|
||||
'question',
|
||||
'to be reviewed',
|
||||
]
|
||||
|
||||
# Internal
|
||||
|
||||
repo = os.environ.get('GITHUB_REPO')
|
||||
github_token = os.environ.get('GITHUB_TOKEN')
|
||||
|
||||
headers = {
|
||||
'Accept': 'application/vnd.github.v3+json',
|
||||
}
|
||||
|
||||
if github_token:
|
||||
headers['Authorization'] = 'Bearer '+github_token
|
||||
|
||||
def get_linked_issues(pr_number):
|
||||
"""
|
||||
Given a PR number, extract all the linked issue numbers from it.
|
||||
Sadly this is not supported by the API yet, so we just scrape the web UI.
|
||||
"""
|
||||
url = f'https://github.com/{repo}/pull/{pr_number}'
|
||||
page = requests.get(url)
|
||||
page.raise_for_status()
|
||||
parsed = html.document_fromstring(page.text)
|
||||
matches = parsed.xpath('//form/div[@class="css-truncate my-1"]/a')
|
||||
for match in matches:
|
||||
yield int(match.attrib['href'].split('/')[-1])
|
||||
|
||||
def get_issue_labels(issue_number):
|
||||
"""
|
||||
Returns all the labels in a given issue / PR
|
||||
"""
|
||||
url = f'https://api.github.com/repos/{repo}/issues/{issue_number}/labels'
|
||||
response = requests.get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
return [ tag['name'] for tag in response.json() ]
|
||||
|
||||
def transfer_issue_labels(pr_number):
|
||||
"""
|
||||
Transfers labels from all the linked issues to the PR
|
||||
"""
|
||||
linked_issues = get_linked_issues(pr_number)
|
||||
all_labels = [ label for issue in linked_issues for label in get_issue_labels(issue) ]
|
||||
to_transfer = [ label for label in all_labels if label not in do_not_transfer ]
|
||||
current_labels = get_issue_labels(pr_number)
|
||||
missing_labels = [ label for label in to_transfer if label not in current_labels ]
|
||||
if not missing_labels:
|
||||
return
|
||||
new_labels = current_labels + missing_labels
|
||||
url = f'https://api.github.com/repos/{repo}/issues/{pr_number}/labels'
|
||||
print(f'adding {missing_labels} to PR #{pr_number}')
|
||||
if not github_token:
|
||||
print('no GITHUB_TOKEN, skipping')
|
||||
else:
|
||||
resp = requests.put(url, headers=headers, data=json.dumps({'labels':new_labels}))
|
||||
resp.raise_for_status()
|
||||
|
||||
if __name__ == '__main__':
|
||||
transfer_issue_labels(sys.argv[1])
|
171
OpenRefine/.github/workflows/pull_request.yml
vendored
171
OpenRefine/.github/workflows/pull_request.yml
vendored
@ -1,171 +0,0 @@
|
||||
name: Continuous Integration
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
branches:
|
||||
- master
|
||||
- '4.0'
|
||||
|
||||
permissions: read-all
|
||||
|
||||
jobs:
|
||||
server_tests:
|
||||
strategy:
|
||||
matrix:
|
||||
java: [ 11, 17 ]
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
ports:
|
||||
- 5432
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: 'postgres'
|
||||
POSTGRES_DB: test_db
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
mysql:
|
||||
image: mysql:8
|
||||
ports:
|
||||
- 3306
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
options: >-
|
||||
--health-cmd "mysqladmin ping"
|
||||
--health-interval 5s
|
||||
--health-timeout 2s
|
||||
--health-retries 3
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2.3.4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.ref }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
|
||||
- name: Restore dependency cache
|
||||
uses: actions/cache@v2.1.7
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Set up Java ${{ matrix.java }}
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
distribution: 'adopt'
|
||||
java-version: ${{ matrix.java }}
|
||||
|
||||
- name: Check Java linting
|
||||
id: java_linting
|
||||
run: |
|
||||
mvn formatter:validate
|
||||
|
||||
- name: Configure connections to databases
|
||||
id: configure_db_connections
|
||||
run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml
|
||||
|
||||
- name: Populate databases with test data
|
||||
id: populate_databases_with_test_data
|
||||
run: |
|
||||
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;'
|
||||
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql
|
||||
psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql
|
||||
env:
|
||||
PGPASSWORD: postgres
|
||||
|
||||
- name: Build and test with Maven
|
||||
run: mvn jacoco:prepare-agent test
|
||||
|
||||
- name: Submit test coverage to Coveralls
|
||||
run: |
|
||||
mvn prepare-package -DskipTests=true
|
||||
mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }}
|
||||
|
||||
prepare_ui_test_matrix:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2.3.4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.ref }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: '12'
|
||||
- id: set-matrix
|
||||
run: npm install --save glob && node main/tests/cypress/build-test-matrix.js
|
||||
env:
|
||||
browsers: chrome
|
||||
ui_test:
|
||||
needs: prepare_ui_test_matrix
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}}
|
||||
steps:
|
||||
- uses: actions/checkout@v2.3.4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.ref }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
|
||||
- name: Restore dependency cache
|
||||
uses: actions/cache@v2.1.7
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Set up Java 11
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
distribution: 'adopt'
|
||||
java-version: 11
|
||||
|
||||
- name: Build OpenRefine
|
||||
run: ./refine build
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: '12'
|
||||
|
||||
- name: Restore Tests dependency cache
|
||||
uses: actions/cache@v2.1.7
|
||||
with:
|
||||
path: |
|
||||
~/cache
|
||||
~/.cache
|
||||
**/node_modules
|
||||
!~/cache/exclude
|
||||
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-yarn
|
||||
|
||||
- name: Install test dependencies
|
||||
run: |
|
||||
cd ./main/tests/cypress
|
||||
npm i -g yarn
|
||||
yarn install
|
||||
|
||||
- name: Test with Cypress on ${{ matrix.browser }}
|
||||
run: |
|
||||
echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
|
||||
echo REFINE_MEMORY=4096M >> ./refine.ini
|
||||
./refine ui_tests
|
||||
env:
|
||||
CYPRESS_BROWSER: ${{ matrix.browser }}
|
||||
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
|
||||
CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
|
||||
CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
|
||||
CYPRESS_SPECS: ${{ matrix.specs }}
|
177
OpenRefine/.github/workflows/snapshot_release.yml
vendored
177
OpenRefine/.github/workflows/snapshot_release.yml
vendored
@ -1,177 +0,0 @@
|
||||
name: Snapshot release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
- '4.0'
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
|
||||
jobs:
|
||||
prepare_ui_test_matrix:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2.3.4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.ref }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: '12'
|
||||
- id: set-matrix
|
||||
run: npm install --save glob && node main/tests/cypress/build-test-matrix.js
|
||||
env:
|
||||
browsers: chrome
|
||||
ui_test:
|
||||
needs: prepare_ui_test_matrix
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}}
|
||||
steps:
|
||||
- uses: actions/checkout@v2.3.4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.ref }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
|
||||
- name: Restore dependency cache
|
||||
uses: actions/cache@v2.1.7
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Set up Java 11
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
distribution: 'adopt'
|
||||
java-version: 11
|
||||
|
||||
- name: Build OpenRefine
|
||||
run: ./refine build
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: '12'
|
||||
|
||||
- name: Restore Tests dependency cache
|
||||
uses: actions/cache@v2.1.7
|
||||
with:
|
||||
path: |
|
||||
~/cache
|
||||
~/.cache
|
||||
**/node_modules
|
||||
!~/cache/exclude
|
||||
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-yarn
|
||||
|
||||
- name: Install test dependencies
|
||||
run: |
|
||||
cd ./main/tests/cypress
|
||||
npm i -g yarn
|
||||
yarn install
|
||||
|
||||
- name: Test with Cypress on ${{ matrix.browser }}
|
||||
run: |
|
||||
echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
|
||||
echo REFINE_MEMORY=4096M >> ./refine.ini
|
||||
./refine ui_tests
|
||||
env:
|
||||
CYPRESS_BROWSER: ${{ matrix.browser }}
|
||||
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
|
||||
CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
|
||||
CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
|
||||
CYPRESS_SPECS: ${{ matrix.specs }}
|
||||
|
||||
|
||||
build:
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
ports:
|
||||
- 5432
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: 'postgres'
|
||||
POSTGRES_DB: test_db
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
mysql:
|
||||
image: mysql:8
|
||||
ports:
|
||||
- 3306
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
options: >-
|
||||
--health-cmd "mysqladmin ping"
|
||||
--health-interval 5s
|
||||
--health-timeout 2s
|
||||
--health-retries 3
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2.3.4
|
||||
with:
|
||||
fetch-depth: 0 # This is wasteful, but needed for git describe
|
||||
|
||||
- name: Restore dependency cache
|
||||
uses: actions/cache@v2.1.7
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Set up Java 11
|
||||
uses: actions/setup-java@v2
|
||||
with:
|
||||
distribution: 'adopt'
|
||||
java-version: 11
|
||||
server-id: ossrh
|
||||
server-username: OSSRH_USER
|
||||
server-password: OSSRH_PASS
|
||||
|
||||
- name: Install genisoimage and jq
|
||||
run: sudo apt-get install genisoimage jq
|
||||
|
||||
- name: Configure connections to databases
|
||||
id: configure_db_connections
|
||||
run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml
|
||||
|
||||
- name: Populate databases with test data
|
||||
id: populate_databases_with_test_data
|
||||
run: |
|
||||
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;'
|
||||
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql
|
||||
psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql
|
||||
env:
|
||||
PGPASSWORD: postgres
|
||||
|
||||
- name: Build and test with Maven
|
||||
run: mvn jacoco:prepare-agent test
|
||||
|
||||
- name: Submit test coverage to Coveralls
|
||||
run: |
|
||||
mvn prepare-package -DskipTests=true
|
||||
mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }} -DserviceName="GitHub Actions" -DserviceBuildNumber=${{ env.GITHUB_RUN_ID }} -Dbranch=master
|
||||
|
||||
- name: Generate dist files
|
||||
run: mvn package -DskipTests=true
|
||||
|
||||
- name: Upload snapshot releases to OSSRH
|
||||
run: mvn deploy -DskipTests=true
|
||||
env:
|
||||
OSSRH_USER: ${{ secrets.OSSRH_USER }}
|
||||
OSSRH_PASS: ${{ secrets.OSSRH_PASS }}
|
||||
|
19
OpenRefine/Dockerfile
Normal file
19
OpenRefine/Dockerfile
Normal file
@ -0,0 +1,19 @@
|
||||
FROM maven:3-jdk-11 as build
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
COPY OpenRefine .
|
||||
WORKDIR /usr/src/app/OpenRefine
|
||||
|
||||
RUN ./refine clean
|
||||
RUN ./refine build
|
||||
|
||||
FROM openjdk:11
|
||||
|
||||
WORKDIR /usr/app
|
||||
COPY --from=build /usr/src/app/OpenRefine .
|
||||
|
||||
VOLUME /data
|
||||
EXPOSE 3333
|
||||
|
||||
ENTRYPOINT ["/usr/app/refine"]
|
||||
CMD ["-i", "0.0.0.0", "-d", "/data", "-m", "3G"]
|
29
OpenRefine/ls1.txt
Normal file
29
OpenRefine/ls1.txt
Normal file
@ -0,0 +1,29 @@
|
||||
razem 164
|
||||
drwxr-xr-x 13 prance prance 4096 01-30 20:46 .
|
||||
drwxr-xr-x 4 prance prance 4096 01-30 20:41 ..
|
||||
-rw-r--r-- 1 prance prance 1603 01-30 20:42 appveyor.yml
|
||||
drwxr-xr-x 3 prance prance 4096 01-30 20:42 benchmark
|
||||
-rw-r--r-- 1 prance prance 3483 01-30 20:42 CODE_OF_CONDUCT.md
|
||||
drwxr-xr-x 2 prance prance 4096 01-30 20:42 conf
|
||||
-rw-r--r-- 1 prance prance 6438 01-30 20:42 CONTRIBUTING.md
|
||||
drwxr-xr-x 7 prance prance 4096 01-30 20:42 docs
|
||||
drwxr-xr-x 9 prance prance 4096 01-30 20:42 extensions
|
||||
drwxr-xr-x 8 prance prance 4096 01-30 20:42 .git
|
||||
-rw-r--r-- 1 prance prance 482 01-30 20:42 .gitattributes
|
||||
drwxr-xr-x 4 prance prance 4096 01-30 20:42 .github
|
||||
-rw-r--r-- 1 prance prance 1254 01-30 20:42 .gitignore
|
||||
-rw-r--r-- 1 prance prance 8613 01-30 20:42 GOVERNANCE.md
|
||||
drwxr-xr-x 4 prance prance 4096 01-30 20:42 graphics
|
||||
drwxr-xr-x 3 prance prance 4096 01-30 20:42 IDEs
|
||||
-rw-r--r-- 1 prance prance 1478 01-30 20:42 LICENSE.txt
|
||||
-rw-r--r-- 1 prance prance 0 01-30 20:46 ls1.txt
|
||||
drwxr-xr-x 6 prance prance 4096 01-30 20:42 main
|
||||
drwxr-xr-x 2 prance prance 4096 01-30 20:42 packaging
|
||||
-rw-r--r-- 1 prance prance 13518 01-30 20:42 pom.xml
|
||||
-rw-r--r-- 1 prance prance 3523 01-30 20:42 README.md
|
||||
-rwxr-xr-x 1 prance prance 29532 01-30 20:42 refine
|
||||
-rw-r--r-- 1 prance prance 7680 01-30 20:42 refine.bat
|
||||
-rw-r--r-- 1 prance prance 1549 01-30 20:42 refine.ini
|
||||
-rw-r--r-- 1 prance prance 729 01-30 20:42 SECURITY.md
|
||||
drwxr-xr-x 5 prance prance 4096 01-30 20:42 server
|
||||
-rw-r--r-- 1 prance prance 1099 01-30 20:42 settings.xml
|
31
OpenRefine/ls2.txt
Normal file
31
OpenRefine/ls2.txt
Normal file
@ -0,0 +1,31 @@
|
||||
razem 172
|
||||
drwxr-xr-x 14 prance prance 4096 01-30 20:47 .
|
||||
drwxr-xr-x 4 prance prance 4096 01-30 20:41 ..
|
||||
-rw-r--r-- 1 prance prance 1603 01-30 20:42 appveyor.yml
|
||||
drwxr-xr-x 3 prance prance 4096 01-30 20:42 benchmark
|
||||
-rw-r--r-- 1 prance prance 3483 01-30 20:42 CODE_OF_CONDUCT.md
|
||||
drwxr-xr-x 2 prance prance 4096 01-30 20:42 conf
|
||||
-rw-r--r-- 1 prance prance 6438 01-30 20:42 CONTRIBUTING.md
|
||||
drwxr-xr-x 7 prance prance 4096 01-30 20:42 docs
|
||||
drwxr-xr-x 9 prance prance 4096 01-30 20:42 extensions
|
||||
drwxr-xr-x 8 prance prance 4096 01-30 20:42 .git
|
||||
-rw-r--r-- 1 prance prance 482 01-30 20:42 .gitattributes
|
||||
drwxr-xr-x 4 prance prance 4096 01-30 20:42 .github
|
||||
-rw-r--r-- 1 prance prance 1254 01-30 20:42 .gitignore
|
||||
-rw-r--r-- 1 prance prance 8613 01-30 20:42 GOVERNANCE.md
|
||||
drwxr-xr-x 4 prance prance 4096 01-30 20:42 graphics
|
||||
drwxr-xr-x 3 prance prance 4096 01-30 20:42 IDEs
|
||||
-rw-r--r-- 1 prance prance 1478 01-30 20:42 LICENSE.txt
|
||||
-rw-r--r-- 1 prance prance 1563 01-30 20:46 ls1.txt
|
||||
-rw-r--r-- 1 prance prance 0 01-30 20:47 ls2.txt
|
||||
drwxr-xr-x 6 prance prance 4096 01-30 20:42 main
|
||||
drwxr-xr-x 2 prance prance 4096 01-30 20:42 packaging
|
||||
-rw-r--r-- 1 prance prance 13518 01-30 20:42 pom.xml
|
||||
-rw-r--r-- 1 prance prance 3523 01-30 20:42 README.md
|
||||
-rwxr-xr-x 1 prance prance 29532 01-30 20:42 refine
|
||||
-rw-r--r-- 1 prance prance 7680 01-30 20:42 refine.bat
|
||||
-rw-r--r-- 1 prance prance 1549 01-30 20:42 refine.ini
|
||||
-rw-r--r-- 1 prance prance 729 01-30 20:42 SECURITY.md
|
||||
drwxr-xr-x 5 prance prance 4096 01-30 20:42 server
|
||||
-rw-r--r-- 1 prance prance 1099 01-30 20:42 settings.xml
|
||||
drwxr-xr-x 2 prance prance 4096 01-30 20:46 tools
|
2
OpenRefine/ls3.txt
Normal file
2
OpenRefine/ls3.txt
Normal file
@ -0,0 +1,2 @@
|
||||
drwxr-xr-x 2 prance prance 4096 01-30 20:47 build
|
||||
drwxr-xr-x 2 prance prance 4096 01-30 20:46 tools
|
@ -1,296 +1,296 @@
|
||||
/*
|
||||
|
||||
Copyright 2010, Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
*/
|
||||
|
||||
package com.google.refine.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.refine.expr.ExpressionUtils;
|
||||
|
||||
public class RecordModel {
|
||||
final static Logger logger = LoggerFactory.getLogger("RecordModel");
|
||||
|
||||
final static public class CellDependency {
|
||||
final public int rowIndex;
|
||||
final public int cellIndex;
|
||||
|
||||
public CellDependency(int rowIndex, int cellIndex) {
|
||||
this.rowIndex = rowIndex;
|
||||
this.cellIndex = cellIndex;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return rowIndex+","+cellIndex;
|
||||
}
|
||||
}
|
||||
|
||||
final static public class RowDependency {
|
||||
public int recordIndex;
|
||||
public CellDependency[] cellDependencies;
|
||||
public List<Integer> contextRows;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Idx: "+recordIndex+" CellDeps: "+Arrays.toString(cellDependencies)+" Rows:"+contextRows;
|
||||
}
|
||||
}
|
||||
|
||||
protected List<RowDependency> _rowDependencies;
|
||||
protected List<Record> _records;
|
||||
|
||||
public RowDependency getRowDependency(int rowIndex) {
|
||||
return _rowDependencies != null && rowIndex >= 0 && rowIndex < _rowDependencies.size() ?
|
||||
_rowDependencies.get(rowIndex) : null;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public int getRecordCount() {
|
||||
return _records.size();
|
||||
}
|
||||
|
||||
public Record getRecord(int recordIndex) {
|
||||
return _records != null && recordIndex >= 0 && recordIndex < _records.size() ?
|
||||
_records.get(recordIndex) : null;
|
||||
}
|
||||
|
||||
public Record getRecordOfRow(int rowIndex) {
|
||||
RowDependency rd = getRowDependency(rowIndex);
|
||||
if (rd != null) {
|
||||
if (rd.recordIndex < 0) {
|
||||
rd = getRowDependency(rd.contextRows.get(0));
|
||||
}
|
||||
return getRecord(rd.recordIndex);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@JsonProperty("hasRecords")
|
||||
public boolean hasRecords() {
|
||||
return _records != null && _rowDependencies != null &&
|
||||
_records.size() < _rowDependencies.size();
|
||||
}
|
||||
|
||||
static protected class KeyedGroup {
|
||||
int[] cellIndices;
|
||||
int keyCellIndex;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuffer sb = new StringBuffer();
|
||||
for (int i:cellIndices) {
|
||||
sb.append(i).append(',');
|
||||
}
|
||||
return "key: " + keyCellIndex + " cells: " + sb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
synchronized public void update(Project project) {
|
||||
synchronized (project) {
|
||||
List<Row> rows = project.rows;
|
||||
int rowCount = rows.size();
|
||||
|
||||
ColumnModel columnModel = project.columnModel;
|
||||
List<KeyedGroup> keyedGroups = computeKeyedGroups(columnModel);
|
||||
int groupCount = keyedGroups.size();
|
||||
|
||||
int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()];
|
||||
for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) {
|
||||
lastNonBlankRowsByGroup[i] = -1;
|
||||
}
|
||||
|
||||
_rowDependencies = new ArrayList<RowDependency>(rowCount);
|
||||
|
||||
int recordIndex = 0;
|
||||
for (int r = 0; r < rowCount; r++) {
|
||||
Row row = rows.get(r);
|
||||
RowDependency rowDependency = new RowDependency();
|
||||
|
||||
for (int g = 0; g < groupCount; g++) {
|
||||
KeyedGroup group = keyedGroups.get(g);
|
||||
|
||||
if (!ExpressionUtils.isNonBlankData(row.getCellValue(keyedGroups.get(0).keyCellIndex)) &&
|
||||
!ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) {
|
||||
int contextRowIndex = lastNonBlankRowsByGroup[g];
|
||||
if (contextRowIndex >= 0) {
|
||||
for (int dependentCellIndex : group.cellIndices) {
|
||||
if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) {
|
||||
setRowDependency(
|
||||
project,
|
||||
rowDependency,
|
||||
dependentCellIndex,
|
||||
contextRowIndex,
|
||||
group.keyCellIndex
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
lastNonBlankRowsByGroup[g] = r;
|
||||
}
|
||||
}
|
||||
|
||||
if (rowDependency.cellDependencies != null && rowDependency.cellDependencies.length > 0) {
|
||||
rowDependency.recordIndex = -1;
|
||||
rowDependency.contextRows = new ArrayList<Integer>();
|
||||
for (CellDependency cd : rowDependency.cellDependencies) {
|
||||
if (cd != null) {
|
||||
rowDependency.contextRows.add(cd.rowIndex);
|
||||
}
|
||||
}
|
||||
Collections.sort(rowDependency.contextRows);
|
||||
} else {
|
||||
rowDependency.recordIndex = recordIndex++;
|
||||
}
|
||||
|
||||
_rowDependencies.add(rowDependency);
|
||||
}
|
||||
|
||||
_records = new ArrayList<Record>(recordIndex);
|
||||
if (recordIndex > 0) {
|
||||
recordIndex = 0;
|
||||
|
||||
int recordRowIndex = 0;
|
||||
for (int r = 1; r < rowCount; r++) {
|
||||
RowDependency rd = _rowDependencies.get(r);
|
||||
if (rd.recordIndex >= 0) {
|
||||
_records.add(new Record(recordRowIndex, r, recordIndex++));
|
||||
|
||||
recordIndex = rd.recordIndex;
|
||||
recordRowIndex = r;
|
||||
}
|
||||
}
|
||||
|
||||
_records.add(new Record(recordRowIndex, rowCount, recordIndex++));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected List<KeyedGroup> computeKeyedGroups(ColumnModel columnModel) {
|
||||
List<KeyedGroup> keyedGroups = new ArrayList<KeyedGroup>();
|
||||
|
||||
addRootKeyedGroup(columnModel, keyedGroups);
|
||||
|
||||
for (ColumnGroup group : columnModel.columnGroups) {
|
||||
if (group.keyColumnIndex >= 0) {
|
||||
KeyedGroup keyedGroup = new KeyedGroup();
|
||||
keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex();
|
||||
keyedGroup.cellIndices = new int[group.columnSpan - 1];
|
||||
|
||||
int c = 0;
|
||||
for (int i = 0; i < group.columnSpan; i++) {
|
||||
int columnIndex = group.startColumnIndex + i;
|
||||
if (columnIndex != group.keyColumnIndex && columnIndex < columnModel.columns.size()) {
|
||||
int cellIndex = columnModel.columns.get(columnIndex).getCellIndex();
|
||||
keyedGroup.cellIndices[c++] = cellIndex;
|
||||
}
|
||||
}
|
||||
|
||||
keyedGroups.add(keyedGroup);
|
||||
}
|
||||
}
|
||||
|
||||
Collections.sort(keyedGroups, new Comparator<KeyedGroup>() {
|
||||
@Override
|
||||
public int compare(KeyedGroup o1, KeyedGroup o2) {
|
||||
return o2.cellIndices.length - o1.cellIndices.length; // larger groups first
|
||||
}
|
||||
});
|
||||
|
||||
dumpKeyedGroups(keyedGroups, columnModel); // for debug
|
||||
|
||||
return keyedGroups;
|
||||
}
|
||||
|
||||
// debugging helper
|
||||
private void dumpKeyedGroups(List<KeyedGroup> groups, ColumnModel columnModel) {
|
||||
for (KeyedGroup g : groups) {
|
||||
String keyColName = columnModel.getColumnByCellIndex(g.keyCellIndex).getName();
|
||||
StringBuffer sb = new StringBuffer();
|
||||
for (int ci : g.cellIndices) {
|
||||
Column col = columnModel.getColumnByCellIndex(ci);
|
||||
if (col != null) {
|
||||
// Old projects have col 0 slot empty
|
||||
sb.append(col.getName()).append(',');
|
||||
}
|
||||
}
|
||||
logger.trace("KeyedGroup " + keyColName + "::" + sb.toString());
|
||||
}
|
||||
}
|
||||
|
||||
protected void addRootKeyedGroup(ColumnModel columnModel, List<KeyedGroup> keyedGroups) {
|
||||
int count = columnModel.getMaxCellIndex() + 1;
|
||||
if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) {
|
||||
KeyedGroup rootKeyedGroup = new KeyedGroup();
|
||||
|
||||
rootKeyedGroup.cellIndices = new int[count - 1];
|
||||
rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex();
|
||||
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (i < rootKeyedGroup.keyCellIndex) {
|
||||
rootKeyedGroup.cellIndices[i] = i;
|
||||
} else if (i > rootKeyedGroup.keyCellIndex) {
|
||||
rootKeyedGroup.cellIndices[i - 1] = i;
|
||||
}
|
||||
}
|
||||
keyedGroups.add(rootKeyedGroup);
|
||||
}
|
||||
}
|
||||
|
||||
protected void setRowDependency(
|
||||
Project project,
|
||||
RowDependency rowDependency,
|
||||
int cellIndex,
|
||||
int contextRowIndex,
|
||||
int contextCellIndex
|
||||
) {
|
||||
if (rowDependency.cellDependencies == null) {
|
||||
int count = project.columnModel.getMaxCellIndex() + 1;
|
||||
|
||||
rowDependency.cellDependencies = new CellDependency[count];
|
||||
}
|
||||
|
||||
rowDependency.cellDependencies[cellIndex] =
|
||||
new CellDependency(contextRowIndex, contextCellIndex);
|
||||
}
|
||||
|
||||
}
|
||||
/*
|
||||
|
||||
Copyright 2010, Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
*/
|
||||
|
||||
package com.google.refine.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.refine.expr.ExpressionUtils;
|
||||
|
||||
public class RecordModel {
|
||||
final static Logger logger = LoggerFactory.getLogger("RecordModel");
|
||||
|
||||
final static public class CellDependency {
|
||||
final public int rowIndex;
|
||||
final public int cellIndex;
|
||||
|
||||
public CellDependency(int rowIndex, int cellIndex) {
|
||||
this.rowIndex = rowIndex;
|
||||
this.cellIndex = cellIndex;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return rowIndex+","+cellIndex;
|
||||
}
|
||||
}
|
||||
|
||||
final static public class RowDependency {
|
||||
public int recordIndex;
|
||||
public CellDependency[] cellDependencies;
|
||||
public List<Integer> contextRows;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Idx: "+recordIndex+" CellDeps: "+Arrays.toString(cellDependencies)+" Rows:"+contextRows;
|
||||
}
|
||||
}
|
||||
|
||||
protected List<RowDependency> _rowDependencies;
|
||||
protected List<Record> _records;
|
||||
|
||||
public RowDependency getRowDependency(int rowIndex) {
|
||||
return _rowDependencies != null && rowIndex >= 0 && rowIndex < _rowDependencies.size() ?
|
||||
_rowDependencies.get(rowIndex) : null;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public int getRecordCount() {
|
||||
return _records.size();
|
||||
}
|
||||
|
||||
public Record getRecord(int recordIndex) {
|
||||
return _records != null && recordIndex >= 0 && recordIndex < _records.size() ?
|
||||
_records.get(recordIndex) : null;
|
||||
}
|
||||
|
||||
public Record getRecordOfRow(int rowIndex) {
|
||||
RowDependency rd = getRowDependency(rowIndex);
|
||||
if (rd != null) {
|
||||
if (rd.recordIndex < 0) {
|
||||
rd = getRowDependency(rd.contextRows.get(0));
|
||||
}
|
||||
return getRecord(rd.recordIndex);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@JsonProperty("hasRecords")
|
||||
public boolean hasRecords() {
|
||||
return _records != null && _rowDependencies != null &&
|
||||
_records.size() < _rowDependencies.size();
|
||||
}
|
||||
|
||||
static protected class KeyedGroup {
|
||||
int[] cellIndices;
|
||||
int keyCellIndex;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuffer sb = new StringBuffer();
|
||||
for (int i:cellIndices) {
|
||||
sb.append(i).append(',');
|
||||
}
|
||||
return "key: " + keyCellIndex + " cells: " + sb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
synchronized public void update(Project project) {
|
||||
synchronized (project) {
|
||||
List<Row> rows = project.rows;
|
||||
int rowCount = rows.size();
|
||||
|
||||
ColumnModel columnModel = project.columnModel;
|
||||
List<KeyedGroup> keyedGroups = computeKeyedGroups(columnModel);
|
||||
int groupCount = keyedGroups.size();
|
||||
|
||||
int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()];
|
||||
for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) {
|
||||
lastNonBlankRowsByGroup[i] = -1;
|
||||
}
|
||||
|
||||
_rowDependencies = new ArrayList<RowDependency>(rowCount);
|
||||
|
||||
int recordIndex = 0;
|
||||
for (int r = 0; r < rowCount; r++) {
|
||||
Row row = rows.get(r);
|
||||
RowDependency rowDependency = new RowDependency();
|
||||
|
||||
for (int g = 0; g < groupCount; g++) {
|
||||
KeyedGroup group = keyedGroups.get(g);
|
||||
|
||||
if (!ExpressionUtils.isNonBlankData(row.getCellValue(keyedGroups.get(0).keyCellIndex)) &&
|
||||
!ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) {
|
||||
int contextRowIndex = lastNonBlankRowsByGroup[g];
|
||||
if (contextRowIndex >= 0) {
|
||||
for (int dependentCellIndex : group.cellIndices) {
|
||||
if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) {
|
||||
setRowDependency(
|
||||
project,
|
||||
rowDependency,
|
||||
dependentCellIndex,
|
||||
contextRowIndex,
|
||||
group.keyCellIndex
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
lastNonBlankRowsByGroup[g] = r;
|
||||
}
|
||||
}
|
||||
|
||||
if (rowDependency.cellDependencies != null && rowDependency.cellDependencies.length > 0) {
|
||||
rowDependency.recordIndex = -1;
|
||||
rowDependency.contextRows = new ArrayList<Integer>();
|
||||
for (CellDependency cd : rowDependency.cellDependencies) {
|
||||
if (cd != null) {
|
||||
rowDependency.contextRows.add(cd.rowIndex);
|
||||
}
|
||||
}
|
||||
Collections.sort(rowDependency.contextRows);
|
||||
} else {
|
||||
rowDependency.recordIndex = recordIndex++;
|
||||
}
|
||||
|
||||
_rowDependencies.add(rowDependency);
|
||||
}
|
||||
|
||||
_records = new ArrayList<Record>(recordIndex);
|
||||
if (recordIndex > 0) {
|
||||
recordIndex = 0;
|
||||
|
||||
int recordRowIndex = 0;
|
||||
for (int r = 1; r < rowCount; r++) {
|
||||
RowDependency rd = _rowDependencies.get(r);
|
||||
if (rd.recordIndex >= 0) {
|
||||
_records.add(new Record(recordRowIndex, r, recordIndex++));
|
||||
|
||||
recordIndex = rd.recordIndex;
|
||||
recordRowIndex = r;
|
||||
}
|
||||
}
|
||||
|
||||
_records.add(new Record(recordRowIndex, rowCount, recordIndex++));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected List<KeyedGroup> computeKeyedGroups(ColumnModel columnModel) {
|
||||
List<KeyedGroup> keyedGroups = new ArrayList<KeyedGroup>();
|
||||
|
||||
addRootKeyedGroup(columnModel, keyedGroups);
|
||||
|
||||
for (ColumnGroup group : columnModel.columnGroups) {
|
||||
if (group.keyColumnIndex >= 0) {
|
||||
KeyedGroup keyedGroup = new KeyedGroup();
|
||||
keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex();
|
||||
keyedGroup.cellIndices = new int[group.columnSpan - 1];
|
||||
|
||||
int c = 0;
|
||||
for (int i = 0; i < group.columnSpan; i++) {
|
||||
int columnIndex = group.startColumnIndex + i;
|
||||
if (columnIndex != group.keyColumnIndex && columnIndex < columnModel.columns.size()) {
|
||||
int cellIndex = columnModel.columns.get(columnIndex).getCellIndex();
|
||||
keyedGroup.cellIndices[c++] = cellIndex;
|
||||
}
|
||||
}
|
||||
|
||||
keyedGroups.add(keyedGroup);
|
||||
}
|
||||
}
|
||||
|
||||
Collections.sort(keyedGroups, new Comparator<KeyedGroup>() {
|
||||
@Override
|
||||
public int compare(KeyedGroup o1, KeyedGroup o2) {
|
||||
return o2.cellIndices.length - o1.cellIndices.length; // larger groups first
|
||||
}
|
||||
});
|
||||
|
||||
dumpKeyedGroups(keyedGroups, columnModel); // for debug
|
||||
|
||||
return keyedGroups;
|
||||
}
|
||||
|
||||
// debugging helper
|
||||
private void dumpKeyedGroups(List<KeyedGroup> groups, ColumnModel columnModel) {
|
||||
for (KeyedGroup g : groups) {
|
||||
String keyColName = columnModel.getColumnByCellIndex(g.keyCellIndex).getName();
|
||||
StringBuffer sb = new StringBuffer();
|
||||
for (int ci : g.cellIndices) {
|
||||
Column col = columnModel.getColumnByCellIndex(ci);
|
||||
if (col != null) {
|
||||
// Old projects have col 0 slot empty
|
||||
sb.append(col.getName()).append(',');
|
||||
}
|
||||
}
|
||||
logger.trace("KeyedGroup " + keyColName + "::" + sb.toString());
|
||||
}
|
||||
}
|
||||
|
||||
protected void addRootKeyedGroup(ColumnModel columnModel, List<KeyedGroup> keyedGroups) {
|
||||
int count = columnModel.getMaxCellIndex() + 1;
|
||||
if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) {
|
||||
KeyedGroup rootKeyedGroup = new KeyedGroup();
|
||||
|
||||
rootKeyedGroup.cellIndices = new int[count - 1];
|
||||
rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex();
|
||||
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (i < rootKeyedGroup.keyCellIndex) {
|
||||
rootKeyedGroup.cellIndices[i] = i;
|
||||
} else if (i > rootKeyedGroup.keyCellIndex) {
|
||||
rootKeyedGroup.cellIndices[i - 1] = i;
|
||||
}
|
||||
}
|
||||
keyedGroups.add(rootKeyedGroup);
|
||||
}
|
||||
}
|
||||
|
||||
protected void setRowDependency(
|
||||
Project project,
|
||||
RowDependency rowDependency,
|
||||
int cellIndex,
|
||||
int contextRowIndex,
|
||||
int contextCellIndex
|
||||
) {
|
||||
if (rowDependency.cellDependencies == null) {
|
||||
int count = project.columnModel.getMaxCellIndex() + 1;
|
||||
|
||||
rowDependency.cellDependencies = new CellDependency[count];
|
||||
}
|
||||
|
||||
rowDependency.cellDependencies[cellIndex] =
|
||||
new CellDependency(contextRowIndex, contextCellIndex);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -1,84 +1,84 @@
|
||||
/*
|
||||
|
||||
Copyright 2010, Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
*/
|
||||
|
||||
package com.google.refine.model.changes;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.util.Properties;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.util.Pool;
|
||||
|
||||
public class CellAtRowCellIndex {
|
||||
|
||||
final public int row;
|
||||
final public int cellIndex;
|
||||
final public Cell cell;
|
||||
final static private Pattern semicolonPattern = Pattern.compile(";");
|
||||
|
||||
public CellAtRowCellIndex(int row, int cellIndex, Cell cell) {
|
||||
this.row = row;
|
||||
this.cell = cell;
|
||||
this.cellIndex = cellIndex;
|
||||
}
|
||||
|
||||
public void save(Writer writer, Properties options) throws IOException {
|
||||
writer.write(Integer.toString(row));
|
||||
writer.write(';');
|
||||
writer.write(Integer.toString(cellIndex));
|
||||
writer.write(';');
|
||||
if (cell != null) {
|
||||
cell.save(writer, options);
|
||||
}
|
||||
}
|
||||
|
||||
static public CellAtRowCellIndex load(String s, Pool pool) throws Exception {
|
||||
|
||||
Matcher m = semicolonPattern.matcher(s);
|
||||
|
||||
m.find();
|
||||
int semicolon = m.start();
|
||||
m.find();
|
||||
int nextSemicolon = m.start();
|
||||
|
||||
int row = Integer.parseInt(s.substring(0, semicolon));
|
||||
int cellIndex = Integer.parseInt(s.substring(semicolon + 1, nextSemicolon));
|
||||
Cell cell = nextSemicolon < s.length() - 1 ? Cell.loadStreaming(s.substring(nextSemicolon + 1), pool)
|
||||
: null;
|
||||
|
||||
return new CellAtRowCellIndex(row, cellIndex, cell);
|
||||
}
|
||||
}
|
||||
/*
|
||||
|
||||
Copyright 2010, Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
*/
|
||||
|
||||
package com.google.refine.model.changes;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.util.Properties;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.util.Pool;
|
||||
|
||||
public class CellAtRowCellIndex {
|
||||
|
||||
final public int row;
|
||||
final public int cellIndex;
|
||||
final public Cell cell;
|
||||
final static private Pattern semicolonPattern = Pattern.compile(";");
|
||||
|
||||
public CellAtRowCellIndex(int row, int cellIndex, Cell cell) {
|
||||
this.row = row;
|
||||
this.cell = cell;
|
||||
this.cellIndex = cellIndex;
|
||||
}
|
||||
|
||||
public void save(Writer writer, Properties options) throws IOException {
|
||||
writer.write(Integer.toString(row));
|
||||
writer.write(';');
|
||||
writer.write(Integer.toString(cellIndex));
|
||||
writer.write(';');
|
||||
if (cell != null) {
|
||||
cell.save(writer, options);
|
||||
}
|
||||
}
|
||||
|
||||
static public CellAtRowCellIndex load(String s, Pool pool) throws Exception {
|
||||
|
||||
Matcher m = semicolonPattern.matcher(s);
|
||||
|
||||
m.find();
|
||||
int semicolon = m.start();
|
||||
m.find();
|
||||
int nextSemicolon = m.start();
|
||||
|
||||
int row = Integer.parseInt(s.substring(0, semicolon));
|
||||
int cellIndex = Integer.parseInt(s.substring(semicolon + 1, nextSemicolon));
|
||||
Cell cell = nextSemicolon < s.length() - 1 ? Cell.loadStreaming(s.substring(nextSemicolon + 1), pool)
|
||||
: null;
|
||||
|
||||
return new CellAtRowCellIndex(row, cellIndex, cell);
|
||||
}
|
||||
}
|
||||
|
@ -1,69 +1,69 @@
|
||||
/*
|
||||
|
||||
Copyright 2010, Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
*/
|
||||
|
||||
class Facet {
|
||||
constructor(div, config, options) {
|
||||
this._div = div;
|
||||
this._config = config;
|
||||
this._options = options || {};
|
||||
this._minimizeState = false;
|
||||
|
||||
Refine.showLeftPanel();
|
||||
};
|
||||
|
||||
_minimize() {
|
||||
if(!this._minimizeState) {
|
||||
this._div.addClass("facet-state-minimize");
|
||||
} else {
|
||||
this._div.removeClass("facet-state-minimize");
|
||||
}
|
||||
|
||||
this._minimizeState = !this._minimizeState;
|
||||
};
|
||||
|
||||
_remove() {
|
||||
ui.browsingEngine.removeFacet(this);
|
||||
|
||||
this._div = null;
|
||||
this._config = null;
|
||||
|
||||
this._selection = null;
|
||||
this._blankChoice = null;
|
||||
this._errorChoice = null;
|
||||
this._data = null;
|
||||
this._options = null;
|
||||
};
|
||||
|
||||
dispose() {
|
||||
};
|
||||
};
|
||||
/*
|
||||
|
||||
Copyright 2010, Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
*/
|
||||
|
||||
class Facet {
|
||||
constructor(div, config, options) {
|
||||
this._div = div;
|
||||
this._config = config;
|
||||
this._options = options || {};
|
||||
this._minimizeState = false;
|
||||
|
||||
Refine.showLeftPanel();
|
||||
};
|
||||
|
||||
_minimize() {
|
||||
if(!this._minimizeState) {
|
||||
this._div.addClass("facet-state-minimize");
|
||||
} else {
|
||||
this._div.removeClass("facet-state-minimize");
|
||||
}
|
||||
|
||||
this._minimizeState = !this._minimizeState;
|
||||
};
|
||||
|
||||
_remove() {
|
||||
ui.browsingEngine.removeFacet(this);
|
||||
|
||||
this._div = null;
|
||||
this._config = null;
|
||||
|
||||
this._selection = null;
|
||||
this._blankChoice = null;
|
||||
this._errorChoice = null;
|
||||
this._data = null;
|
||||
this._options = null;
|
||||
};
|
||||
|
||||
dispose() {
|
||||
};
|
||||
};
|
||||
|
@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
font-size: 110%;
|
||||
color: @near_black;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
#left-panel-body {
|
||||
margin-left: @padding_tight;
|
||||
|
@ -1,104 +1,104 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2020, OpenRefine contributors
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
|
||||
package com.google.refine;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.regex.Pattern;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import org.eclipse.jetty.server.handler.HandlerWrapper;
|
||||
import org.eclipse.jetty.server.Request;
|
||||
|
||||
/**
|
||||
* Validate the Host header of the HTTP request to see if it matches either a loopback IP
|
||||
* address, localhost or an explicitly specified hostname. This is required to avoid DNS
|
||||
* rebinding attacks against users running OpenRefine on their desktop computers.
|
||||
*/
|
||||
class ValidateHostHandler extends HandlerWrapper {
|
||||
|
||||
/**
|
||||
* Matches:
|
||||
* - addresses in the 127.0.0.0/8 subnet
|
||||
* - IPv4-mapped addresses in the ::ffff:7f00:00/104 subnet
|
||||
* - different representations of ::1
|
||||
* - localhost
|
||||
* Matching is a little fuzzy to simplify the regular expression - it expects the Host
|
||||
* header to be well-formed. Some invalid addresses would be accepted, for example:
|
||||
* - 127.6..64.245
|
||||
* - 0::0:::0:00:1
|
||||
* This is not a problem however, as these are not valid DNS names either, and should
|
||||
* never be sent by a well-behaved browser - and validating the host header only ever
|
||||
* helps if the browser works as expected and cannot be used to fake the Host header.
|
||||
*/
|
||||
static private final Pattern LOOPBACK_PATTERN = Pattern
|
||||
.compile("^(?:127\\.[0-9\\.]*|\\[[0\\:]*\\:(?:ffff\\:7f[0-9a-f]{2}:[0-9a-f]{1,4}|0{0,3}1)\\]|localhost)(?:\\:[0-9]+)?$", Pattern.CASE_INSENSITIVE);
|
||||
|
||||
private String expectedHost;
|
||||
|
||||
public ValidateHostHandler(String expectedHost) {
|
||||
this.expectedHost = expectedHost;
|
||||
}
|
||||
|
||||
public boolean isValidHost(String host) {
|
||||
|
||||
// Allow loopback IPv4 and IPv6 addresses, as well as localhost
|
||||
if (LOOPBACK_PATTERN.matcher(host).find()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Strip port from hostname - for IPv6 addresses, if
|
||||
// they end with a bracket, then there is no port
|
||||
int index = host.lastIndexOf(':');
|
||||
if (index > 0 && !host.endsWith("]")) {
|
||||
host = host.substring(0, index);
|
||||
}
|
||||
|
||||
// Strip brackets from IPv6 addresses
|
||||
if (host.startsWith("[") && host.endsWith("]")) {
|
||||
host = host.substring(1, host.length() - 2);
|
||||
}
|
||||
|
||||
// Allow only if stripped hostname matches expected hostname
|
||||
return expectedHost.equalsIgnoreCase(host);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
|
||||
throws IOException, ServletException {
|
||||
String host = request.getHeader("Host");
|
||||
if (isValidHost(host)) {
|
||||
super.handle(target, baseRequest, request, response);
|
||||
} else {
|
||||
// Return HTTP 404 Not Found, since we are
|
||||
// not serving content for the requested URL
|
||||
response.sendError(HttpServletResponse.SC_NOT_FOUND, "Invalid hostname");
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
* Copyright (C) 2020, OpenRefine contributors
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
******************************************************************************/
|
||||
|
||||
package com.google.refine;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.regex.Pattern;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import org.eclipse.jetty.server.handler.HandlerWrapper;
|
||||
import org.eclipse.jetty.server.Request;
|
||||
|
||||
/**
|
||||
* Validate the Host header of the HTTP request to see if it matches either a loopback IP
|
||||
* address, localhost or an explicitly specified hostname. This is required to avoid DNS
|
||||
* rebinding attacks against users running OpenRefine on their desktop computers.
|
||||
*/
|
||||
class ValidateHostHandler extends HandlerWrapper {
|
||||
|
||||
/**
|
||||
* Matches:
|
||||
* - addresses in the 127.0.0.0/8 subnet
|
||||
* - IPv4-mapped addresses in the ::ffff:7f00:00/104 subnet
|
||||
* - different representations of ::1
|
||||
* - localhost
|
||||
* Matching is a little fuzzy to simplify the regular expression - it expects the Host
|
||||
* header to be well-formed. Some invalid addresses would be accepted, for example:
|
||||
* - 127.6..64.245
|
||||
* - 0::0:::0:00:1
|
||||
* This is not a problem however, as these are not valid DNS names either, and should
|
||||
* never be sent by a well-behaved browser - and validating the host header only ever
|
||||
* helps if the browser works as expected and cannot be used to fake the Host header.
|
||||
*/
|
||||
static private final Pattern LOOPBACK_PATTERN = Pattern
|
||||
.compile("^(?:127\\.[0-9\\.]*|\\[[0\\:]*\\:(?:ffff\\:7f[0-9a-f]{2}:[0-9a-f]{1,4}|0{0,3}1)\\]|localhost)(?:\\:[0-9]+)?$", Pattern.CASE_INSENSITIVE);
|
||||
|
||||
private String expectedHost;
|
||||
|
||||
public ValidateHostHandler(String expectedHost) {
|
||||
this.expectedHost = expectedHost;
|
||||
}
|
||||
|
||||
public boolean isValidHost(String host) {
|
||||
|
||||
// Allow loopback IPv4 and IPv6 addresses, as well as localhost
|
||||
if (LOOPBACK_PATTERN.matcher(host).find()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Strip port from hostname - for IPv6 addresses, if
|
||||
// they end with a bracket, then there is no port
|
||||
int index = host.lastIndexOf(':');
|
||||
if (index > 0 && !host.endsWith("]")) {
|
||||
host = host.substring(0, index);
|
||||
}
|
||||
|
||||
// Strip brackets from IPv6 addresses
|
||||
if (host.startsWith("[") && host.endsWith("]")) {
|
||||
host = host.substring(1, host.length() - 2);
|
||||
}
|
||||
|
||||
// Allow only if stripped hostname matches expected hostname
|
||||
return expectedHost.equalsIgnoreCase(host);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
|
||||
throws IOException, ServletException {
|
||||
String host = request.getHeader("Host");
|
||||
if (isValidHost(host)) {
|
||||
super.handle(target, baseRequest, request, response);
|
||||
} else {
|
||||
// Return HTTP 404 Not Found, since we are
|
||||
// not serving content for the requested URL
|
||||
response.sendError(HttpServletResponse.SC_NOT_FOUND, "Invalid hostname");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
Loading…
Reference in New Issue
Block a user