change OR version

This commit is contained in:
prance 2022-01-30 22:06:15 +01:00
parent 8694374775
commit 2a7e2e66b5
22 changed files with 634 additions and 1132 deletions

View File

@ -1 +0,0 @@
github: OpenRefine

View File

@ -1,40 +0,0 @@
---
name: Bug report
about: Create a report to help us improve OpenRefine
title: ''
labels: bug, to be reviewed
assignees: ''
---
<!-- Describe the bug - Please add a clear and concise description of the bug above this line. You can delete this line if you want. It will be hidden in the final bug report -->
### To Reproduce
Steps to reproduce the behavior:
1. First, do ...
2. Then, ...
3. Finally, ...
### Current Results
<!-- What results occurred or were shown. -->
### Expected Behavior
<!-- A clear and concise description of what you expected to happen or to show. -->
### Screenshots
<!-- If applicable, add screenshots to help explain your problem. -->
### Versions<!-- (please complete the following information)-->
- Operating System: <!-- e.g. iOS, Windows 10, Linux, Ubuntu 18.04 -->
- Browser Version: <!-- e.g. Chrome 19, Firefox 61, Safari, NOTE: OpenRefine does not support IE but works OK in most cases -->
- JRE or JDK Version: <!-- output of "java -version" e.g. JRE 1.8.0_181 -->
- OpenRefine: <!-- e.g. OpenRefine 3.0 Beta] -->
### Datasets
<!-- If you are allowed and are OK with making your data public, it would be awesome if you can include or attach the data causing the issue or a URL pointing to where the data is.
If you are concerned about keeping your data private, ping us on our [mailing list](https://groups.google.com/forum/#!forum/openrefine) -->
### Additional context
<!-- Add any other context about the problem here. -->

View File

@ -1,8 +0,0 @@
blank_issues_enabled: false
contact_links:
- name: Ask a question (mailing list)
url: https://groups.google.com/d/forum/openrefine
about: Please ask and answer questions here.
- name: Gitter chat
url: https://gitter.im/OpenRefine/OpenRefine
about: General and developer discussions

View File

@ -1,19 +0,0 @@
---
name: Feature request
about: Suggest an idea for OpenRefine
title: ''
labels: enhancement, to be reviewed
assignees: ''
---
<!-- Please provide a clear and concise description of your problem or unsatisfied needs. Ex. I'm always frustrated when [...] or, It would be easier if OpenRefine did [...]. This comment can be deleted, if desired, but it will be hidden in your final submission. Please make sure that your new text is outside the enclosing angle brackets. -->
### Proposed solution
<!-- If you have a proposal for how this need could be met, please provide a clear and concise description of what you want to happen. -->
### Alternatives considered
<!-- If there alternative solutions that you have considered or think should be considered, please list them here -->
### Additional context
<!-- Add any other context or screenshots about the feature request here. -->

View File

@ -1,5 +0,0 @@
If you are having trouble with OpenRefine we suggest the following steps:
1. Read through our [FAQ (frequently Asked Questions)](https://github.com/OpenRefine/OpenRefine/wiki/FAQ)
2. Search for similar issues in our community email list archives: http://groups.google.com/group/openrefine/
3. Send an email to our community mailing list: openrefine@googlegroups.com

View File

@ -1,3 +0,0 @@
documentation: ["/docs"]
"current docs": ["/docs/docs"]
"historical docs": ["/docs/versioned_docs"]

View File

@ -1,45 +0,0 @@
# Documentation for all configuration options:
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
# For openrefine java deps
- package-ecosystem: maven
directory: "/"
schedule:
interval: daily
open-pull-requests-limit: 10
ignore:
- dependency-name: com.thoughtworks.xstream:xstream
versions:
- "> 1.4.12"
- "< 2"
# Same, on 4.0 branch
- package-ecosystem: maven
directory: "/"
schedule:
interval: daily
open-pull-requests-limit: 10
ignore:
- dependency-name: com.thoughtworks.xstream:xstream
versions:
- "> 1.4.12"
- "< 2"
target-branch: 4.0
# For documentation website
- package-ecosystem: "npm" # For Yarn
directory: "docs/"
schedule:
interval: "monthly"
# For github actions
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
# For cypress test_suite
- package-ecosystem: "npm"
directory: "main/tests/cypress"
schedule:
interval: "daily"

View File

@ -1,6 +0,0 @@
Fixes #{issue number here}
Changes proposed in this pull request:
-
-
-

View File

@ -1,26 +0,0 @@
name: Copy labels from issue to pull request
on:
pull_request_target:
types: [opened, edited]
jobs:
transfer_tags:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.9
uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install dependencies
run: |
python -m pip install --upgrade pip setuptools
pip install -r .github/workflows/label_transfer/requirements.txt
pip freeze
- name: Run Python label transfer script
run: python .github/workflows/label_transfer/script.py ${{ github.event.number }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_REPO: ${{ github.repository }}

View File

@ -1,2 +0,0 @@
requests
lxml

View File

@ -1,76 +0,0 @@
import requests
import os
from lxml import html
import sys
import json
# Config
# list of labels that should not be transferred to PRs
do_not_transfer = [
'good first issue',
'good second issue',
'imported from old code repo',
'help wanted',
'duplicate',
'invalid',
'question',
'to be reviewed',
]
# Internal
repo = os.environ.get('GITHUB_REPO')
github_token = os.environ.get('GITHUB_TOKEN')
headers = {
'Accept': 'application/vnd.github.v3+json',
}
if github_token:
headers['Authorization'] = 'Bearer '+github_token
def get_linked_issues(pr_number):
"""
Given a PR number, extract all the linked issue numbers from it.
Sadly this is not supported by the API yet, so we just scrape the web UI.
"""
url = f'https://github.com/{repo}/pull/{pr_number}'
page = requests.get(url)
page.raise_for_status()
parsed = html.document_fromstring(page.text)
matches = parsed.xpath('//form/div[@class="css-truncate my-1"]/a')
for match in matches:
yield int(match.attrib['href'].split('/')[-1])
def get_issue_labels(issue_number):
"""
Returns all the labels in a given issue / PR
"""
url = f'https://api.github.com/repos/{repo}/issues/{issue_number}/labels'
response = requests.get(url, headers=headers)
response.raise_for_status()
return [ tag['name'] for tag in response.json() ]
def transfer_issue_labels(pr_number):
"""
Transfers labels from all the linked issues to the PR
"""
linked_issues = get_linked_issues(pr_number)
all_labels = [ label for issue in linked_issues for label in get_issue_labels(issue) ]
to_transfer = [ label for label in all_labels if label not in do_not_transfer ]
current_labels = get_issue_labels(pr_number)
missing_labels = [ label for label in to_transfer if label not in current_labels ]
if not missing_labels:
return
new_labels = current_labels + missing_labels
url = f'https://api.github.com/repos/{repo}/issues/{pr_number}/labels'
print(f'adding {missing_labels} to PR #{pr_number}')
if not github_token:
print('no GITHUB_TOKEN, skipping')
else:
resp = requests.put(url, headers=headers, data=json.dumps({'labels':new_labels}))
resp.raise_for_status()
if __name__ == '__main__':
transfer_issue_labels(sys.argv[1])

View File

@ -1,171 +0,0 @@
name: Continuous Integration
on:
pull_request_target:
paths-ignore:
- 'docs/**'
branches:
- master
- '4.0'
permissions: read-all
jobs:
server_tests:
strategy:
matrix:
java: [ 11, 17 ]
runs-on: ubuntu-latest
services:
postgres:
image: postgres
ports:
- 5432
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: 'postgres'
POSTGRES_DB: test_db
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
mysql:
image: mysql:8
ports:
- 3306
env:
MYSQL_ROOT_PASSWORD: root
options: >-
--health-cmd "mysqladmin ping"
--health-interval 5s
--health-timeout 2s
--health-retries 3
steps:
- uses: actions/checkout@v2.3.4
with:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Restore dependency cache
uses: actions/cache@v2.1.7
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up Java ${{ matrix.java }}
uses: actions/setup-java@v2
with:
distribution: 'adopt'
java-version: ${{ matrix.java }}
- name: Check Java linting
id: java_linting
run: |
mvn formatter:validate
- name: Configure connections to databases
id: configure_db_connections
run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml
- name: Populate databases with test data
id: populate_databases_with_test_data
run: |
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;'
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql
psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql
env:
PGPASSWORD: postgres
- name: Build and test with Maven
run: mvn jacoco:prepare-agent test
- name: Submit test coverage to Coveralls
run: |
mvn prepare-package -DskipTests=true
mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }}
prepare_ui_test_matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v2.3.4
with:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Setup Node
uses: actions/setup-node@v2
with:
node-version: '12'
- id: set-matrix
run: npm install --save glob && node main/tests/cypress/build-test-matrix.js
env:
browsers: chrome
ui_test:
needs: prepare_ui_test_matrix
runs-on: ubuntu-latest
strategy:
matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}}
steps:
- uses: actions/checkout@v2.3.4
with:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Restore dependency cache
uses: actions/cache@v2.1.7
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up Java 11
uses: actions/setup-java@v2
with:
distribution: 'adopt'
java-version: 11
- name: Build OpenRefine
run: ./refine build
- name: Setup Node
uses: actions/setup-node@v2
with:
node-version: '12'
- name: Restore Tests dependency cache
uses: actions/cache@v2.1.7
with:
path: |
~/cache
~/.cache
**/node_modules
!~/cache/exclude
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn
- name: Install test dependencies
run: |
cd ./main/tests/cypress
npm i -g yarn
yarn install
- name: Test with Cypress on ${{ matrix.browser }}
run: |
echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
echo REFINE_MEMORY=4096M >> ./refine.ini
./refine ui_tests
env:
CYPRESS_BROWSER: ${{ matrix.browser }}
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
CYPRESS_SPECS: ${{ matrix.specs }}

View File

@ -1,177 +0,0 @@
name: Snapshot release
on:
push:
branches:
- 'master'
- '4.0'
paths-ignore:
- 'docs/**'
jobs:
prepare_ui_test_matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v2.3.4
with:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Setup Node
uses: actions/setup-node@v2
with:
node-version: '12'
- id: set-matrix
run: npm install --save glob && node main/tests/cypress/build-test-matrix.js
env:
browsers: chrome
ui_test:
needs: prepare_ui_test_matrix
runs-on: ubuntu-latest
strategy:
matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}}
steps:
- uses: actions/checkout@v2.3.4
with:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Restore dependency cache
uses: actions/cache@v2.1.7
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up Java 11
uses: actions/setup-java@v2
with:
distribution: 'adopt'
java-version: 11
- name: Build OpenRefine
run: ./refine build
- name: Setup Node
uses: actions/setup-node@v2
with:
node-version: '12'
- name: Restore Tests dependency cache
uses: actions/cache@v2.1.7
with:
path: |
~/cache
~/.cache
**/node_modules
!~/cache/exclude
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn
- name: Install test dependencies
run: |
cd ./main/tests/cypress
npm i -g yarn
yarn install
- name: Test with Cypress on ${{ matrix.browser }}
run: |
echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
echo REFINE_MEMORY=4096M >> ./refine.ini
./refine ui_tests
env:
CYPRESS_BROWSER: ${{ matrix.browser }}
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
CYPRESS_SPECS: ${{ matrix.specs }}
build:
services:
postgres:
image: postgres
ports:
- 5432
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: 'postgres'
POSTGRES_DB: test_db
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
mysql:
image: mysql:8
ports:
- 3306
env:
MYSQL_ROOT_PASSWORD: root
options: >-
--health-cmd "mysqladmin ping"
--health-interval 5s
--health-timeout 2s
--health-retries 3
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.3.4
with:
fetch-depth: 0 # This is wasteful, but needed for git describe
- name: Restore dependency cache
uses: actions/cache@v2.1.7
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up Java 11
uses: actions/setup-java@v2
with:
distribution: 'adopt'
java-version: 11
server-id: ossrh
server-username: OSSRH_USER
server-password: OSSRH_PASS
- name: Install genisoimage and jq
run: sudo apt-get install genisoimage jq
- name: Configure connections to databases
id: configure_db_connections
run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml
- name: Populate databases with test data
id: populate_databases_with_test_data
run: |
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;'
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql
psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql
env:
PGPASSWORD: postgres
- name: Build and test with Maven
run: mvn jacoco:prepare-agent test
- name: Submit test coverage to Coveralls
run: |
mvn prepare-package -DskipTests=true
mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }} -DserviceName="GitHub Actions" -DserviceBuildNumber=${{ env.GITHUB_RUN_ID }} -Dbranch=master
- name: Generate dist files
run: mvn package -DskipTests=true
- name: Upload snapshot releases to OSSRH
run: mvn deploy -DskipTests=true
env:
OSSRH_USER: ${{ secrets.OSSRH_USER }}
OSSRH_PASS: ${{ secrets.OSSRH_PASS }}

19
OpenRefine/Dockerfile Normal file
View File

@ -0,0 +1,19 @@
FROM maven:3-jdk-11 as build
WORKDIR /usr/src/app
COPY OpenRefine .
WORKDIR /usr/src/app/OpenRefine
RUN ./refine clean
RUN ./refine build
FROM openjdk:11
WORKDIR /usr/app
COPY --from=build /usr/src/app/OpenRefine .
VOLUME /data
EXPOSE 3333
ENTRYPOINT ["/usr/app/refine"]
CMD ["-i", "0.0.0.0", "-d", "/data", "-m", "3G"]

29
OpenRefine/ls1.txt Normal file
View File

@ -0,0 +1,29 @@
razem 164
drwxr-xr-x 13 prance prance 4096 01-30 20:46 .
drwxr-xr-x 4 prance prance 4096 01-30 20:41 ..
-rw-r--r-- 1 prance prance 1603 01-30 20:42 appveyor.yml
drwxr-xr-x 3 prance prance 4096 01-30 20:42 benchmark
-rw-r--r-- 1 prance prance 3483 01-30 20:42 CODE_OF_CONDUCT.md
drwxr-xr-x 2 prance prance 4096 01-30 20:42 conf
-rw-r--r-- 1 prance prance 6438 01-30 20:42 CONTRIBUTING.md
drwxr-xr-x 7 prance prance 4096 01-30 20:42 docs
drwxr-xr-x 9 prance prance 4096 01-30 20:42 extensions
drwxr-xr-x 8 prance prance 4096 01-30 20:42 .git
-rw-r--r-- 1 prance prance 482 01-30 20:42 .gitattributes
drwxr-xr-x 4 prance prance 4096 01-30 20:42 .github
-rw-r--r-- 1 prance prance 1254 01-30 20:42 .gitignore
-rw-r--r-- 1 prance prance 8613 01-30 20:42 GOVERNANCE.md
drwxr-xr-x 4 prance prance 4096 01-30 20:42 graphics
drwxr-xr-x 3 prance prance 4096 01-30 20:42 IDEs
-rw-r--r-- 1 prance prance 1478 01-30 20:42 LICENSE.txt
-rw-r--r-- 1 prance prance 0 01-30 20:46 ls1.txt
drwxr-xr-x 6 prance prance 4096 01-30 20:42 main
drwxr-xr-x 2 prance prance 4096 01-30 20:42 packaging
-rw-r--r-- 1 prance prance 13518 01-30 20:42 pom.xml
-rw-r--r-- 1 prance prance 3523 01-30 20:42 README.md
-rwxr-xr-x 1 prance prance 29532 01-30 20:42 refine
-rw-r--r-- 1 prance prance 7680 01-30 20:42 refine.bat
-rw-r--r-- 1 prance prance 1549 01-30 20:42 refine.ini
-rw-r--r-- 1 prance prance 729 01-30 20:42 SECURITY.md
drwxr-xr-x 5 prance prance 4096 01-30 20:42 server
-rw-r--r-- 1 prance prance 1099 01-30 20:42 settings.xml

31
OpenRefine/ls2.txt Normal file
View File

@ -0,0 +1,31 @@
razem 172
drwxr-xr-x 14 prance prance 4096 01-30 20:47 .
drwxr-xr-x 4 prance prance 4096 01-30 20:41 ..
-rw-r--r-- 1 prance prance 1603 01-30 20:42 appveyor.yml
drwxr-xr-x 3 prance prance 4096 01-30 20:42 benchmark
-rw-r--r-- 1 prance prance 3483 01-30 20:42 CODE_OF_CONDUCT.md
drwxr-xr-x 2 prance prance 4096 01-30 20:42 conf
-rw-r--r-- 1 prance prance 6438 01-30 20:42 CONTRIBUTING.md
drwxr-xr-x 7 prance prance 4096 01-30 20:42 docs
drwxr-xr-x 9 prance prance 4096 01-30 20:42 extensions
drwxr-xr-x 8 prance prance 4096 01-30 20:42 .git
-rw-r--r-- 1 prance prance 482 01-30 20:42 .gitattributes
drwxr-xr-x 4 prance prance 4096 01-30 20:42 .github
-rw-r--r-- 1 prance prance 1254 01-30 20:42 .gitignore
-rw-r--r-- 1 prance prance 8613 01-30 20:42 GOVERNANCE.md
drwxr-xr-x 4 prance prance 4096 01-30 20:42 graphics
drwxr-xr-x 3 prance prance 4096 01-30 20:42 IDEs
-rw-r--r-- 1 prance prance 1478 01-30 20:42 LICENSE.txt
-rw-r--r-- 1 prance prance 1563 01-30 20:46 ls1.txt
-rw-r--r-- 1 prance prance 0 01-30 20:47 ls2.txt
drwxr-xr-x 6 prance prance 4096 01-30 20:42 main
drwxr-xr-x 2 prance prance 4096 01-30 20:42 packaging
-rw-r--r-- 1 prance prance 13518 01-30 20:42 pom.xml
-rw-r--r-- 1 prance prance 3523 01-30 20:42 README.md
-rwxr-xr-x 1 prance prance 29532 01-30 20:42 refine
-rw-r--r-- 1 prance prance 7680 01-30 20:42 refine.bat
-rw-r--r-- 1 prance prance 1549 01-30 20:42 refine.ini
-rw-r--r-- 1 prance prance 729 01-30 20:42 SECURITY.md
drwxr-xr-x 5 prance prance 4096 01-30 20:42 server
-rw-r--r-- 1 prance prance 1099 01-30 20:42 settings.xml
drwxr-xr-x 2 prance prance 4096 01-30 20:46 tools

2
OpenRefine/ls3.txt Normal file
View File

@ -0,0 +1,2 @@
drwxr-xr-x 2 prance prance 4096 01-30 20:47 build
drwxr-xr-x 2 prance prance 4096 01-30 20:46 tools

View File

@ -1,296 +1,296 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010, Google Inc.
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are modification, are permitted provided that the following conditions are
met: met:
* Redistributions of source code must retain the above copyright * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the in the documentation and/or other materials provided with the
distribution. distribution.
* Neither the name of Google Inc. nor the names of its * Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from contributors may be used to endorse or promote products derived from
this software without specific prior written permission. this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package com.google.refine.model; package com.google.refine.model;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
public class RecordModel { public class RecordModel {
final static Logger logger = LoggerFactory.getLogger("RecordModel"); final static Logger logger = LoggerFactory.getLogger("RecordModel");
final static public class CellDependency { final static public class CellDependency {
final public int rowIndex; final public int rowIndex;
final public int cellIndex; final public int cellIndex;
public CellDependency(int rowIndex, int cellIndex) { public CellDependency(int rowIndex, int cellIndex) {
this.rowIndex = rowIndex; this.rowIndex = rowIndex;
this.cellIndex = cellIndex; this.cellIndex = cellIndex;
} }
@Override @Override
public String toString() { public String toString() {
return rowIndex+","+cellIndex; return rowIndex+","+cellIndex;
} }
} }
final static public class RowDependency { final static public class RowDependency {
public int recordIndex; public int recordIndex;
public CellDependency[] cellDependencies; public CellDependency[] cellDependencies;
public List<Integer> contextRows; public List<Integer> contextRows;
@Override @Override
public String toString() { public String toString() {
return "Idx: "+recordIndex+" CellDeps: "+Arrays.toString(cellDependencies)+" Rows:"+contextRows; return "Idx: "+recordIndex+" CellDeps: "+Arrays.toString(cellDependencies)+" Rows:"+contextRows;
} }
} }
protected List<RowDependency> _rowDependencies; protected List<RowDependency> _rowDependencies;
protected List<Record> _records; protected List<Record> _records;
public RowDependency getRowDependency(int rowIndex) { public RowDependency getRowDependency(int rowIndex) {
return _rowDependencies != null && rowIndex >= 0 && rowIndex < _rowDependencies.size() ? return _rowDependencies != null && rowIndex >= 0 && rowIndex < _rowDependencies.size() ?
_rowDependencies.get(rowIndex) : null; _rowDependencies.get(rowIndex) : null;
} }
@JsonIgnore @JsonIgnore
public int getRecordCount() { public int getRecordCount() {
return _records.size(); return _records.size();
} }
public Record getRecord(int recordIndex) { public Record getRecord(int recordIndex) {
return _records != null && recordIndex >= 0 && recordIndex < _records.size() ? return _records != null && recordIndex >= 0 && recordIndex < _records.size() ?
_records.get(recordIndex) : null; _records.get(recordIndex) : null;
} }
public Record getRecordOfRow(int rowIndex) { public Record getRecordOfRow(int rowIndex) {
RowDependency rd = getRowDependency(rowIndex); RowDependency rd = getRowDependency(rowIndex);
if (rd != null) { if (rd != null) {
if (rd.recordIndex < 0) { if (rd.recordIndex < 0) {
rd = getRowDependency(rd.contextRows.get(0)); rd = getRowDependency(rd.contextRows.get(0));
} }
return getRecord(rd.recordIndex); return getRecord(rd.recordIndex);
} }
return null; return null;
} }
@JsonProperty("hasRecords") @JsonProperty("hasRecords")
public boolean hasRecords() { public boolean hasRecords() {
return _records != null && _rowDependencies != null && return _records != null && _rowDependencies != null &&
_records.size() < _rowDependencies.size(); _records.size() < _rowDependencies.size();
} }
static protected class KeyedGroup { static protected class KeyedGroup {
int[] cellIndices; int[] cellIndices;
int keyCellIndex; int keyCellIndex;
@Override @Override
public String toString() { public String toString() {
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
for (int i:cellIndices) { for (int i:cellIndices) {
sb.append(i).append(','); sb.append(i).append(',');
} }
return "key: " + keyCellIndex + " cells: " + sb.toString(); return "key: " + keyCellIndex + " cells: " + sb.toString();
} }
} }
synchronized public void update(Project project) { synchronized public void update(Project project) {
synchronized (project) { synchronized (project) {
List<Row> rows = project.rows; List<Row> rows = project.rows;
int rowCount = rows.size(); int rowCount = rows.size();
ColumnModel columnModel = project.columnModel; ColumnModel columnModel = project.columnModel;
List<KeyedGroup> keyedGroups = computeKeyedGroups(columnModel); List<KeyedGroup> keyedGroups = computeKeyedGroups(columnModel);
int groupCount = keyedGroups.size(); int groupCount = keyedGroups.size();
int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()]; int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()];
for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) { for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) {
lastNonBlankRowsByGroup[i] = -1; lastNonBlankRowsByGroup[i] = -1;
} }
_rowDependencies = new ArrayList<RowDependency>(rowCount); _rowDependencies = new ArrayList<RowDependency>(rowCount);
int recordIndex = 0; int recordIndex = 0;
for (int r = 0; r < rowCount; r++) { for (int r = 0; r < rowCount; r++) {
Row row = rows.get(r); Row row = rows.get(r);
RowDependency rowDependency = new RowDependency(); RowDependency rowDependency = new RowDependency();
for (int g = 0; g < groupCount; g++) { for (int g = 0; g < groupCount; g++) {
KeyedGroup group = keyedGroups.get(g); KeyedGroup group = keyedGroups.get(g);
if (!ExpressionUtils.isNonBlankData(row.getCellValue(keyedGroups.get(0).keyCellIndex)) && if (!ExpressionUtils.isNonBlankData(row.getCellValue(keyedGroups.get(0).keyCellIndex)) &&
!ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) { !ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) {
int contextRowIndex = lastNonBlankRowsByGroup[g]; int contextRowIndex = lastNonBlankRowsByGroup[g];
if (contextRowIndex >= 0) { if (contextRowIndex >= 0) {
for (int dependentCellIndex : group.cellIndices) { for (int dependentCellIndex : group.cellIndices) {
if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) { if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) {
setRowDependency( setRowDependency(
project, project,
rowDependency, rowDependency,
dependentCellIndex, dependentCellIndex,
contextRowIndex, contextRowIndex,
group.keyCellIndex group.keyCellIndex
); );
} }
} }
} }
} else { } else {
lastNonBlankRowsByGroup[g] = r; lastNonBlankRowsByGroup[g] = r;
} }
} }
if (rowDependency.cellDependencies != null && rowDependency.cellDependencies.length > 0) { if (rowDependency.cellDependencies != null && rowDependency.cellDependencies.length > 0) {
rowDependency.recordIndex = -1; rowDependency.recordIndex = -1;
rowDependency.contextRows = new ArrayList<Integer>(); rowDependency.contextRows = new ArrayList<Integer>();
for (CellDependency cd : rowDependency.cellDependencies) { for (CellDependency cd : rowDependency.cellDependencies) {
if (cd != null) { if (cd != null) {
rowDependency.contextRows.add(cd.rowIndex); rowDependency.contextRows.add(cd.rowIndex);
} }
} }
Collections.sort(rowDependency.contextRows); Collections.sort(rowDependency.contextRows);
} else { } else {
rowDependency.recordIndex = recordIndex++; rowDependency.recordIndex = recordIndex++;
} }
_rowDependencies.add(rowDependency); _rowDependencies.add(rowDependency);
} }
_records = new ArrayList<Record>(recordIndex); _records = new ArrayList<Record>(recordIndex);
if (recordIndex > 0) { if (recordIndex > 0) {
recordIndex = 0; recordIndex = 0;
int recordRowIndex = 0; int recordRowIndex = 0;
for (int r = 1; r < rowCount; r++) { for (int r = 1; r < rowCount; r++) {
RowDependency rd = _rowDependencies.get(r); RowDependency rd = _rowDependencies.get(r);
if (rd.recordIndex >= 0) { if (rd.recordIndex >= 0) {
_records.add(new Record(recordRowIndex, r, recordIndex++)); _records.add(new Record(recordRowIndex, r, recordIndex++));
recordIndex = rd.recordIndex; recordIndex = rd.recordIndex;
recordRowIndex = r; recordRowIndex = r;
} }
} }
_records.add(new Record(recordRowIndex, rowCount, recordIndex++)); _records.add(new Record(recordRowIndex, rowCount, recordIndex++));
} }
} }
} }
protected List<KeyedGroup> computeKeyedGroups(ColumnModel columnModel) { protected List<KeyedGroup> computeKeyedGroups(ColumnModel columnModel) {
List<KeyedGroup> keyedGroups = new ArrayList<KeyedGroup>(); List<KeyedGroup> keyedGroups = new ArrayList<KeyedGroup>();
addRootKeyedGroup(columnModel, keyedGroups); addRootKeyedGroup(columnModel, keyedGroups);
for (ColumnGroup group : columnModel.columnGroups) { for (ColumnGroup group : columnModel.columnGroups) {
if (group.keyColumnIndex >= 0) { if (group.keyColumnIndex >= 0) {
KeyedGroup keyedGroup = new KeyedGroup(); KeyedGroup keyedGroup = new KeyedGroup();
keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex(); keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex();
keyedGroup.cellIndices = new int[group.columnSpan - 1]; keyedGroup.cellIndices = new int[group.columnSpan - 1];
int c = 0; int c = 0;
for (int i = 0; i < group.columnSpan; i++) { for (int i = 0; i < group.columnSpan; i++) {
int columnIndex = group.startColumnIndex + i; int columnIndex = group.startColumnIndex + i;
if (columnIndex != group.keyColumnIndex && columnIndex < columnModel.columns.size()) { if (columnIndex != group.keyColumnIndex && columnIndex < columnModel.columns.size()) {
int cellIndex = columnModel.columns.get(columnIndex).getCellIndex(); int cellIndex = columnModel.columns.get(columnIndex).getCellIndex();
keyedGroup.cellIndices[c++] = cellIndex; keyedGroup.cellIndices[c++] = cellIndex;
} }
} }
keyedGroups.add(keyedGroup); keyedGroups.add(keyedGroup);
} }
} }
Collections.sort(keyedGroups, new Comparator<KeyedGroup>() { Collections.sort(keyedGroups, new Comparator<KeyedGroup>() {
@Override @Override
public int compare(KeyedGroup o1, KeyedGroup o2) { public int compare(KeyedGroup o1, KeyedGroup o2) {
return o2.cellIndices.length - o1.cellIndices.length; // larger groups first return o2.cellIndices.length - o1.cellIndices.length; // larger groups first
} }
}); });
dumpKeyedGroups(keyedGroups, columnModel); // for debug dumpKeyedGroups(keyedGroups, columnModel); // for debug
return keyedGroups; return keyedGroups;
} }
// debugging helper // debugging helper
private void dumpKeyedGroups(List<KeyedGroup> groups, ColumnModel columnModel) { private void dumpKeyedGroups(List<KeyedGroup> groups, ColumnModel columnModel) {
for (KeyedGroup g : groups) { for (KeyedGroup g : groups) {
String keyColName = columnModel.getColumnByCellIndex(g.keyCellIndex).getName(); String keyColName = columnModel.getColumnByCellIndex(g.keyCellIndex).getName();
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
for (int ci : g.cellIndices) { for (int ci : g.cellIndices) {
Column col = columnModel.getColumnByCellIndex(ci); Column col = columnModel.getColumnByCellIndex(ci);
if (col != null) { if (col != null) {
// Old projects have col 0 slot empty // Old projects have col 0 slot empty
sb.append(col.getName()).append(','); sb.append(col.getName()).append(',');
} }
} }
logger.trace("KeyedGroup " + keyColName + "::" + sb.toString()); logger.trace("KeyedGroup " + keyColName + "::" + sb.toString());
} }
} }
protected void addRootKeyedGroup(ColumnModel columnModel, List<KeyedGroup> keyedGroups) { protected void addRootKeyedGroup(ColumnModel columnModel, List<KeyedGroup> keyedGroups) {
int count = columnModel.getMaxCellIndex() + 1; int count = columnModel.getMaxCellIndex() + 1;
if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) { if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) {
KeyedGroup rootKeyedGroup = new KeyedGroup(); KeyedGroup rootKeyedGroup = new KeyedGroup();
rootKeyedGroup.cellIndices = new int[count - 1]; rootKeyedGroup.cellIndices = new int[count - 1];
rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex(); rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex();
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
if (i < rootKeyedGroup.keyCellIndex) { if (i < rootKeyedGroup.keyCellIndex) {
rootKeyedGroup.cellIndices[i] = i; rootKeyedGroup.cellIndices[i] = i;
} else if (i > rootKeyedGroup.keyCellIndex) { } else if (i > rootKeyedGroup.keyCellIndex) {
rootKeyedGroup.cellIndices[i - 1] = i; rootKeyedGroup.cellIndices[i - 1] = i;
} }
} }
keyedGroups.add(rootKeyedGroup); keyedGroups.add(rootKeyedGroup);
} }
} }
protected void setRowDependency( protected void setRowDependency(
Project project, Project project,
RowDependency rowDependency, RowDependency rowDependency,
int cellIndex, int cellIndex,
int contextRowIndex, int contextRowIndex,
int contextCellIndex int contextCellIndex
) { ) {
if (rowDependency.cellDependencies == null) { if (rowDependency.cellDependencies == null) {
int count = project.columnModel.getMaxCellIndex() + 1; int count = project.columnModel.getMaxCellIndex() + 1;
rowDependency.cellDependencies = new CellDependency[count]; rowDependency.cellDependencies = new CellDependency[count];
} }
rowDependency.cellDependencies[cellIndex] = rowDependency.cellDependencies[cellIndex] =
new CellDependency(contextRowIndex, contextCellIndex); new CellDependency(contextRowIndex, contextCellIndex);
} }
} }

View File

@ -1,84 +1,84 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010, Google Inc.
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are modification, are permitted provided that the following conditions are
met: met:
* Redistributions of source code must retain the above copyright * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the in the documentation and/or other materials provided with the
distribution. distribution.
* Neither the name of Google Inc. nor the names of its * Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from contributors may be used to endorse or promote products derived from
this software without specific prior written permission. this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package com.google.refine.model.changes; package com.google.refine.model.changes;
import java.io.IOException; import java.io.IOException;
import java.io.Writer; import java.io.Writer;
import java.util.Properties; import java.util.Properties;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
import com.google.refine.util.Pool; import com.google.refine.util.Pool;
public class CellAtRowCellIndex { public class CellAtRowCellIndex {
final public int row; final public int row;
final public int cellIndex; final public int cellIndex;
final public Cell cell; final public Cell cell;
final static private Pattern semicolonPattern = Pattern.compile(";"); final static private Pattern semicolonPattern = Pattern.compile(";");
public CellAtRowCellIndex(int row, int cellIndex, Cell cell) { public CellAtRowCellIndex(int row, int cellIndex, Cell cell) {
this.row = row; this.row = row;
this.cell = cell; this.cell = cell;
this.cellIndex = cellIndex; this.cellIndex = cellIndex;
} }
public void save(Writer writer, Properties options) throws IOException { public void save(Writer writer, Properties options) throws IOException {
writer.write(Integer.toString(row)); writer.write(Integer.toString(row));
writer.write(';'); writer.write(';');
writer.write(Integer.toString(cellIndex)); writer.write(Integer.toString(cellIndex));
writer.write(';'); writer.write(';');
if (cell != null) { if (cell != null) {
cell.save(writer, options); cell.save(writer, options);
} }
} }
static public CellAtRowCellIndex load(String s, Pool pool) throws Exception { static public CellAtRowCellIndex load(String s, Pool pool) throws Exception {
Matcher m = semicolonPattern.matcher(s); Matcher m = semicolonPattern.matcher(s);
m.find(); m.find();
int semicolon = m.start(); int semicolon = m.start();
m.find(); m.find();
int nextSemicolon = m.start(); int nextSemicolon = m.start();
int row = Integer.parseInt(s.substring(0, semicolon)); int row = Integer.parseInt(s.substring(0, semicolon));
int cellIndex = Integer.parseInt(s.substring(semicolon + 1, nextSemicolon)); int cellIndex = Integer.parseInt(s.substring(semicolon + 1, nextSemicolon));
Cell cell = nextSemicolon < s.length() - 1 ? Cell.loadStreaming(s.substring(nextSemicolon + 1), pool) Cell cell = nextSemicolon < s.length() - 1 ? Cell.loadStreaming(s.substring(nextSemicolon + 1), pool)
: null; : null;
return new CellAtRowCellIndex(row, cellIndex, cell); return new CellAtRowCellIndex(row, cellIndex, cell);
} }
} }

View File

@ -1,69 +1,69 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010, Google Inc.
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are modification, are permitted provided that the following conditions are
met: met:
* Redistributions of source code must retain the above copyright * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the in the documentation and/or other materials provided with the
distribution. distribution.
* Neither the name of Google Inc. nor the names of its * Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from contributors may be used to endorse or promote products derived from
this software without specific prior written permission. this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
class Facet { class Facet {
constructor(div, config, options) { constructor(div, config, options) {
this._div = div; this._div = div;
this._config = config; this._config = config;
this._options = options || {}; this._options = options || {};
this._minimizeState = false; this._minimizeState = false;
Refine.showLeftPanel(); Refine.showLeftPanel();
}; };
_minimize() { _minimize() {
if(!this._minimizeState) { if(!this._minimizeState) {
this._div.addClass("facet-state-minimize"); this._div.addClass("facet-state-minimize");
} else { } else {
this._div.removeClass("facet-state-minimize"); this._div.removeClass("facet-state-minimize");
} }
this._minimizeState = !this._minimizeState; this._minimizeState = !this._minimizeState;
}; };
_remove() { _remove() {
ui.browsingEngine.removeFacet(this); ui.browsingEngine.removeFacet(this);
this._div = null; this._div = null;
this._config = null; this._config = null;
this._selection = null; this._selection = null;
this._blankChoice = null; this._blankChoice = null;
this._errorChoice = null; this._errorChoice = null;
this._data = null; this._data = null;
this._options = null; this._options = null;
}; };
dispose() { dispose() {
}; };
}; };

View File

@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
font-size: 110%; font-size: 110%;
color: @near_black; color: @near_black;
white-space: nowrap; white-space: nowrap;
} }
#left-panel-body { #left-panel-body {
margin-left: @padding_tight; margin-left: @padding_tight;

View File

@ -1,104 +1,104 @@
/******************************************************************************* /*******************************************************************************
* Copyright (C) 2020, OpenRefine contributors * Copyright (C) 2020, OpenRefine contributors
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met: * modification, are permitted provided that the following conditions are met:
* *
* 1. Redistributions of source code must retain the above copyright notice, * 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer. * this list of conditions and the following disclaimer.
* *
* 2. Redistributions in binary form must reproduce the above copyright notice, * 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation * this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution. * and/or other materials provided with the distribution.
* *
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/ ******************************************************************************/
package com.google.refine; package com.google.refine;
import java.io.IOException; import java.io.IOException;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.eclipse.jetty.server.handler.HandlerWrapper; import org.eclipse.jetty.server.handler.HandlerWrapper;
import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Request;
/** /**
* Validate the Host header of the HTTP request to see if it matches either a loopback IP * Validate the Host header of the HTTP request to see if it matches either a loopback IP
* address, localhost or an explicitly specified hostname. This is required to avoid DNS * address, localhost or an explicitly specified hostname. This is required to avoid DNS
* rebinding attacks against users running OpenRefine on their desktop computers. * rebinding attacks against users running OpenRefine on their desktop computers.
*/ */
class ValidateHostHandler extends HandlerWrapper { class ValidateHostHandler extends HandlerWrapper {
/** /**
* Matches: * Matches:
* - addresses in the 127.0.0.0/8 subnet * - addresses in the 127.0.0.0/8 subnet
* - IPv4-mapped addresses in the ::ffff:7f00:00/104 subnet * - IPv4-mapped addresses in the ::ffff:7f00:00/104 subnet
* - different representations of ::1 * - different representations of ::1
* - localhost * - localhost
* Matching is a little fuzzy to simplify the regular expression - it expects the Host * Matching is a little fuzzy to simplify the regular expression - it expects the Host
* header to be well-formed. Some invalid addresses would be accepted, for example: * header to be well-formed. Some invalid addresses would be accepted, for example:
* - 127.6..64.245 * - 127.6..64.245
* - 0::0:::0:00:1 * - 0::0:::0:00:1
* This is not a problem however, as these are not valid DNS names either, and should * This is not a problem however, as these are not valid DNS names either, and should
* never be sent by a well-behaved browser - and validating the host header only ever * never be sent by a well-behaved browser - and validating the host header only ever
* helps if the browser works as expected and cannot be used to fake the Host header. * helps if the browser works as expected and cannot be used to fake the Host header.
*/ */
static private final Pattern LOOPBACK_PATTERN = Pattern static private final Pattern LOOPBACK_PATTERN = Pattern
.compile("^(?:127\\.[0-9\\.]*|\\[[0\\:]*\\:(?:ffff\\:7f[0-9a-f]{2}:[0-9a-f]{1,4}|0{0,3}1)\\]|localhost)(?:\\:[0-9]+)?$", Pattern.CASE_INSENSITIVE); .compile("^(?:127\\.[0-9\\.]*|\\[[0\\:]*\\:(?:ffff\\:7f[0-9a-f]{2}:[0-9a-f]{1,4}|0{0,3}1)\\]|localhost)(?:\\:[0-9]+)?$", Pattern.CASE_INSENSITIVE);
private String expectedHost; private String expectedHost;
public ValidateHostHandler(String expectedHost) { public ValidateHostHandler(String expectedHost) {
this.expectedHost = expectedHost; this.expectedHost = expectedHost;
} }
public boolean isValidHost(String host) { public boolean isValidHost(String host) {
// Allow loopback IPv4 and IPv6 addresses, as well as localhost // Allow loopback IPv4 and IPv6 addresses, as well as localhost
if (LOOPBACK_PATTERN.matcher(host).find()) { if (LOOPBACK_PATTERN.matcher(host).find()) {
return true; return true;
} }
// Strip port from hostname - for IPv6 addresses, if // Strip port from hostname - for IPv6 addresses, if
// they end with a bracket, then there is no port // they end with a bracket, then there is no port
int index = host.lastIndexOf(':'); int index = host.lastIndexOf(':');
if (index > 0 && !host.endsWith("]")) { if (index > 0 && !host.endsWith("]")) {
host = host.substring(0, index); host = host.substring(0, index);
} }
// Strip brackets from IPv6 addresses // Strip brackets from IPv6 addresses
if (host.startsWith("[") && host.endsWith("]")) { if (host.startsWith("[") && host.endsWith("]")) {
host = host.substring(1, host.length() - 2); host = host.substring(1, host.length() - 2);
} }
// Allow only if stripped hostname matches expected hostname // Allow only if stripped hostname matches expected hostname
return expectedHost.equalsIgnoreCase(host); return expectedHost.equalsIgnoreCase(host);
} }
@Override @Override
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
throws IOException, ServletException { throws IOException, ServletException {
String host = request.getHeader("Host"); String host = request.getHeader("Host");
if (isValidHost(host)) { if (isValidHost(host)) {
super.handle(target, baseRequest, request, response); super.handle(target, baseRequest, request, response);
} else { } else {
// Return HTTP 404 Not Found, since we are // Return HTTP 404 Not Found, since we are
// not serving content for the requested URL // not serving content for the requested URL
response.sendError(HttpServletResponse.SC_NOT_FOUND, "Invalid hostname"); response.sendError(HttpServletResponse.SC_NOT_FOUND, "Invalid hostname");
} }
} }
} }