From 8694374775ca33412af706752413a6789cdffdcb Mon Sep 17 00:00:00 2001 From: prance Date: Sun, 30 Jan 2022 22:02:21 +0100 Subject: [PATCH 1/2] idk??XD --- Dockerfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..6a0097a5d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM maven:3-jdk-11 as build + +WORKDIR /usr/src/app +COPY OpenRefine OpenRefine +WORKDIR /usr/src/app/OpenRefine + +RUN ./refine clean +RUN ./refine build + +FROM openjdk:11 + +WORKDIR /usr/app +COPY --from=build /usr/src/app/OpenRefine . + +VOLUME /data +EXPOSE 3333 + +ENTRYPOINT ["/usr/app/refine"] +CMD ["-i", "0.0.0.0", "-d", "/data", "-m", "3G"] From 2a7e2e66b522d34f9788524fcc2951c754e6b0ae Mon Sep 17 00:00:00 2001 From: prance Date: Sun, 30 Jan 2022 22:06:15 +0100 Subject: [PATCH 2/2] change OR version --- OpenRefine/.github/FUNDING.yml | 1 - .../.github/ISSUE_TEMPLATE/bug_report.md | 40 -- OpenRefine/.github/ISSUE_TEMPLATE/config.yml | 8 - .../.github/ISSUE_TEMPLATE/feature_request.md | 19 - OpenRefine/.github/SUPPORT.md | 5 - OpenRefine/.github/autolabeler.yml | 3 - OpenRefine/.github/dependabot.yml | 45 -- OpenRefine/.github/pull_request_template.md | 6 - .../.github/workflows/label_transfer.yml | 26 - .../workflows/label_transfer/requirements.txt | 2 - .../workflows/label_transfer/script.py | 76 --- OpenRefine/.github/workflows/pull_request.yml | 171 ----- .../.github/workflows/snapshot_release.yml | 177 ------ OpenRefine/Dockerfile | 19 + OpenRefine/ls1.txt | 29 + OpenRefine/ls2.txt | 31 + OpenRefine/ls3.txt | 2 + .../com/google/refine/model/RecordModel.java | 592 +++++++++--------- .../model/changes/CellAtRowCellIndex.java | 168 ++--- .../modules/core/scripts/facets/facet.js | 138 ++-- .../webapp/modules/core/styles/index.less | 2 +- .../google/refine/ValidateHostHandler.java | 206 +++--- 22 files changed, 634 insertions(+), 1132 deletions(-) delete mode 100644 OpenRefine/.github/FUNDING.yml delete mode 100644 OpenRefine/.github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 OpenRefine/.github/ISSUE_TEMPLATE/config.yml delete mode 100644 OpenRefine/.github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 OpenRefine/.github/SUPPORT.md delete mode 100644 OpenRefine/.github/autolabeler.yml delete mode 100644 OpenRefine/.github/dependabot.yml delete mode 100644 OpenRefine/.github/pull_request_template.md delete mode 100644 OpenRefine/.github/workflows/label_transfer.yml delete mode 100644 OpenRefine/.github/workflows/label_transfer/requirements.txt delete mode 100644 OpenRefine/.github/workflows/label_transfer/script.py delete mode 100644 OpenRefine/.github/workflows/pull_request.yml delete mode 100644 OpenRefine/.github/workflows/snapshot_release.yml create mode 100644 OpenRefine/Dockerfile create mode 100644 OpenRefine/ls1.txt create mode 100644 OpenRefine/ls2.txt create mode 100644 OpenRefine/ls3.txt diff --git a/OpenRefine/.github/FUNDING.yml b/OpenRefine/.github/FUNDING.yml deleted file mode 100644 index 944ccfa9d..000000000 --- a/OpenRefine/.github/FUNDING.yml +++ /dev/null @@ -1 +0,0 @@ -github: OpenRefine diff --git a/OpenRefine/.github/ISSUE_TEMPLATE/bug_report.md b/OpenRefine/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 7ff83400e..000000000 --- a/OpenRefine/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve OpenRefine -title: '' -labels: bug, to be reviewed -assignees: '' - ---- - - - -### To Reproduce -Steps to reproduce the behavior: -1. First, do ... -2. Then, ... -3. Finally, ... - - - -### Current Results - - -### Expected Behavior - - -### Screenshots - - -### Versions - - Operating System: - - Browser Version: - - JRE or JDK Version: - - OpenRefine: - -### Datasets - - -### Additional context - diff --git a/OpenRefine/.github/ISSUE_TEMPLATE/config.yml b/OpenRefine/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 8dbc191ab..000000000 --- a/OpenRefine/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,8 +0,0 @@ -blank_issues_enabled: false -contact_links: - - name: Ask a question (mailing list) - url: https://groups.google.com/d/forum/openrefine - about: Please ask and answer questions here. - - name: Gitter chat - url: https://gitter.im/OpenRefine/OpenRefine - about: General and developer discussions diff --git a/OpenRefine/.github/ISSUE_TEMPLATE/feature_request.md b/OpenRefine/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index e386dc313..000000000 --- a/OpenRefine/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for OpenRefine -title: '' -labels: enhancement, to be reviewed -assignees: '' - ---- - - - -### Proposed solution - - -### Alternatives considered - - -### Additional context - diff --git a/OpenRefine/.github/SUPPORT.md b/OpenRefine/.github/SUPPORT.md deleted file mode 100644 index 197b6d6fc..000000000 --- a/OpenRefine/.github/SUPPORT.md +++ /dev/null @@ -1,5 +0,0 @@ -If you are having trouble with OpenRefine we suggest the following steps: - -1. Read through our [FAQ (frequently Asked Questions)](https://github.com/OpenRefine/OpenRefine/wiki/FAQ) -2. Search for similar issues in our community email list archives: http://groups.google.com/group/openrefine/ -3. Send an email to our community mailing list: openrefine@googlegroups.com diff --git a/OpenRefine/.github/autolabeler.yml b/OpenRefine/.github/autolabeler.yml deleted file mode 100644 index 2f72abf88..000000000 --- a/OpenRefine/.github/autolabeler.yml +++ /dev/null @@ -1,3 +0,0 @@ -documentation: ["/docs"] -"current docs": ["/docs/docs"] -"historical docs": ["/docs/versioned_docs"] diff --git a/OpenRefine/.github/dependabot.yml b/OpenRefine/.github/dependabot.yml deleted file mode 100644 index 9d57a9ca1..000000000 --- a/OpenRefine/.github/dependabot.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Documentation for all configuration options: -# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: -# For openrefine java deps -- package-ecosystem: maven - directory: "/" - schedule: - interval: daily - open-pull-requests-limit: 10 - ignore: - - dependency-name: com.thoughtworks.xstream:xstream - versions: - - "> 1.4.12" - - "< 2" - -# Same, on 4.0 branch -- package-ecosystem: maven - directory: "/" - schedule: - interval: daily - open-pull-requests-limit: 10 - ignore: - - dependency-name: com.thoughtworks.xstream:xstream - versions: - - "> 1.4.12" - - "< 2" - target-branch: 4.0 - -# For documentation website -- package-ecosystem: "npm" # For Yarn - directory: "docs/" - schedule: - interval: "monthly" -# For github actions -- package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "monthly" -# For cypress test_suite -- package-ecosystem: "npm" - directory: "main/tests/cypress" - schedule: - interval: "daily" diff --git a/OpenRefine/.github/pull_request_template.md b/OpenRefine/.github/pull_request_template.md deleted file mode 100644 index 74cfebb6b..000000000 --- a/OpenRefine/.github/pull_request_template.md +++ /dev/null @@ -1,6 +0,0 @@ -Fixes #{issue number here} - -Changes proposed in this pull request: -- -- -- diff --git a/OpenRefine/.github/workflows/label_transfer.yml b/OpenRefine/.github/workflows/label_transfer.yml deleted file mode 100644 index 2bdec3360..000000000 --- a/OpenRefine/.github/workflows/label_transfer.yml +++ /dev/null @@ -1,26 +0,0 @@ - -name: Copy labels from issue to pull request - -on: - pull_request_target: - types: [opened, edited] - -jobs: - transfer_tags: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -r .github/workflows/label_transfer/requirements.txt - pip freeze - - name: Run Python label transfer script - run: python .github/workflows/label_transfer/script.py ${{ github.event.number }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPO: ${{ github.repository }} diff --git a/OpenRefine/.github/workflows/label_transfer/requirements.txt b/OpenRefine/.github/workflows/label_transfer/requirements.txt deleted file mode 100644 index 828889b82..000000000 --- a/OpenRefine/.github/workflows/label_transfer/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -lxml diff --git a/OpenRefine/.github/workflows/label_transfer/script.py b/OpenRefine/.github/workflows/label_transfer/script.py deleted file mode 100644 index 3855b61c9..000000000 --- a/OpenRefine/.github/workflows/label_transfer/script.py +++ /dev/null @@ -1,76 +0,0 @@ -import requests -import os -from lxml import html -import sys -import json - -# Config - -# list of labels that should not be transferred to PRs -do_not_transfer = [ - 'good first issue', - 'good second issue', - 'imported from old code repo', - 'help wanted', - 'duplicate', - 'invalid', - 'question', - 'to be reviewed', -] - -# Internal - -repo = os.environ.get('GITHUB_REPO') -github_token = os.environ.get('GITHUB_TOKEN') - -headers = { - 'Accept': 'application/vnd.github.v3+json', -} - -if github_token: - headers['Authorization'] = 'Bearer '+github_token - -def get_linked_issues(pr_number): - """ - Given a PR number, extract all the linked issue numbers from it. - Sadly this is not supported by the API yet, so we just scrape the web UI. - """ - url = f'https://github.com/{repo}/pull/{pr_number}' - page = requests.get(url) - page.raise_for_status() - parsed = html.document_fromstring(page.text) - matches = parsed.xpath('//form/div[@class="css-truncate my-1"]/a') - for match in matches: - yield int(match.attrib['href'].split('/')[-1]) - -def get_issue_labels(issue_number): - """ - Returns all the labels in a given issue / PR - """ - url = f'https://api.github.com/repos/{repo}/issues/{issue_number}/labels' - response = requests.get(url, headers=headers) - response.raise_for_status() - return [ tag['name'] for tag in response.json() ] - -def transfer_issue_labels(pr_number): - """ - Transfers labels from all the linked issues to the PR - """ - linked_issues = get_linked_issues(pr_number) - all_labels = [ label for issue in linked_issues for label in get_issue_labels(issue) ] - to_transfer = [ label for label in all_labels if label not in do_not_transfer ] - current_labels = get_issue_labels(pr_number) - missing_labels = [ label for label in to_transfer if label not in current_labels ] - if not missing_labels: - return - new_labels = current_labels + missing_labels - url = f'https://api.github.com/repos/{repo}/issues/{pr_number}/labels' - print(f'adding {missing_labels} to PR #{pr_number}') - if not github_token: - print('no GITHUB_TOKEN, skipping') - else: - resp = requests.put(url, headers=headers, data=json.dumps({'labels':new_labels})) - resp.raise_for_status() - -if __name__ == '__main__': - transfer_issue_labels(sys.argv[1]) diff --git a/OpenRefine/.github/workflows/pull_request.yml b/OpenRefine/.github/workflows/pull_request.yml deleted file mode 100644 index 151fcaff7..000000000 --- a/OpenRefine/.github/workflows/pull_request.yml +++ /dev/null @@ -1,171 +0,0 @@ -name: Continuous Integration - -on: - pull_request_target: - paths-ignore: - - 'docs/**' - branches: - - master - - '4.0' - -permissions: read-all - -jobs: - server_tests: - strategy: - matrix: - java: [ 11, 17 ] - - runs-on: ubuntu-latest - - services: - postgres: - image: postgres - ports: - - 5432 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: 'postgres' - POSTGRES_DB: test_db - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - mysql: - image: mysql:8 - ports: - - 3306 - env: - MYSQL_ROOT_PASSWORD: root - options: >- - --health-cmd "mysqladmin ping" - --health-interval 5s - --health-timeout 2s - --health-retries 3 - - steps: - - uses: actions/checkout@v2.3.4 - with: - ref: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - - - name: Restore dependency cache - uses: actions/cache@v2.1.7 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Set up Java ${{ matrix.java }} - uses: actions/setup-java@v2 - with: - distribution: 'adopt' - java-version: ${{ matrix.java }} - - - name: Check Java linting - id: java_linting - run: | - mvn formatter:validate - - - name: Configure connections to databases - id: configure_db_connections - run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml - - - name: Populate databases with test data - id: populate_databases_with_test_data - run: | - mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;' - mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql - psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql - env: - PGPASSWORD: postgres - - - name: Build and test with Maven - run: mvn jacoco:prepare-agent test - - - name: Submit test coverage to Coveralls - run: | - mvn prepare-package -DskipTests=true - mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }} - - prepare_ui_test_matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v2.3.4 - with: - ref: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: '12' - - id: set-matrix - run: npm install --save glob && node main/tests/cypress/build-test-matrix.js - env: - browsers: chrome - ui_test: - needs: prepare_ui_test_matrix - runs-on: ubuntu-latest - strategy: - matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}} - steps: - - uses: actions/checkout@v2.3.4 - with: - ref: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - - - name: Restore dependency cache - uses: actions/cache@v2.1.7 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Set up Java 11 - uses: actions/setup-java@v2 - with: - distribution: 'adopt' - java-version: 11 - - - name: Build OpenRefine - run: ./refine build - - - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: '12' - - - name: Restore Tests dependency cache - uses: actions/cache@v2.1.7 - with: - path: | - ~/cache - ~/.cache - **/node_modules - !~/cache/exclude - key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn - - - name: Install test dependencies - run: | - cd ./main/tests/cypress - npm i -g yarn - yarn install - - - name: Test with Cypress on ${{ matrix.browser }} - run: | - echo REFINE_MIN_MEMORY=1400M >> ./refine.ini - echo REFINE_MEMORY=4096M >> ./refine.ini - ./refine ui_tests - env: - CYPRESS_BROWSER: ${{ matrix.browser }} - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} - CYPRESS_CI_BUILD_ID: '${{ github.run_id }}' - CYPRESS_SPECS: ${{ matrix.specs }} diff --git a/OpenRefine/.github/workflows/snapshot_release.yml b/OpenRefine/.github/workflows/snapshot_release.yml deleted file mode 100644 index 772292888..000000000 --- a/OpenRefine/.github/workflows/snapshot_release.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: Snapshot release - -on: - push: - branches: - - 'master' - - '4.0' - paths-ignore: - - 'docs/**' - -jobs: - prepare_ui_test_matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v2.3.4 - with: - ref: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: '12' - - id: set-matrix - run: npm install --save glob && node main/tests/cypress/build-test-matrix.js - env: - browsers: chrome - ui_test: - needs: prepare_ui_test_matrix - runs-on: ubuntu-latest - strategy: - matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}} - steps: - - uses: actions/checkout@v2.3.4 - with: - ref: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - - - name: Restore dependency cache - uses: actions/cache@v2.1.7 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Set up Java 11 - uses: actions/setup-java@v2 - with: - distribution: 'adopt' - java-version: 11 - - - name: Build OpenRefine - run: ./refine build - - - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: '12' - - - name: Restore Tests dependency cache - uses: actions/cache@v2.1.7 - with: - path: | - ~/cache - ~/.cache - **/node_modules - !~/cache/exclude - key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn - - - name: Install test dependencies - run: | - cd ./main/tests/cypress - npm i -g yarn - yarn install - - - name: Test with Cypress on ${{ matrix.browser }} - run: | - echo REFINE_MIN_MEMORY=1400M >> ./refine.ini - echo REFINE_MEMORY=4096M >> ./refine.ini - ./refine ui_tests - env: - CYPRESS_BROWSER: ${{ matrix.browser }} - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} - CYPRESS_CI_BUILD_ID: '${{ github.run_id }}' - CYPRESS_SPECS: ${{ matrix.specs }} - - - build: - - services: - postgres: - image: postgres - ports: - - 5432 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: 'postgres' - POSTGRES_DB: test_db - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - mysql: - image: mysql:8 - ports: - - 3306 - env: - MYSQL_ROOT_PASSWORD: root - options: >- - --health-cmd "mysqladmin ping" - --health-interval 5s - --health-timeout 2s - --health-retries 3 - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2.3.4 - with: - fetch-depth: 0 # This is wasteful, but needed for git describe - - - name: Restore dependency cache - uses: actions/cache@v2.1.7 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Set up Java 11 - uses: actions/setup-java@v2 - with: - distribution: 'adopt' - java-version: 11 - server-id: ossrh - server-username: OSSRH_USER - server-password: OSSRH_PASS - - - name: Install genisoimage and jq - run: sudo apt-get install genisoimage jq - - - name: Configure connections to databases - id: configure_db_connections - run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml - - - name: Populate databases with test data - id: populate_databases_with_test_data - run: | - mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;' - mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql - psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql - env: - PGPASSWORD: postgres - - - name: Build and test with Maven - run: mvn jacoco:prepare-agent test - - - name: Submit test coverage to Coveralls - run: | - mvn prepare-package -DskipTests=true - mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }} -DserviceName="GitHub Actions" -DserviceBuildNumber=${{ env.GITHUB_RUN_ID }} -Dbranch=master - - - name: Generate dist files - run: mvn package -DskipTests=true - - - name: Upload snapshot releases to OSSRH - run: mvn deploy -DskipTests=true - env: - OSSRH_USER: ${{ secrets.OSSRH_USER }} - OSSRH_PASS: ${{ secrets.OSSRH_PASS }} - diff --git a/OpenRefine/Dockerfile b/OpenRefine/Dockerfile new file mode 100644 index 000000000..d2d9c4b0a --- /dev/null +++ b/OpenRefine/Dockerfile @@ -0,0 +1,19 @@ +FROM maven:3-jdk-11 as build + +WORKDIR /usr/src/app +COPY OpenRefine . +WORKDIR /usr/src/app/OpenRefine + +RUN ./refine clean +RUN ./refine build + +FROM openjdk:11 + +WORKDIR /usr/app +COPY --from=build /usr/src/app/OpenRefine . + +VOLUME /data +EXPOSE 3333 + +ENTRYPOINT ["/usr/app/refine"] +CMD ["-i", "0.0.0.0", "-d", "/data", "-m", "3G"] diff --git a/OpenRefine/ls1.txt b/OpenRefine/ls1.txt new file mode 100644 index 000000000..63c33b771 --- /dev/null +++ b/OpenRefine/ls1.txt @@ -0,0 +1,29 @@ +razem 164 +drwxr-xr-x 13 prance prance 4096 01-30 20:46 . +drwxr-xr-x 4 prance prance 4096 01-30 20:41 .. +-rw-r--r-- 1 prance prance 1603 01-30 20:42 appveyor.yml +drwxr-xr-x 3 prance prance 4096 01-30 20:42 benchmark +-rw-r--r-- 1 prance prance 3483 01-30 20:42 CODE_OF_CONDUCT.md +drwxr-xr-x 2 prance prance 4096 01-30 20:42 conf +-rw-r--r-- 1 prance prance 6438 01-30 20:42 CONTRIBUTING.md +drwxr-xr-x 7 prance prance 4096 01-30 20:42 docs +drwxr-xr-x 9 prance prance 4096 01-30 20:42 extensions +drwxr-xr-x 8 prance prance 4096 01-30 20:42 .git +-rw-r--r-- 1 prance prance 482 01-30 20:42 .gitattributes +drwxr-xr-x 4 prance prance 4096 01-30 20:42 .github +-rw-r--r-- 1 prance prance 1254 01-30 20:42 .gitignore +-rw-r--r-- 1 prance prance 8613 01-30 20:42 GOVERNANCE.md +drwxr-xr-x 4 prance prance 4096 01-30 20:42 graphics +drwxr-xr-x 3 prance prance 4096 01-30 20:42 IDEs +-rw-r--r-- 1 prance prance 1478 01-30 20:42 LICENSE.txt +-rw-r--r-- 1 prance prance 0 01-30 20:46 ls1.txt +drwxr-xr-x 6 prance prance 4096 01-30 20:42 main +drwxr-xr-x 2 prance prance 4096 01-30 20:42 packaging +-rw-r--r-- 1 prance prance 13518 01-30 20:42 pom.xml +-rw-r--r-- 1 prance prance 3523 01-30 20:42 README.md +-rwxr-xr-x 1 prance prance 29532 01-30 20:42 refine +-rw-r--r-- 1 prance prance 7680 01-30 20:42 refine.bat +-rw-r--r-- 1 prance prance 1549 01-30 20:42 refine.ini +-rw-r--r-- 1 prance prance 729 01-30 20:42 SECURITY.md +drwxr-xr-x 5 prance prance 4096 01-30 20:42 server +-rw-r--r-- 1 prance prance 1099 01-30 20:42 settings.xml diff --git a/OpenRefine/ls2.txt b/OpenRefine/ls2.txt new file mode 100644 index 000000000..493431eef --- /dev/null +++ b/OpenRefine/ls2.txt @@ -0,0 +1,31 @@ +razem 172 +drwxr-xr-x 14 prance prance 4096 01-30 20:47 . +drwxr-xr-x 4 prance prance 4096 01-30 20:41 .. +-rw-r--r-- 1 prance prance 1603 01-30 20:42 appveyor.yml +drwxr-xr-x 3 prance prance 4096 01-30 20:42 benchmark +-rw-r--r-- 1 prance prance 3483 01-30 20:42 CODE_OF_CONDUCT.md +drwxr-xr-x 2 prance prance 4096 01-30 20:42 conf +-rw-r--r-- 1 prance prance 6438 01-30 20:42 CONTRIBUTING.md +drwxr-xr-x 7 prance prance 4096 01-30 20:42 docs +drwxr-xr-x 9 prance prance 4096 01-30 20:42 extensions +drwxr-xr-x 8 prance prance 4096 01-30 20:42 .git +-rw-r--r-- 1 prance prance 482 01-30 20:42 .gitattributes +drwxr-xr-x 4 prance prance 4096 01-30 20:42 .github +-rw-r--r-- 1 prance prance 1254 01-30 20:42 .gitignore +-rw-r--r-- 1 prance prance 8613 01-30 20:42 GOVERNANCE.md +drwxr-xr-x 4 prance prance 4096 01-30 20:42 graphics +drwxr-xr-x 3 prance prance 4096 01-30 20:42 IDEs +-rw-r--r-- 1 prance prance 1478 01-30 20:42 LICENSE.txt +-rw-r--r-- 1 prance prance 1563 01-30 20:46 ls1.txt +-rw-r--r-- 1 prance prance 0 01-30 20:47 ls2.txt +drwxr-xr-x 6 prance prance 4096 01-30 20:42 main +drwxr-xr-x 2 prance prance 4096 01-30 20:42 packaging +-rw-r--r-- 1 prance prance 13518 01-30 20:42 pom.xml +-rw-r--r-- 1 prance prance 3523 01-30 20:42 README.md +-rwxr-xr-x 1 prance prance 29532 01-30 20:42 refine +-rw-r--r-- 1 prance prance 7680 01-30 20:42 refine.bat +-rw-r--r-- 1 prance prance 1549 01-30 20:42 refine.ini +-rw-r--r-- 1 prance prance 729 01-30 20:42 SECURITY.md +drwxr-xr-x 5 prance prance 4096 01-30 20:42 server +-rw-r--r-- 1 prance prance 1099 01-30 20:42 settings.xml +drwxr-xr-x 2 prance prance 4096 01-30 20:46 tools diff --git a/OpenRefine/ls3.txt b/OpenRefine/ls3.txt new file mode 100644 index 000000000..80c566e7e --- /dev/null +++ b/OpenRefine/ls3.txt @@ -0,0 +1,2 @@ +drwxr-xr-x 2 prance prance 4096 01-30 20:47 build +drwxr-xr-x 2 prance prance 4096 01-30 20:46 tools diff --git a/OpenRefine/main/src/com/google/refine/model/RecordModel.java b/OpenRefine/main/src/com/google/refine/model/RecordModel.java index 86836d837..843922241 100644 --- a/OpenRefine/main/src/com/google/refine/model/RecordModel.java +++ b/OpenRefine/main/src/com/google/refine/model/RecordModel.java @@ -1,296 +1,296 @@ -/* - -Copyright 2010, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -package com.google.refine.model; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.refine.expr.ExpressionUtils; - -public class RecordModel { - final static Logger logger = LoggerFactory.getLogger("RecordModel"); - - final static public class CellDependency { - final public int rowIndex; - final public int cellIndex; - - public CellDependency(int rowIndex, int cellIndex) { - this.rowIndex = rowIndex; - this.cellIndex = cellIndex; - } - - @Override - public String toString() { - return rowIndex+","+cellIndex; - } - } - - final static public class RowDependency { - public int recordIndex; - public CellDependency[] cellDependencies; - public List contextRows; - - @Override - public String toString() { - return "Idx: "+recordIndex+" CellDeps: "+Arrays.toString(cellDependencies)+" Rows:"+contextRows; - } - } - - protected List _rowDependencies; - protected List _records; - - public RowDependency getRowDependency(int rowIndex) { - return _rowDependencies != null && rowIndex >= 0 && rowIndex < _rowDependencies.size() ? - _rowDependencies.get(rowIndex) : null; - } - - @JsonIgnore - public int getRecordCount() { - return _records.size(); - } - - public Record getRecord(int recordIndex) { - return _records != null && recordIndex >= 0 && recordIndex < _records.size() ? - _records.get(recordIndex) : null; - } - - public Record getRecordOfRow(int rowIndex) { - RowDependency rd = getRowDependency(rowIndex); - if (rd != null) { - if (rd.recordIndex < 0) { - rd = getRowDependency(rd.contextRows.get(0)); - } - return getRecord(rd.recordIndex); - } - return null; - } - - @JsonProperty("hasRecords") - public boolean hasRecords() { - return _records != null && _rowDependencies != null && - _records.size() < _rowDependencies.size(); - } - - static protected class KeyedGroup { - int[] cellIndices; - int keyCellIndex; - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - for (int i:cellIndices) { - sb.append(i).append(','); - } - return "key: " + keyCellIndex + " cells: " + sb.toString(); - } - } - - synchronized public void update(Project project) { - synchronized (project) { - List rows = project.rows; - int rowCount = rows.size(); - - ColumnModel columnModel = project.columnModel; - List keyedGroups = computeKeyedGroups(columnModel); - int groupCount = keyedGroups.size(); - - int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()]; - for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) { - lastNonBlankRowsByGroup[i] = -1; - } - - _rowDependencies = new ArrayList(rowCount); - - int recordIndex = 0; - for (int r = 0; r < rowCount; r++) { - Row row = rows.get(r); - RowDependency rowDependency = new RowDependency(); - - for (int g = 0; g < groupCount; g++) { - KeyedGroup group = keyedGroups.get(g); - - if (!ExpressionUtils.isNonBlankData(row.getCellValue(keyedGroups.get(0).keyCellIndex)) && - !ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) { - int contextRowIndex = lastNonBlankRowsByGroup[g]; - if (contextRowIndex >= 0) { - for (int dependentCellIndex : group.cellIndices) { - if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) { - setRowDependency( - project, - rowDependency, - dependentCellIndex, - contextRowIndex, - group.keyCellIndex - ); - } - } - } - } else { - lastNonBlankRowsByGroup[g] = r; - } - } - - if (rowDependency.cellDependencies != null && rowDependency.cellDependencies.length > 0) { - rowDependency.recordIndex = -1; - rowDependency.contextRows = new ArrayList(); - for (CellDependency cd : rowDependency.cellDependencies) { - if (cd != null) { - rowDependency.contextRows.add(cd.rowIndex); - } - } - Collections.sort(rowDependency.contextRows); - } else { - rowDependency.recordIndex = recordIndex++; - } - - _rowDependencies.add(rowDependency); - } - - _records = new ArrayList(recordIndex); - if (recordIndex > 0) { - recordIndex = 0; - - int recordRowIndex = 0; - for (int r = 1; r < rowCount; r++) { - RowDependency rd = _rowDependencies.get(r); - if (rd.recordIndex >= 0) { - _records.add(new Record(recordRowIndex, r, recordIndex++)); - - recordIndex = rd.recordIndex; - recordRowIndex = r; - } - } - - _records.add(new Record(recordRowIndex, rowCount, recordIndex++)); - } - } - } - - protected List computeKeyedGroups(ColumnModel columnModel) { - List keyedGroups = new ArrayList(); - - addRootKeyedGroup(columnModel, keyedGroups); - - for (ColumnGroup group : columnModel.columnGroups) { - if (group.keyColumnIndex >= 0) { - KeyedGroup keyedGroup = new KeyedGroup(); - keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex(); - keyedGroup.cellIndices = new int[group.columnSpan - 1]; - - int c = 0; - for (int i = 0; i < group.columnSpan; i++) { - int columnIndex = group.startColumnIndex + i; - if (columnIndex != group.keyColumnIndex && columnIndex < columnModel.columns.size()) { - int cellIndex = columnModel.columns.get(columnIndex).getCellIndex(); - keyedGroup.cellIndices[c++] = cellIndex; - } - } - - keyedGroups.add(keyedGroup); - } - } - - Collections.sort(keyedGroups, new Comparator() { - @Override - public int compare(KeyedGroup o1, KeyedGroup o2) { - return o2.cellIndices.length - o1.cellIndices.length; // larger groups first - } - }); - - dumpKeyedGroups(keyedGroups, columnModel); // for debug - - return keyedGroups; - } - - // debugging helper - private void dumpKeyedGroups(List groups, ColumnModel columnModel) { - for (KeyedGroup g : groups) { - String keyColName = columnModel.getColumnByCellIndex(g.keyCellIndex).getName(); - StringBuffer sb = new StringBuffer(); - for (int ci : g.cellIndices) { - Column col = columnModel.getColumnByCellIndex(ci); - if (col != null) { - // Old projects have col 0 slot empty - sb.append(col.getName()).append(','); - } - } - logger.trace("KeyedGroup " + keyColName + "::" + sb.toString()); - } - } - - protected void addRootKeyedGroup(ColumnModel columnModel, List keyedGroups) { - int count = columnModel.getMaxCellIndex() + 1; - if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) { - KeyedGroup rootKeyedGroup = new KeyedGroup(); - - rootKeyedGroup.cellIndices = new int[count - 1]; - rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex(); - - for (int i = 0; i < count; i++) { - if (i < rootKeyedGroup.keyCellIndex) { - rootKeyedGroup.cellIndices[i] = i; - } else if (i > rootKeyedGroup.keyCellIndex) { - rootKeyedGroup.cellIndices[i - 1] = i; - } - } - keyedGroups.add(rootKeyedGroup); - } - } - - protected void setRowDependency( - Project project, - RowDependency rowDependency, - int cellIndex, - int contextRowIndex, - int contextCellIndex - ) { - if (rowDependency.cellDependencies == null) { - int count = project.columnModel.getMaxCellIndex() + 1; - - rowDependency.cellDependencies = new CellDependency[count]; - } - - rowDependency.cellDependencies[cellIndex] = - new CellDependency(contextRowIndex, contextCellIndex); - } - -} +/* + +Copyright 2010, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.model; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.refine.expr.ExpressionUtils; + +public class RecordModel { + final static Logger logger = LoggerFactory.getLogger("RecordModel"); + + final static public class CellDependency { + final public int rowIndex; + final public int cellIndex; + + public CellDependency(int rowIndex, int cellIndex) { + this.rowIndex = rowIndex; + this.cellIndex = cellIndex; + } + + @Override + public String toString() { + return rowIndex+","+cellIndex; + } + } + + final static public class RowDependency { + public int recordIndex; + public CellDependency[] cellDependencies; + public List contextRows; + + @Override + public String toString() { + return "Idx: "+recordIndex+" CellDeps: "+Arrays.toString(cellDependencies)+" Rows:"+contextRows; + } + } + + protected List _rowDependencies; + protected List _records; + + public RowDependency getRowDependency(int rowIndex) { + return _rowDependencies != null && rowIndex >= 0 && rowIndex < _rowDependencies.size() ? + _rowDependencies.get(rowIndex) : null; + } + + @JsonIgnore + public int getRecordCount() { + return _records.size(); + } + + public Record getRecord(int recordIndex) { + return _records != null && recordIndex >= 0 && recordIndex < _records.size() ? + _records.get(recordIndex) : null; + } + + public Record getRecordOfRow(int rowIndex) { + RowDependency rd = getRowDependency(rowIndex); + if (rd != null) { + if (rd.recordIndex < 0) { + rd = getRowDependency(rd.contextRows.get(0)); + } + return getRecord(rd.recordIndex); + } + return null; + } + + @JsonProperty("hasRecords") + public boolean hasRecords() { + return _records != null && _rowDependencies != null && + _records.size() < _rowDependencies.size(); + } + + static protected class KeyedGroup { + int[] cellIndices; + int keyCellIndex; + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + for (int i:cellIndices) { + sb.append(i).append(','); + } + return "key: " + keyCellIndex + " cells: " + sb.toString(); + } + } + + synchronized public void update(Project project) { + synchronized (project) { + List rows = project.rows; + int rowCount = rows.size(); + + ColumnModel columnModel = project.columnModel; + List keyedGroups = computeKeyedGroups(columnModel); + int groupCount = keyedGroups.size(); + + int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()]; + for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) { + lastNonBlankRowsByGroup[i] = -1; + } + + _rowDependencies = new ArrayList(rowCount); + + int recordIndex = 0; + for (int r = 0; r < rowCount; r++) { + Row row = rows.get(r); + RowDependency rowDependency = new RowDependency(); + + for (int g = 0; g < groupCount; g++) { + KeyedGroup group = keyedGroups.get(g); + + if (!ExpressionUtils.isNonBlankData(row.getCellValue(keyedGroups.get(0).keyCellIndex)) && + !ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) { + int contextRowIndex = lastNonBlankRowsByGroup[g]; + if (contextRowIndex >= 0) { + for (int dependentCellIndex : group.cellIndices) { + if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) { + setRowDependency( + project, + rowDependency, + dependentCellIndex, + contextRowIndex, + group.keyCellIndex + ); + } + } + } + } else { + lastNonBlankRowsByGroup[g] = r; + } + } + + if (rowDependency.cellDependencies != null && rowDependency.cellDependencies.length > 0) { + rowDependency.recordIndex = -1; + rowDependency.contextRows = new ArrayList(); + for (CellDependency cd : rowDependency.cellDependencies) { + if (cd != null) { + rowDependency.contextRows.add(cd.rowIndex); + } + } + Collections.sort(rowDependency.contextRows); + } else { + rowDependency.recordIndex = recordIndex++; + } + + _rowDependencies.add(rowDependency); + } + + _records = new ArrayList(recordIndex); + if (recordIndex > 0) { + recordIndex = 0; + + int recordRowIndex = 0; + for (int r = 1; r < rowCount; r++) { + RowDependency rd = _rowDependencies.get(r); + if (rd.recordIndex >= 0) { + _records.add(new Record(recordRowIndex, r, recordIndex++)); + + recordIndex = rd.recordIndex; + recordRowIndex = r; + } + } + + _records.add(new Record(recordRowIndex, rowCount, recordIndex++)); + } + } + } + + protected List computeKeyedGroups(ColumnModel columnModel) { + List keyedGroups = new ArrayList(); + + addRootKeyedGroup(columnModel, keyedGroups); + + for (ColumnGroup group : columnModel.columnGroups) { + if (group.keyColumnIndex >= 0) { + KeyedGroup keyedGroup = new KeyedGroup(); + keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex(); + keyedGroup.cellIndices = new int[group.columnSpan - 1]; + + int c = 0; + for (int i = 0; i < group.columnSpan; i++) { + int columnIndex = group.startColumnIndex + i; + if (columnIndex != group.keyColumnIndex && columnIndex < columnModel.columns.size()) { + int cellIndex = columnModel.columns.get(columnIndex).getCellIndex(); + keyedGroup.cellIndices[c++] = cellIndex; + } + } + + keyedGroups.add(keyedGroup); + } + } + + Collections.sort(keyedGroups, new Comparator() { + @Override + public int compare(KeyedGroup o1, KeyedGroup o2) { + return o2.cellIndices.length - o1.cellIndices.length; // larger groups first + } + }); + + dumpKeyedGroups(keyedGroups, columnModel); // for debug + + return keyedGroups; + } + + // debugging helper + private void dumpKeyedGroups(List groups, ColumnModel columnModel) { + for (KeyedGroup g : groups) { + String keyColName = columnModel.getColumnByCellIndex(g.keyCellIndex).getName(); + StringBuffer sb = new StringBuffer(); + for (int ci : g.cellIndices) { + Column col = columnModel.getColumnByCellIndex(ci); + if (col != null) { + // Old projects have col 0 slot empty + sb.append(col.getName()).append(','); + } + } + logger.trace("KeyedGroup " + keyColName + "::" + sb.toString()); + } + } + + protected void addRootKeyedGroup(ColumnModel columnModel, List keyedGroups) { + int count = columnModel.getMaxCellIndex() + 1; + if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) { + KeyedGroup rootKeyedGroup = new KeyedGroup(); + + rootKeyedGroup.cellIndices = new int[count - 1]; + rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex(); + + for (int i = 0; i < count; i++) { + if (i < rootKeyedGroup.keyCellIndex) { + rootKeyedGroup.cellIndices[i] = i; + } else if (i > rootKeyedGroup.keyCellIndex) { + rootKeyedGroup.cellIndices[i - 1] = i; + } + } + keyedGroups.add(rootKeyedGroup); + } + } + + protected void setRowDependency( + Project project, + RowDependency rowDependency, + int cellIndex, + int contextRowIndex, + int contextCellIndex + ) { + if (rowDependency.cellDependencies == null) { + int count = project.columnModel.getMaxCellIndex() + 1; + + rowDependency.cellDependencies = new CellDependency[count]; + } + + rowDependency.cellDependencies[cellIndex] = + new CellDependency(contextRowIndex, contextCellIndex); + } + +} diff --git a/OpenRefine/main/src/com/google/refine/model/changes/CellAtRowCellIndex.java b/OpenRefine/main/src/com/google/refine/model/changes/CellAtRowCellIndex.java index 2de9d1f96..90c645dac 100644 --- a/OpenRefine/main/src/com/google/refine/model/changes/CellAtRowCellIndex.java +++ b/OpenRefine/main/src/com/google/refine/model/changes/CellAtRowCellIndex.java @@ -1,84 +1,84 @@ -/* - -Copyright 2010, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -package com.google.refine.model.changes; - -import java.io.IOException; -import java.io.Writer; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.google.refine.model.Cell; -import com.google.refine.util.Pool; - -public class CellAtRowCellIndex { - - final public int row; - final public int cellIndex; - final public Cell cell; - final static private Pattern semicolonPattern = Pattern.compile(";"); - - public CellAtRowCellIndex(int row, int cellIndex, Cell cell) { - this.row = row; - this.cell = cell; - this.cellIndex = cellIndex; - } - - public void save(Writer writer, Properties options) throws IOException { - writer.write(Integer.toString(row)); - writer.write(';'); - writer.write(Integer.toString(cellIndex)); - writer.write(';'); - if (cell != null) { - cell.save(writer, options); - } - } - - static public CellAtRowCellIndex load(String s, Pool pool) throws Exception { - - Matcher m = semicolonPattern.matcher(s); - - m.find(); - int semicolon = m.start(); - m.find(); - int nextSemicolon = m.start(); - - int row = Integer.parseInt(s.substring(0, semicolon)); - int cellIndex = Integer.parseInt(s.substring(semicolon + 1, nextSemicolon)); - Cell cell = nextSemicolon < s.length() - 1 ? Cell.loadStreaming(s.substring(nextSemicolon + 1), pool) - : null; - - return new CellAtRowCellIndex(row, cellIndex, cell); - } -} +/* + +Copyright 2010, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.model.changes; + +import java.io.IOException; +import java.io.Writer; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.refine.model.Cell; +import com.google.refine.util.Pool; + +public class CellAtRowCellIndex { + + final public int row; + final public int cellIndex; + final public Cell cell; + final static private Pattern semicolonPattern = Pattern.compile(";"); + + public CellAtRowCellIndex(int row, int cellIndex, Cell cell) { + this.row = row; + this.cell = cell; + this.cellIndex = cellIndex; + } + + public void save(Writer writer, Properties options) throws IOException { + writer.write(Integer.toString(row)); + writer.write(';'); + writer.write(Integer.toString(cellIndex)); + writer.write(';'); + if (cell != null) { + cell.save(writer, options); + } + } + + static public CellAtRowCellIndex load(String s, Pool pool) throws Exception { + + Matcher m = semicolonPattern.matcher(s); + + m.find(); + int semicolon = m.start(); + m.find(); + int nextSemicolon = m.start(); + + int row = Integer.parseInt(s.substring(0, semicolon)); + int cellIndex = Integer.parseInt(s.substring(semicolon + 1, nextSemicolon)); + Cell cell = nextSemicolon < s.length() - 1 ? Cell.loadStreaming(s.substring(nextSemicolon + 1), pool) + : null; + + return new CellAtRowCellIndex(row, cellIndex, cell); + } +} diff --git a/OpenRefine/main/webapp/modules/core/scripts/facets/facet.js b/OpenRefine/main/webapp/modules/core/scripts/facets/facet.js index 36af6a843..55442a6af 100644 --- a/OpenRefine/main/webapp/modules/core/scripts/facets/facet.js +++ b/OpenRefine/main/webapp/modules/core/scripts/facets/facet.js @@ -1,69 +1,69 @@ -/* - -Copyright 2010, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - */ - -class Facet { - constructor(div, config, options) { - this._div = div; - this._config = config; - this._options = options || {}; - this._minimizeState = false; - - Refine.showLeftPanel(); - }; - - _minimize() { - if(!this._minimizeState) { - this._div.addClass("facet-state-minimize"); - } else { - this._div.removeClass("facet-state-minimize"); - } - - this._minimizeState = !this._minimizeState; - }; - - _remove() { - ui.browsingEngine.removeFacet(this); - - this._div = null; - this._config = null; - - this._selection = null; - this._blankChoice = null; - this._errorChoice = null; - this._data = null; - this._options = null; - }; - - dispose() { - }; -}; +/* + +Copyright 2010, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + */ + +class Facet { + constructor(div, config, options) { + this._div = div; + this._config = config; + this._options = options || {}; + this._minimizeState = false; + + Refine.showLeftPanel(); + }; + + _minimize() { + if(!this._minimizeState) { + this._div.addClass("facet-state-minimize"); + } else { + this._div.removeClass("facet-state-minimize"); + } + + this._minimizeState = !this._minimizeState; + }; + + _remove() { + ui.browsingEngine.removeFacet(this); + + this._div = null; + this._config = null; + + this._selection = null; + this._blankChoice = null; + this._errorChoice = null; + this._data = null; + this._options = null; + }; + + dispose() { + }; +}; diff --git a/OpenRefine/main/webapp/modules/core/styles/index.less b/OpenRefine/main/webapp/modules/core/styles/index.less index 3905ae01a..228cee706 100644 --- a/OpenRefine/main/webapp/modules/core/styles/index.less +++ b/OpenRefine/main/webapp/modules/core/styles/index.less @@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. font-size: 110%; color: @near_black; white-space: nowrap; - } +} #left-panel-body { margin-left: @padding_tight; diff --git a/OpenRefine/server/src/com/google/refine/ValidateHostHandler.java b/OpenRefine/server/src/com/google/refine/ValidateHostHandler.java index 2b867cbe0..f41da8a17 100644 --- a/OpenRefine/server/src/com/google/refine/ValidateHostHandler.java +++ b/OpenRefine/server/src/com/google/refine/ValidateHostHandler.java @@ -1,104 +1,104 @@ -/******************************************************************************* - * Copyright (C) 2020, OpenRefine contributors - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - ******************************************************************************/ - -package com.google.refine; - -import java.io.IOException; -import java.util.regex.Pattern; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.eclipse.jetty.server.handler.HandlerWrapper; -import org.eclipse.jetty.server.Request; - -/** - * Validate the Host header of the HTTP request to see if it matches either a loopback IP - * address, localhost or an explicitly specified hostname. This is required to avoid DNS - * rebinding attacks against users running OpenRefine on their desktop computers. - */ -class ValidateHostHandler extends HandlerWrapper { - - /** - * Matches: - * - addresses in the 127.0.0.0/8 subnet - * - IPv4-mapped addresses in the ::ffff:7f00:00/104 subnet - * - different representations of ::1 - * - localhost - * Matching is a little fuzzy to simplify the regular expression - it expects the Host - * header to be well-formed. Some invalid addresses would be accepted, for example: - * - 127.6..64.245 - * - 0::0:::0:00:1 - * This is not a problem however, as these are not valid DNS names either, and should - * never be sent by a well-behaved browser - and validating the host header only ever - * helps if the browser works as expected and cannot be used to fake the Host header. - */ - static private final Pattern LOOPBACK_PATTERN = Pattern - .compile("^(?:127\\.[0-9\\.]*|\\[[0\\:]*\\:(?:ffff\\:7f[0-9a-f]{2}:[0-9a-f]{1,4}|0{0,3}1)\\]|localhost)(?:\\:[0-9]+)?$", Pattern.CASE_INSENSITIVE); - - private String expectedHost; - - public ValidateHostHandler(String expectedHost) { - this.expectedHost = expectedHost; - } - - public boolean isValidHost(String host) { - - // Allow loopback IPv4 and IPv6 addresses, as well as localhost - if (LOOPBACK_PATTERN.matcher(host).find()) { - return true; - } - - // Strip port from hostname - for IPv6 addresses, if - // they end with a bracket, then there is no port - int index = host.lastIndexOf(':'); - if (index > 0 && !host.endsWith("]")) { - host = host.substring(0, index); - } - - // Strip brackets from IPv6 addresses - if (host.startsWith("[") && host.endsWith("]")) { - host = host.substring(1, host.length() - 2); - } - - // Allow only if stripped hostname matches expected hostname - return expectedHost.equalsIgnoreCase(host); - } - - @Override - public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) - throws IOException, ServletException { - String host = request.getHeader("Host"); - if (isValidHost(host)) { - super.handle(target, baseRequest, request, response); - } else { - // Return HTTP 404 Not Found, since we are - // not serving content for the requested URL - response.sendError(HttpServletResponse.SC_NOT_FOUND, "Invalid hostname"); - } - } - +/******************************************************************************* + * Copyright (C) 2020, OpenRefine contributors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ + +package com.google.refine; + +import java.io.IOException; +import java.util.regex.Pattern; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.eclipse.jetty.server.handler.HandlerWrapper; +import org.eclipse.jetty.server.Request; + +/** + * Validate the Host header of the HTTP request to see if it matches either a loopback IP + * address, localhost or an explicitly specified hostname. This is required to avoid DNS + * rebinding attacks against users running OpenRefine on their desktop computers. + */ +class ValidateHostHandler extends HandlerWrapper { + + /** + * Matches: + * - addresses in the 127.0.0.0/8 subnet + * - IPv4-mapped addresses in the ::ffff:7f00:00/104 subnet + * - different representations of ::1 + * - localhost + * Matching is a little fuzzy to simplify the regular expression - it expects the Host + * header to be well-formed. Some invalid addresses would be accepted, for example: + * - 127.6..64.245 + * - 0::0:::0:00:1 + * This is not a problem however, as these are not valid DNS names either, and should + * never be sent by a well-behaved browser - and validating the host header only ever + * helps if the browser works as expected and cannot be used to fake the Host header. + */ + static private final Pattern LOOPBACK_PATTERN = Pattern + .compile("^(?:127\\.[0-9\\.]*|\\[[0\\:]*\\:(?:ffff\\:7f[0-9a-f]{2}:[0-9a-f]{1,4}|0{0,3}1)\\]|localhost)(?:\\:[0-9]+)?$", Pattern.CASE_INSENSITIVE); + + private String expectedHost; + + public ValidateHostHandler(String expectedHost) { + this.expectedHost = expectedHost; + } + + public boolean isValidHost(String host) { + + // Allow loopback IPv4 and IPv6 addresses, as well as localhost + if (LOOPBACK_PATTERN.matcher(host).find()) { + return true; + } + + // Strip port from hostname - for IPv6 addresses, if + // they end with a bracket, then there is no port + int index = host.lastIndexOf(':'); + if (index > 0 && !host.endsWith("]")) { + host = host.substring(0, index); + } + + // Strip brackets from IPv6 addresses + if (host.startsWith("[") && host.endsWith("]")) { + host = host.substring(1, host.length() - 2); + } + + // Allow only if stripped hostname matches expected hostname + return expectedHost.equalsIgnoreCase(host); + } + + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException { + String host = request.getHeader("Host"); + if (isValidHost(host)) { + super.handle(target, baseRequest, request, response); + } else { + // Return HTTP 404 Not Found, since we are + // not serving content for the requested URL + response.sendError(HttpServletResponse.SC_NOT_FOUND, "Invalid hostname"); + } + } + } \ No newline at end of file