s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update OpenRefine

Browse Source
This commit is contained in:
Michał Najborowski 2022-01-30 21:12:14 +01:00
parent b633a4c51e
commit b37b4153bf
494 changed files with 9347 additions and 9234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
OpenRefine/.github/dependabot.yml vendored
View File

@ -14,16 +14,30 @@ updates:
versions: versions:
- "> 1.4.12" - "> 1.4.12"
- "< 2" - "< 2"
# Same, on 4.0 branch
- package-ecosystem: maven
directory: "/"
schedule:
interval: daily
open-pull-requests-limit: 10
ignore:
- dependency-name: com.thoughtworks.xstream:xstream
versions:
- "> 1.4.12"
- "< 2"
target-branch: 4.0
# For documentation website # For documentation website
- package-ecosystem: "npm" # For Yarn - package-ecosystem: "npm" # For Yarn
directory: "docs/" directory: "docs/"
schedule: schedule:
interval: "daily" interval: "monthly"
# For github actions # For github actions
- package-ecosystem: "github-actions" - package-ecosystem: "github-actions"
directory: "/" directory: "/"
schedule: schedule:
interval: "daily" interval: "monthly"
# For cypress test_suite # For cypress test_suite
- package-ecosystem: "npm" - package-ecosystem: "npm"
directory: "main/tests/cypress" directory: "main/tests/cypress"

7
OpenRefine/.github/workflows/pull_request.yml vendored
View File

@ -14,7 +14,7 @@ jobs:
server_tests: server_tests:
strategy: strategy:
matrix: matrix:
java: [ 11, 14 ] java: [ 11, 17 ]
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -64,6 +64,11 @@ jobs:
distribution: 'adopt' distribution: 'adopt'
java-version: ${{ matrix.java }} java-version: ${{ matrix.java }}
- name: Check Java linting
id: java_linting
run: |
mvn formatter:validate
- name: Configure connections to databases - name: Configure connections to databases
id: configure_db_connections id: configure_db_connections
run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml

78
OpenRefine/AUTHORS.md
View File

@ -1,78 +0,0 @@
This file lists the contributors to OpenRefine.
EMERITUS CREATORS
---------------------
No longer active but honored for bringing OpenRefine to life.
- David Huynh
- Stefano Mazzocchi
EMERITUS CONTRIBUTORS
---------------------
No longer active but honored here for their previous contributions
- Vishal Talwar
- Jeff Fry
- Will Moffat
- James Home
- Heather Campbell
CURRENT CONTRIBUTORS
--------------------
See up to date list at https://github.com/OpenRefine/OpenRefine/graphs/contributors
- Iain Sproat
- Tom Morris
- Thad Guidry
- Martin Magdinier
- Paul Makepeace
- Tomaž Šolc
- Gabriel Sjoberg
- Rod Salazar
- pxb
- Qi
- Antonin Delpeuch
- Owen Stephens
- Ettore Rizza
- Fabio Tacchelli
- noamoss
- ROMitat
- Jesus M. Castagnetto
- Pablo Moyano
- David Leoni
- Cora Johnson-Roberson
- Pei Long Hui
- Rudy Alvarez
- Mateja Verlic Bruncic
- nestorjal
- Alexey Medvetsky
- Remi Rampin
- lispc
- Bob Harper
- Felix Lohmeier
- Shixiong Zhu
- Ankit Sardesai
- Scott Wiedemann
- Ryo Yamazaki
- Ralf Claussnitzer
- Charles Pritchard
- Maxim Galushka
- Evelyn Mitchell
- Andreas Kohn
- Honza
- Ram Ezrach
- Daniel Berthereau
- Gideon Thomas
- José Vitor Hisse
- Vitor Baptista
- Joe Wicentowski
- mpc9000
- Dan Michael O.
- Adi Eyal
- Shrubhra Sharma
- Fadi Maali
- Aubrey Mcfato
- Matthias Findeisen
- Mathieu Saby
- Allan Nordhøy
- Tony Opara

284
OpenRefine/CHANGES.txt
View File

@ -1,284 +0,0 @@
2.6 Release (TBD) - First release as OpenRefine - Maintenance release
Features:
- Issue 144: Use subdirectory for archives (tar, zip)
- Issue 467: Provide feedback to user on JVM heap memory usage
- Issue 524: Shorten name of anonymous JSON nodes in column names
Bugs fixed:
- Issue 154: Can't import RDF/XML Data
- Issue 226: Load into freebase: date objects are not converted into proper freebase dates
- Issue 361: Headless operation on OS X leaves Dock icon bouncing
- Issue 390: Latest POI fixes some import issues
- Issue 432: cross() failing
- Issue 436: Google refine don't start on debian squeeze
- Issue 490: Unable to guess filetype is CSV when importing a simple CSV file
- Issue 496: google-refine-2.5-rc2-r2379 failed to start if using refine.ini
- Issue 515: "Open Project" page shows negative interval for "Last Modified"
- Issue 517: Combin() function does not work as intended
- Issue 521: Machine readable string (/type/rawstring) missing
- Issue 523: Return all available HTTP error information on when Fetch URL fails
- Issue 535: Refine fails to import Excel 2010 XLSX file with null hyperlinks
- Issue 537: ToNumber should try to convert to integer first
- Issue 541: Can't constrain a Freebase key query when using add column
- Issue 542: ToDate should work on integers
- Issue 543: Project creation from URL can't handle gzip Content-Encoding
- Issue 544: FileNotFound exception when importing archive from URL
- Issue 548: escape(value, encoding) returns "null" for non-string values
- Issue 551: MQL preview missing "insert" clause
- Issue 553: Upload Error - Not a directory
- Issue 554: Format guesser ranking isn't working correctly in some cases
- Issue 558: 'refine windows_dist' does not work on MacOS Lion
- Issue 559: Deadlock between autosave thread and history code
- Issue 578: json import on create project deletes useful blank data from arrays
- Issue 586: Only one parse date format is attempted from list in toDate(format1,format2)
- Issue 594: Date diff function doesn't work for two Calendar objects
- Issue 596: JSON importer should not deserialize nulls as the string "null"
- Issue 597: Deselect all button on Custom Tabular Exporter dialog doesn't work
- Issue 599: RDF/XML parser preview not wired up
- Issue 601: Allow selection of root element on JSON import
- Issue 604: The common transform “Trim leading and trailing whitespace” doesn’t trim non-breaking spaces
2.5 Release (December 11, 2011)
Major changes:
- Entirely new importer architecture and user interface for Create Project
- New import/export formats: Google Fusion Tables, OpenOffice Calc,
and fixed width (import only)
Features:
- Issue 131: Read multiple files in a directory
- Issue 179: Progress feedback during upload
- Issue 260: XML Importer silently fails if file doesn't meet its criteria
- Issue 279: Add support for reading from non-public Google Spreadsheets docs
- Issue 280: Spreadsheet/table importers should allow selection of sheet/table to import
- Issue 281: Add sheet selection support to Google Spreadsheet importer
- Issue 31: Maximum number of facet values should be configurable.
- Issue 38: Fix the table header so that it's always visible when scrolling a long page
- Issue 433: Usability: use HTML <label> (e.g., for checkboxes
- Issue 447: Extend toTitlecase() function with support for char[] delimiters in Apache WordUtils
- Issue 452: Importing using Clipboard function does not guess structure correctly for XML or JSON
- Issue 483: Project custom metadata while importing
- Issue 504: Allow "sort" keyword in constraint subquery for "Add from Freebase"
- Issue 84: Add a cut and paste textbox importer
- Issue 85: Fixed width column importer
- Issue 88: Improve operation history robustness by fixing Json.
- Issue 92: Add Columns From Freebase dialog should support constraints
- Issue 97: Exporting CSV should allow for optional columns
Bugs fixed:
- Issue 149: rows button stays strike-through when clicking on it
- Issue 229: 'Ignore lines on import' bug when import from URL?
- Issue 232: Stray characters before <?xml> (including blank lines) causes XML import to fail
- Issue 233: 404 Error upon Launch
- Issue 313: Excel export loses date cell format
- Issue 314: Reconciliation (Freebase) of accented characters doesn't work
- Issue 330: On Project creation - Unchecked Split Columns Box still splits by comma
- Issue 366: Error when using Add column from Freebase on restored project
- Issue 375: Importing data from Google spreadsheet via visualisation API query
- Issue 378: Transform involving NaN causes crash
- Issue 383: Updating Preview hangs on 2.5 branch when checking Load at most...
- Issue 410: Convert line-endings to DOS format
- Issue 419: Values with Characters like é split across several lines
- Issue 424: Duplicate column names don't allow user to recover
- Issue 426: filter with custom facet adds zero lines choice
- Issue 428: Excel import sometimes drops last row of data
- Issue 430: Timeline Facet not working
- Issue 435: refine sh script only checks for java 1.6, not > 1.6
- Issue 440: Fetch URLs aborts while saving/loading/computing facets
- Issue 441: onError - "keep-original" / "store-blank" working oddly for value.toDate()
- Issue 442: Two column transforms to date on the same column turns the cells blank
- Issue 449: Uncaught exception from Excel importer
- Issue 450: Parse error uploading to Fusion Table
- Issue 454: Reselect Files after Configuring Parsing Options does not allow Select
- Issue 459: Undefined error with some CSV files (incorrectly detected as EXCEL)
- Issue 462: Don't trim whitespace in Excel importer
- Issue 465: Data text file with extension .dta within a .ZIP is not automatically extracted
- Issue 473: Cancel button doesn't work during project creation
- Issue 474: XML importer ignores record limit
- Issue 475: New Importer does not accept special separator characters completely such as Unicode chars
- Issue 477: Implement or remove the line separator option
- Issue 487: Transform -> To Date doesn't support ISO 8601 date parsing
- Issue 488: ISO 8601 dates not supported in cell editing - cell-ui.js
- Issue 489: Custom Exporter no longer behaves as expected with line & field separators
- Issue 491: Importing Excel files - blank columns and time formatting
- Issue 492: Table header cells misaligning with table cells
- Issue 502: Fetch URLs does not return the exact HTTP payload, like Create Project from URLs does.
- Issue 513: No more Json Tokens in stream ERROR
2.1 Release (July 10, 2011) - Maintenance release
Features:
- Issue 157: Google Spreadsheet import/export plugin integrated
- Issue 220: HTML parsing functions added to GREL (using Jsoup)
- Issue 222: Added the ability to save favorite transforms
- Issue 224: added ABS math function
- Issue 349: Clustering not finding duplicates when facet is showing groupings
- Issue 399: Added Cologne Phonetic clustering for German names
Bugs fixed:
- Issue 102: Process-Panel div sometimes disappears in Google Chrome
- Issue 107: Reconciliation options not showing international characters correctly
- Issue 143: Windows refine.bat file gives NoClassDefFoundError
- Issue 163: Refine doesn't retain the characters for flat or sharp
- Issue 172: inconsistencies in encoding guessing during load
- Issue 184: Support formatting dates into strings
- Issue 185: same reconciliation candidate for two cells seems to be overridden
- Issue 187: Linux start instruction incorrect
- Issue 188: ArrayIndexOutOfBoundsException when importing excel file
- Issue 196: failure and error dialog attempting to remove columns
- Issue 197: New project was modified "0 years ago"
- Issue 202: Sort text with accents
- Issue 203: Please add TextPipe to Related Software page
- Issue 227: Documentation states extensions are mounted at /extensions, but they are really mounted at /extension.
- Issue 228: Import .zip archive fails
- Issue 237: reinterpret() no longer seems to work as expected
- Issue 258: JAVA_HOME ignored if "which java" returns nonzero
- Issue 262: Refine does not start
- Issue 263: & symbol being parsed in url's in xml
- Issue 276: strange character while create new project with a chinese project name
- Issue 294: Exporting date type column to TSV/CSV shows java debugging information instead of value
- Issue 295: Binaries under 'root' user on *nix
- Issue 304: CsvExporter tests fail after commit for issue 294
- Issue 311: "Ignore" and "Skip" fields in the "Create a New Project" form have the same input[name]
- Issue 312: toString(date_val, 'yyyy-MM-dd') doesn't seem to work
- Issue 325: Refine cannot load data from URL when run behind Proxy Server
- Issue 328: Don't retry failed key-based reconciliations
- Issue 334: Fusion Tables CSV import produces NullPointerException
- Issue 351: Error 500 STREAM on Excel export
- Issue 355: Broken images for SchemaAlignment page
- Issue 358: org.json.JSONException: JSON does not allow non-finite numbers.
- Issue 364: Build is broken
- Issue 374: Problems with date functions INC
- Issue 391: Patch to refine.{bat,ini}
- Issue 401: Error in exception handling for ExportRows command
- Issue 404: Intermittent charset detection failure
- Issue 415: Evaluation precedence wrong for arithmetic expressions
- Issue 61: multiple rows per column from 1 xml element
2.0 Release (November 10, 2010 - first release as Google Refine)
Major Changes:
- New extension architecture.
- Generalized reconciliation framework that allows plugging in standard reconciliation services.
- Support for QA on data loads into Freebase.
- Timeline Facet (Issue 40 and 95)
Features:
- New commands:
- Fill Down
- Blank Down
- Transpose Cells in Columns into Rows
- Transpose Cells in Rows into Columns (Issue 82)
- Move Column to Beginning, Move Column to End, Move Column Left, Move Column Right, Reorder Columns
- Add Column by Fetching URLs
- Recon commands:
- Clear recon data for all matching rows
- Clear recon data for one cell
- Clear recon data for similar cells
- Copy recon judgments across columns
- GREL:
- JSON support
- New functions: smartSplit, escape, parseJson, hasField, uniques
- New controls: forEachIndex, forRange, filter
- New parameters:
- preserveAllTokens on split function
- Regexp groups capturing GEL function
- Importers
- New: RDF exporter (as extension)
- New: Json importer
- CSV and TSV importers: added support for ignoring quotation marks
- Added support for creating a project by pointing to a data file URL.
- Text facet's choice count limit is now configurable through preference page
- Select All and Unselect All buttons in History Extract dialog
- Schema skeleton: support for multiple cells per cell-as nodes, and for conditional links
- Optionally convert strings to numbers during split columns operation
Fixes:
- TSV/CSV exporter bug: Gridworks crashed when there were empty cells.
- Issue 29: Delivered "Collapse whitespace" transformation does not work
- Issue 57: Schema skeleton disappears
- Issue 66: Records not excluded with inverted text facet
- Issue 69: ControlFunctionRegistry now correctly registers Chomp expression as "chomp" key.
- Issue 99: Diff for dates fails with "unknown error" always
- Issue 110: Import of single column text file with Postal Codes shows only 1 row with lots of <20> chars (?)
- Issue 113: Export filtered rows as tsv or csv fails; html and excel OK
- Issue 115: datePart(value,"month") gives zero-based month numbers
- Issue 116: CSV/TSV export data includes blank fields for deleted columns
- Issue 121: Importing attached file strips backslashes
- Issue 122: Exporting to Excel on attached project raises server exception
- Issue 125: jsonize not serializing arrays
- Issue 126: Large integers formatted in scientific notation in formulas
- Issue 127: Add column from Freebase raises exception
- Issue 135: Hangs when setting cell value to large JSON string
- Issue 138: Numbers should be right-justified
- Issue 140: Fix Open Workspace Directory command to work on Windows and Linux
- Issue 146: In "Cluster and Edit Column", clicking on entry value to set "Merge?" checkbox does not reflect the final value of operation
- Issue 155: Blank browser shown when non-GZIP format is detected during import
- Issue 160: Cancel button on Search for Match dialog sometimes not working
- Issue 161: Authorize error when trying to Sign In before loading to Freebase
- Issue 166: Expression Editor dialog needs UI cleanup on long expressions
1.1 Release (May 27, 2010)
Features:
- Row/record sorting (Issue 32)
- CSV exporter (Issue 59)
- Mqlwrite exporter
- Templating exporter (experimental)
Fixes:
- Issue 34: "Behavior of Text Filter is unpredictable when "regular expression" mode is enabled."
Regex was not compiled with case insensitivity flag.
- Issue 4: "Match All bug with ZIP code". Numeric values in cells were not stringified first
before comparison.
- Issue 41: "Envelope quotation marks are removed by CSV importer"
- Issue 19: "CSV import is too basic"
- Issue 15: "Ability to rename projects"
- Issue 16: "Column name collision when adding data from Freebase"
- Issue 28: "mql-like preview is not properly unquoting numbers"
- Issue 45: "Renaming Cells with Ctrl-Enter produced ERROR"
Tentative fix for a concurrent bug.
- Issue 46: "Array literals in GEL"
- Issue 55: "Use stable sorting for text facets sorted by count"
- Issue 53: "Moving the cursor inside the Text Filter box by clicking"
- Issue 58: "Meta facet"
Supported by the function facetCount()
- Issue 14: "Limiting Freebase load to starred records"
We load whatever rows that are filtered through, not particularly starred rows.
- Issue 49: "Add Edit Cells / Set Null"
- Issue 30: "Transform dialog should remember preferred language."
- Issue 62: "It'd be nice if URIs were hyperlinked in the data cells"
Other Changes:
- Moved unit tests from JUnit to TestNG
1.0.1 Release (May 12, 2010)
Fixes:
- Issue 2: "Undo History bug" - bulk row starring and flagging operations could not be undone.
- Issue 5: "Localized Windows cause save problems for Gridworks" -
Windows user IDs that contain unicode characters were not retrieved correctly.
- Issue 10: "OAuth fails on sign in" - due to clock offset.
- Issue 11: "missing "lang" attribute in MQL generated in schema alignment"
- Issue 13: "float rejected from sandbox upload as Json object" - everything was sent as a string.
- Issue 17: "Conflated triples - all rows are producing triple with "s" :" $Name_0"" -
The Create A New Topic for Each Cell command created shared recon objects.
- Issue 18: "Error converting russian characters during edit of single cell"
- [partial fix] Issue 19: "CSV import is too basic" - fixed for CSV, not for TSV
1.0 Release (May 10, 2010)
First Public Release as Freebase Gridworks

657
OpenRefine/IDEs/eclipse/Refine.style.xml
View File

@ -1,269 +1,390 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8" standalone="no"?>
<profiles version="11"> <profiles version="21">
<profile kind="CodeFormatterProfile" name="OpenRefine" version="11"> <profile kind="CodeFormatterProfile" name="OpenRefine" version="21">
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_for_statment" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="false"/> <setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/> <setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="1"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.align_with_spaces" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_member" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/> <setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_before_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_switch_case_expressions" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="8"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="1"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="1"/> <setting id="org.eclipse.jdt.core.formatter.enabling_tag" value="@formatter:on"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/> <setting id="org.eclipse.jdt.core.formatter.comment.count_line_length_from_starting_position" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_binary_operator" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_record_components" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.wrap_before_multiplicative_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="16"/> <setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="0"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="1"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="80"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="true"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_parameterized_type_references" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_logical_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_annotation_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_record_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_multiplicative_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/> <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_abstract_method" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_enum_constant_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.align_variable_declarations_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="16"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_catch_clause" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="120"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_multiplicative_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_anonymous_type_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_switch_case_expressions" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.wrap_before_shift_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="false"/> <setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_bitwise_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_type_parameters" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_loops" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="16"/> <setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_simple_for_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="false"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_annotation" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="16"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_enum_constant" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.text_block_indentation" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="true"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_module_statements" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="false"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_binary_operator" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.comment.align_tags_names_descriptions" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_if_then_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="16"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/> <setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.align_assignment_statements_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression_chain" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/> <setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="16"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_type_annotations" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.wrap_before_assertion_message_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_method_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/> <setting id="org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines" value="2147483647"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_bitwise_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="true"/> <setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_resources_in_try" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_binary_expression" value="16"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_binary_operator" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="16"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
<setting id="org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode" value="enabled"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_not_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_type_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_method_delcaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="false"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="16"/> <setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_type_arguments" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_package" value="49"/>
<setting id="org.eclipse.jdt.core.compiler.problem.assertIdentifier" value="error"/> <setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="16"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_label" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="1"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="false"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_record_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.compiler.source" value="1.5"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.wrap_before_bitwise_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="80"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_lambda_arrow" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.comment.indent_tag_description" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="true"/>
<setting id="org.eclipse.jdt.core.compiler.compliance" value="1.5"/> <setting id="org.eclipse.jdt.core.formatter.brace_position_for_record_constructor" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_string_concatenation" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_shift_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="16"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_shift_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_simple_do_while_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_record_components" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_for_loop_header" value="0"/>
<setting id="org.eclipse.jdt.core.compiler.problem.enumIdentifier" value="error"/> <setting id="org.eclipse.jdt.core.formatter.wrap_before_additive_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_simple_getter_setter_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_string_concatenation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_lambda_arrow" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_record_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_relational_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.compiler.codegen.targetPlatform" value="1.5"/> <setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_logical_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_method_invocation" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/> <setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_record_declaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="16"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_default" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.disabling_tag" value="@formatter:off"/>
<setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="true"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_method_body" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/> <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_if_while_statement" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="16"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/> <setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/> <setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="true"/> <setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="0"/> <setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="16"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="16"/> <setting id="org.eclipse.jdt.core.formatter.comment.align_tags_descriptions_grouped" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="space"/> <setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="120"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="1"/> <setting id="org.eclipse.jdt.core.formatter.use_on_off_tags" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_method_body_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_loop_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="true"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.keep_type_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.alignment_for_additive_operator" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_constructor" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_relational_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/> <setting id="org.eclipse.jdt.core.formatter.keep_record_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.brace_position_for_lambda_body" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
</profile> <setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_parameter" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_relational_operator" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_additive_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_string_concatenation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_record_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_after_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_type" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_local_variable" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_default" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_between_different_tags" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_additive_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_field" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_conditional_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_shift_operator" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_try_clause" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.keep_code_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_record_components" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_bitwise_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_record_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_assignment_operator" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_lambda_body_block_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_method" value="49"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_record_constructor_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_assertion_message" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_logical_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_record_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_relational_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_last_class_body_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.keep_simple_while_body_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_logical_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_statement_group_in_switch" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_lambda_declaration" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_enum_declaration_on_one_line" value="one_line_never"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_multiplicative_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_code_block" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="space"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_string_concatenation" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="140"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
</profile>
</profiles> </profiles>

16
OpenRefine/README.md
View File

@ -1,13 +1,3 @@
# MultiUserOpenRefine
![MultiUserOpenRefine](logo.svg)
MultiUserOpenRefine is an extension to open-source tool that, in addition to
all the features described below, allows creating user accounts, each providing
private workspace and leaving the convenience of running the basic OpenRefine
tool on a remote server.
Project is being developed at Adam Mickiewicz University in Poznań by students.
# OpenRefine # OpenRefine
[![DOI](https://zenodo.org/badge/6220644.svg)](https://zenodo.org/badge/latestdoi/6220644) [![DOI](https://zenodo.org/badge/6220644.svg)](https://zenodo.org/badge/latestdoi/6220644)
@ -27,8 +17,8 @@ the web. All from a web browser and the comfort and privacy of your own computer
Latest development version, packaged for: Latest development version, packaged for:
* [Linux](https://oss.sonatype.org/service/local/artifact/maven/content?r=snapshots&g=org.openrefine&a=openrefine&v=3.6-SNAPSHOT&c=linux&p=tar.gz) * [Linux](https://oss.sonatype.org/service/local/artifact/maven/content?r=snapshots&g=org.openrefine&a=openrefine&v=3.6-SNAPSHOT&c=linux&p=tar.gz)
* [Windows](https://oss.sonatype.org/service/local/artifact/maven/content?r=snapshots&g=org.openrefine&a=openrefine&v=3.6-SNAPSHOT&c=mac&p=dmg) * [MacOS](https://oss.sonatype.org/service/local/artifact/maven/content?r=snapshots&g=org.openrefine&a=openrefine&v=3.6-SNAPSHOT&c=mac&p=dmg)
* [MacOS](https://oss.sonatype.org/service/local/artifact/maven/content?r=snapshots&g=org.openrefine&a=openrefine&v=3.6-SNAPSHOT&c=win&p=zip) * [Windows](https://oss.sonatype.org/service/local/artifact/maven/content?r=snapshots&g=org.openrefine&a=openrefine&v=3.6-SNAPSHOT&c=win&p=zip)
## Run from source ## Run from source
@ -72,4 +62,4 @@ was acquired by Google, Inc. in July 2010 and the product was renamed Google Ref
In October 2012, it was renamed OpenRefine as it transitioned to a In October 2012, it was renamed OpenRefine as it transitioned to a
community-supported product. community-supported product.
See [AUTHORS.md](./AUTHORS.md) for the list of OpenRefine contributors and [CONTRIBUTING.md](./CONTRIBUTING.md) for instructions on how to contribute yourself. See [CONTRIBUTING.md](./CONTRIBUTING.md) for instructions on how to contribute yourself.

4
OpenRefine/SECURITY.md
View File

@ -4,8 +4,8 @@
| Version | Supported | | Version | Supported |
| ------- | ------------------ | | ------- | ------------------ |
| 3.4.x | :white_check_mark: | | 3.5.x | :white_check_mark: |
| <= 3.3 | :x: | | <= 3.4 | :x: |
## Reporting a Vulnerability ## Reporting a Vulnerability

8
OpenRefine/WeLoveOurBackers.md
View File

@ -1,8 +0,0 @@
An evergrowing list of our forever immortalized OpenRefine Backers that we LOVE because they help support further development !
1. Owen Stephens - Wiki contributor, extension author, and supporting you in our mailing list.
2. PaulZH -
3. Jens Willmer -
4. mroswell -
5. Google News Initiative - https://newsinitiative.withgoogle.com/
6. Chan Zuckerberg Initiative - https://chanzuckerberg.com/

8
OpenRefine/benchmark/pom.xml
View File

@ -48,7 +48,7 @@
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<jmh.version>1.33</jmh.version> <jmh.version>1.34</jmh.version>
<javac.target>${java.minversion}</javac.target> <javac.target>${java.minversion}</javac.target>
<uberjar.name>openrefine-benchmarks</uberjar.name> <uberjar.name>openrefine-benchmarks</uberjar.name>
</properties> </properties>
@ -58,7 +58,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId> <artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version> <version>3.9.0</version>
<configuration> <configuration>
<compilerVersion>${javac.target}</compilerVersion> <compilerVersion>${javac.target}</compilerVersion>
<source>${javac.target}</source> <source>${javac.target}</source>
@ -119,7 +119,7 @@
</plugin> </plugin>
<plugin> <plugin>
<artifactId>maven-jar-plugin</artifactId> <artifactId>maven-jar-plugin</artifactId>
<version>2.4</version> <version>${maven-jar-plugin.version}</version>
</plugin> </plugin>
<plugin> <plugin>
<artifactId>maven-javadoc-plugin</artifactId> <artifactId>maven-javadoc-plugin</artifactId>
@ -131,7 +131,7 @@
</plugin> </plugin>
<plugin> <plugin>
<artifactId>maven-site-plugin</artifactId> <artifactId>maven-site-plugin</artifactId>
<version>3.9.1</version> <version>3.10.0</version>
</plugin> </plugin>
<plugin> <plugin>
<artifactId>maven-source-plugin</artifactId> <artifactId>maven-source-plugin</artifactId>

24
OpenRefine/benchmark/src/main/java/org/openrefine/benchmark/ToNumberBenchmark.java
View File

@ -24,6 +24,7 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/ ******************************************************************************/
package org.openrefine.benchmark; package org.openrefine.benchmark;
import java.util.Properties; import java.util.Properties;
@ -53,15 +54,15 @@ public class ToNumberBenchmark {
@State(Scope.Benchmark) @State(Scope.Benchmark)
public static class ExecutionPlan { public static class ExecutionPlan {
@Param({"1000", "10000" }) @Param({ "1000", "10000" })
public int iterations; public int iterations;
public ToNumber f; public ToNumber f;
String[] args = new String[1]; String[] args = new String[1];
String testData; String testData;
String testDataInt; String testDataInt;
Random rnd = new Random(); Random rnd = new Random();
@Setup(Level.Invocation) @Setup(Level.Invocation)
public void setUp() { public void setUp() {
f = new ToNumber(); f = new ToNumber();
@ -73,8 +74,8 @@ public class ToNumberBenchmark {
@Benchmark @Benchmark
@BenchmarkMode(Mode.AverageTime) @BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS) @OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 3, time = 200, timeUnit = TimeUnit.MILLISECONDS) @Warmup(iterations = 3, time = 200, timeUnit = TimeUnit.MILLISECONDS)
@Measurement(iterations = 5, time = 200, timeUnit = TimeUnit.MILLISECONDS) @Measurement(iterations = 5, time = 200, timeUnit = TimeUnit.MILLISECONDS)
public void toDoubleNew(ExecutionPlan plan, Blackhole blackhole) { public void toDoubleNew(ExecutionPlan plan, Blackhole blackhole) {
plan.args[0] = plan.testData; plan.args[0] = plan.testData;
blackhole.consume(plan.f.call(bindings, plan.args)); blackhole.consume(plan.f.call(bindings, plan.args));
@ -83,12 +84,11 @@ public class ToNumberBenchmark {
@Benchmark @Benchmark
@BenchmarkMode(Mode.AverageTime) @BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS) @OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 3, time = 200, timeUnit = TimeUnit.MILLISECONDS) @Warmup(iterations = 3, time = 200, timeUnit = TimeUnit.MILLISECONDS)
@Measurement(iterations = 5, time = 200, timeUnit = TimeUnit.MILLISECONDS) @Measurement(iterations = 5, time = 200, timeUnit = TimeUnit.MILLISECONDS)
@Fork(1) @Fork(1)
public void toLongNew(ExecutionPlan plan, Blackhole blackhole) { public void toLongNew(ExecutionPlan plan, Blackhole blackhole) {
plan.args[0] = plan.testDataInt; plan.args[0] = plan.testDataInt;
blackhole.consume(plan.f.call(bindings, plan.args)); blackhole.consume(plan.f.call(bindings, plan.args));
} }
} }

14
OpenRefine/docs/README.md
View File

@ -5,7 +5,7 @@ We use [Docusaurus 2](https://v2.docusaurus.io/) for our docs, a modern static w
### Requirements ### Requirements
Assuming you have [Node.js](https://nodejs.org/en/download/) installed (which includes npm), you can install Docusaurus with: Assuming you have [Node.js (LTS recommended)](https://nodejs.org/en/download/) installed (which includes npm), you can install Docusaurus with:
You will need to install [Yarn](https://yarnpkg.com/getting-started/install) before you can build the site. You will need to install [Yarn](https://yarnpkg.com/getting-started/install) before you can build the site.
@ -66,6 +66,18 @@ yarn build
``` ```
This command generates static content into the `build` directory and can be served using any static contents hosting service. This command generates static content into the `build` directory and can be served using any static contents hosting service.
### (Optional) Test Build Locally
You can locally test ([with parameters](https://docusaurus.io/docs/cli#docusaurus-serve-sitedir)) the static content in the `build` directory (in case you don't have access to a hosting service) by using:
```sh
yarn serve
```
or to build and then serve locally with one command:
```sh
yarn serve --build
```
### Deployment ### Deployment

2
OpenRefine/docs/docs/manual/installing.md
View File

@ -20,7 +20,7 @@ OpenRefine is designed to work with **Windows**, **Mac**, and **Linux** operatin
If you install and start OpenRefine on a Windows computer without Java, it will automatically open up a browser window to the [Java downloads page](https://java.com/en/download/), and you can simply follow the instructions there. If you install and start OpenRefine on a Windows computer without Java, it will automatically open up a browser window to the [Java downloads page](https://java.com/en/download/), and you can simply follow the instructions there.
We recommend you [download](https://java.com/en/download/) and install Java before proceeding with the OpenRefine installation. Please note that OpenRefine works with Java 8 to Java 15 but not Java 16 or later versions. We recommend you [download](https://java.com/en/download/) and install Java before proceeding with the OpenRefine installation. Please note that OpenRefine 3.5 works with Java 8 to Java 17.
#### Compatible browser {#compatible-browser} #### Compatible browser {#compatible-browser}

2
OpenRefine/docs/docs/manual/running.md
View File

@ -321,7 +321,7 @@ From the home screen, look in the options to the left for <span class="menuItems
To leave the Language Settings screen, click on the diamond “OpenRefine” logo. To leave the Language Settings screen, click on the diamond “OpenRefine” logo.
:::info Help us Translate OpenRefine :::info Help us Translate OpenRefine
We use Weblate to provide translations for the interface. You can check [our profile on Weblate](https://hosted.weblate.org/projects/openrefine/translations/) to see which languages are in the process of being supported. See [our technical reference if you are interested in contributing translation work](https://docs.openrefine.org/technical-reference/translating) to make OpenRefine accessible to people in other languages. We use Weblate to provide translations for the interface. You can check [our profile on Weblate](https://hosted.weblate.org/projects/openrefine/translations/) to see which languages are in the process of being supported. See [our technical reference if you are interested in contributing translation work](../technical-reference/translating-ui) to make OpenRefine accessible to people in other languages.
::: :::
### Preferences {#preferences} ### Preferences {#preferences}

6
OpenRefine/docs/docs/manual/sortview.md
View File

@ -33,3 +33,9 @@ You can control what data you view in the grid. On each column, you will see a <
You can find, under <span class="menuItems">All</span> → <span class="menuItems">View</span>, the option to show and hide [“null” values](exploring#data-types). A small grey “null” will appear in each applicable cell. Remember that a null cell is not the same thing as an empty cell. You can find, under <span class="menuItems">All</span> → <span class="menuItems">View</span>, the option to show and hide [“null” values](exploring#data-types). A small grey “null” will appear in each applicable cell. Remember that a null cell is not the same thing as an empty cell.
![A screenshot of what a null value looks like.](/img/null.png) ![A screenshot of what a null value looks like.](/img/null.png)
## Page navigation {#page-navigation}
You can go directly to any page by changing the page number on the right-hand side. Using the up and down arrow keys in this input lets you go to the next and previous pages. You can also change the number of rows or records per page on the left-hand side of this view header bar.
![A screenshot of the Page Navigation Feature.](/img/goto.png)

4
OpenRefine/docs/docs/manual/troubleshooting.md
View File

@ -22,8 +22,8 @@ If you don’t find your problem and solution there, continue on to the resource
* Then, if you wish, you can create a Github issue. * Then, if you wish, you can create a Github issue.
### If you want to contribute: {#if-you-want-to-contribute} ### If you want to contribute: {#if-you-want-to-contribute}
* [Help us translate the tool into more languages](https://docs.openrefine.org/technical-reference/translating), using Weblate * [Help us translate the tool into more languages](../technical-reference/translating-ui), using Weblate
* [We have a guide to contributing](technical-reference/contributing) in the [Technical Reference](technical-reference/technical-reference-index) section * [We have a guide to contributing](../technical-reference/contributing) in the [Technical Reference](../technical-reference/technical-reference-index) section
* Contribute your feature requests in the [User forum](https://groups.google.com/g/openrefine) or as [Github issues](https://github.com/OpenRefine/OpenRefine/issues/new/choose) * Contribute your feature requests in the [User forum](https://groups.google.com/g/openrefine) or as [Github issues](https://github.com/OpenRefine/OpenRefine/issues/new/choose)
* Join the User Forum and/or the [Developer Forum](https://groups.google.com/g/openrefine-dev) * Join the User Forum and/or the [Developer Forum](https://groups.google.com/g/openrefine-dev)
* Share your successes and use cases with us, in the User forum * Share your successes and use cases with us, in the User forum

7
OpenRefine/docs/docs/technical-reference/build-test-run.md
View File

@ -230,6 +230,13 @@ Right click on the `server` subproject, click `Run as...` and `Run configuration
This will add a run configuration that you can then use to run OpenRefine from Eclipse. This will add a run configuration that you can then use to run OpenRefine from Eclipse.
## Code style in Eclipse
You can apply the supplied Eclipse code style (in `IDEs/eclipse/Refine.style.xml`) to make sure Eclipse lints your files according to the existing style.
Pull requests deviating from this style will fail in the CI.
You can manually apply the code style (regardless of your IDE) with the `mvn formatter:format` command.
## Testing in Eclipse {#testing-in-eclipse} ## Testing in Eclipse {#testing-in-eclipse}
You can run the server tests directly from Eclipse. To do that you need to have the TestNG launcher plugin installed, as well as the TestNG M2E plugin (for integration with Maven). If you don't have it, you can get it by [installing new software](https://help.eclipse.org/2020-03/index.jsp?topic=/org.eclipse.platform.doc.user/tasks/tasks-129.htm) from this update URL http://dl.bintray.com/testng-team/testng-eclipse-release/ You can run the server tests directly from Eclipse. To do that you need to have the TestNG launcher plugin installed, as well as the TestNG M2E plugin (for integration with Maven). If you don't have it, you can get it by [installing new software](https://help.eclipse.org/2020-03/index.jsp?topic=/org.eclipse.platform.doc.user/tasks/tasks-129.htm) from this update URL http://dl.bintray.com/testng-team/testng-eclipse-release/

2
OpenRefine/docs/docs/technical-reference/contributing.md
View File

@ -61,6 +61,8 @@ This describes the overall steps to your first code contribution in OpenRefine.
- Make changes to the code to fix the issue. If you are changing backend code, it would be great if you could also write a test in Java to demonstrate the fix. You can imitate existing tests for that. We currently do not have frontend tests. - Make changes to the code to fix the issue. If you are changing backend code, it would be great if you could also write a test in Java to demonstrate the fix. You can imitate existing tests for that. We currently do not have frontend tests.
- If you made Java changes, run linting to make sure they conform to our code style, with `mvn formatter:format`.
- commit your changes, using a message that contains one of the special words "closes" and "fixes" which are detected by Github, followed by the issue number, e.g. "closes #1234" or "fixes #1234", this will link the commit to the issue you are working on. - commit your changes, using a message that contains one of the special words "closes" and "fixes" which are detected by Github, followed by the issue number, e.g. "closes #1234" or "fixes #1234", this will link the commit to the issue you are working on.
- push your branch to your fork and create a pull request for it, explaining the approach you have used, any design decisions you have made. - push your branch to your fork and create a pull request for it, explaining the approach you have used, any design decisions you have made.

2
OpenRefine/docs/docs/technical-reference/version-release-process.md
View File

@ -10,7 +10,7 @@ When releasing a new version of Refine, the following steps should be followed:
2. Change the version number in [RefineServlet.java](http://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/RefineServlet.java#L62) and in the POM files using `mvn versions:set -DnewVersion=2.6-beta -DgenerateBackupPoms=false`. Commit the changes. 2. Change the version number in [RefineServlet.java](http://github.com/OpenRefine/OpenRefine/blob/master/main/src/com/google/refine/RefineServlet.java#L62) and in the POM files using `mvn versions:set -DnewVersion=2.6-beta -DgenerateBackupPoms=false`. Commit the changes.
3. Compose the list of changes in the code and on the wiki. If the issues have been updated with the appropriate milestone, the Github issue tracker should be able to provide a good starting point for this. 3. Compose the list of changes in the code and on the wiki. If the issues have been updated with the appropriate milestone, the Github issue tracker should be able to provide a good starting point for this.
4. Set up build machine. This needs to be Mac OS X or Linux. 4. Set up build machine. This needs to be Mac OS X or Linux.
5. Download Windows and Mac JREs to bundle them in the Windows and Mac packages from [AdoptOpenJDK](https://adoptopenjdk.net/). You only need the JREs, not the JDKs. Use the lowest version of Java supported (Java 8 currently). Configure the location of these JREs in the `settings.xml` file at the root of the repository. It is important to download recent versions of the JREs as this impacts which HTTPS certificates are accepted by the tool. 5. Download Windows and Mac JREs to bundle them in the Windows and Mac packages from [AdoptOpenJDK](https://adoptopenjdk.net/). You only need the JREs, not the JDKs. Use the lowest version of Java supported (Java 11 currently). Configure the location of these JREs in the `settings.xml` file at the root of the repository. It is important to download recent versions of the JREs as this impacts which HTTPS certificates are accepted by the tool.
6. Insert the production Google credentials in https://github.com/OpenRefine/OpenRefine/blob/bc540a880eceb88e54f85ca43eb54769de3bfa4f/extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java#L36-L39 without committing the changes. 6. Insert the production Google credentials in https://github.com/OpenRefine/OpenRefine/blob/bc540a880eceb88e54f85ca43eb54769de3bfa4f/extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java#L36-L39 without committing the changes.
7. [Build the release candidate kits using the shell script (not just Maven)](https://github.com/OpenRefine/OpenRefine/wiki/Building-OpenRefine-From-Source). This must be done on Mac OS X or Linux to be able to build all 3 kits. On Linux you will need to install the `genisoimage` program first. 7. [Build the release candidate kits using the shell script (not just Maven)](https://github.com/OpenRefine/OpenRefine/wiki/Building-OpenRefine-From-Source). This must be done on Mac OS X or Linux to be able to build all 3 kits. On Linux you will need to install the `genisoimage` program first.
```shell ```shell

BIN
OpenRefine/docs/static/img/goto.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 18 KiB

2
OpenRefine/docs/versioned_docs/version-3.4/manual/running.md
View File

@ -319,7 +319,7 @@ From the home screen, look in the options to the left for <span class="menuItems
To leave the Language Settings screen, click on the diamond “OpenRefine” logo. To leave the Language Settings screen, click on the diamond “OpenRefine” logo.
:::info Help us Translate OpenRefine :::info Help us Translate OpenRefine
We use Weblate to provide translations for the interface. You can check [our profile on Weblate](https://hosted.weblate.org/projects/openrefine/translations/) to see which languages are in the process of being supported. See [our technical reference if you are interested in contributing translation work](https://docs.openrefine.org/technical-reference/translating) to make OpenRefine accessible to people in other languages. We use Weblate to provide translations for the interface. You can check [our profile on Weblate](https://hosted.weblate.org/projects/openrefine/translations/) to see which languages are in the process of being supported. See [our technical reference if you are interested in contributing translation work](../technical-reference/translating) to make OpenRefine accessible to people in other languages.
::: :::
### Preferences {#preferences} ### Preferences {#preferences}

4
OpenRefine/docs/versioned_docs/version-3.4/manual/troubleshooting.md
View File

@ -22,8 +22,8 @@ If you don’t find your problem and solution there, continue on to the resource
* Then, if you wish, you can create a Github issue. * Then, if you wish, you can create a Github issue.
### If you want to contribute: {#if-you-want-to-contribute} ### If you want to contribute: {#if-you-want-to-contribute}
* [Help us translate the tool into more languages](https://docs.openrefine.org/technical-reference/translating), using Weblate * [Help us translate the tool into more languages](../technical-reference/translating), using Weblate
* [We have a guide to contributing](technical-reference/contributing) in the [Technical Reference](technical-reference/technical-reference-index) section * [We have a guide to contributing](../technical-reference/contributing) in the [Technical Reference](../technical-reference/technical-reference-index) section
* Contribute your feature requests in the [User forum](https://groups.google.com/g/openrefine) or as [Github issues](https://github.com/OpenRefine/OpenRefine/issues/new/choose) * Contribute your feature requests in the [User forum](https://groups.google.com/g/openrefine) or as [Github issues](https://github.com/OpenRefine/OpenRefine/issues/new/choose)
* Join the User Forum and/or the [Developer Forum](https://groups.google.com/g/openrefine-dev) * Join the User Forum and/or the [Developer Forum](https://groups.google.com/g/openrefine-dev)
* Share your successes and use cases with us, in the User forum * Share your successes and use cases with us, in the User forum

2
OpenRefine/docs/versioned_docs/version-3.5/manual/installing.md
View File

@ -20,7 +20,7 @@ OpenRefine is designed to work with **Windows**, **Mac**, and **Linux** operatin
If you install and start OpenRefine on a Windows computer without Java, it will automatically open up a browser window to the [Java downloads page](https://java.com/en/download/), and you can simply follow the instructions there. If you install and start OpenRefine on a Windows computer without Java, it will automatically open up a browser window to the [Java downloads page](https://java.com/en/download/), and you can simply follow the instructions there.
We recommend you [download](https://java.com/en/download/) and install Java before proceeding with the OpenRefine installation. Please note that OpenRefine works with Java 8 to Java 15 but not Java 16 or later versions. We recommend you [download](https://java.com/en/download/) and install Java before proceeding with the OpenRefine installation. Please note that OpenRefine works with Java 8 to Java 17 for OpenRefine 3.5.
#### Compatible browser {#compatible-browser} #### Compatible browser {#compatible-browser}

2
OpenRefine/docs/versioned_docs/version-3.5/manual/running.md
View File

@ -321,7 +321,7 @@ From the home screen, look in the options to the left for <span class="menuItems
To leave the Language Settings screen, click on the diamond “OpenRefine” logo. To leave the Language Settings screen, click on the diamond “OpenRefine” logo.
:::info Help us Translate OpenRefine :::info Help us Translate OpenRefine
We use Weblate to provide translations for the interface. You can check [our profile on Weblate](https://hosted.weblate.org/projects/openrefine/translations/) to see which languages are in the process of being supported. See [our technical reference if you are interested in contributing translation work](https://docs.openrefine.org/technical-reference/translating) to make OpenRefine accessible to people in other languages. We use Weblate to provide translations for the interface. You can check [our profile on Weblate](https://hosted.weblate.org/projects/openrefine/translations/) to see which languages are in the process of being supported. See [our technical reference if you are interested in contributing translation work](../technical-reference/translating-ui) to make OpenRefine accessible to people in other languages.
::: :::
### Preferences {#preferences} ### Preferences {#preferences}

6
OpenRefine/docs/versioned_docs/version-3.5/manual/sortview.md
View File

@ -33,3 +33,9 @@ You can control what data you view in the grid. On each column, you will see a <
You can find, under <span class="menuItems">All</span> → <span class="menuItems">View</span>, the option to show and hide [“null” values](exploring#data-types). A small grey “null” will appear in each applicable cell. Remember that a null cell is not the same thing as an empty cell. You can find, under <span class="menuItems">All</span> → <span class="menuItems">View</span>, the option to show and hide [“null” values](exploring#data-types). A small grey “null” will appear in each applicable cell. Remember that a null cell is not the same thing as an empty cell.
![A screenshot of what a null value looks like.](/img/null.png) ![A screenshot of what a null value looks like.](/img/null.png)
## Page navigation {#page-navigation}
You can go directly to any page by changing the page number on the right-hand side. Using the up and down arrow keys in this input lets you go to the next and previous pages. You can also change the number of rows or records per page on the left-hand side of this view header bar.
![A screenshot of the Page Navigation Feature.](/img/goto.png)

4
OpenRefine/docs/versioned_docs/version-3.5/manual/troubleshooting.md
View File

@ -22,8 +22,8 @@ If you don’t find your problem and solution there, continue on to the resource
* Then, if you wish, you can create a Github issue. * Then, if you wish, you can create a Github issue.
### If you want to contribute: {#if-you-want-to-contribute} ### If you want to contribute: {#if-you-want-to-contribute}
* [Help us translate the tool into more languages](https://docs.openrefine.org/technical-reference/translating), using Weblate * [Help us translate the tool into more languages](../technical-reference/translating-ui), using Weblate
* [We have a guide to contributing](technical-reference/contributing) in the [Technical Reference](technical-reference/technical-reference-index) section * [We have a guide to contributing](../technical-reference/contributing) in the [Technical Reference](../technical-reference/technical-reference-index) section
* Contribute your feature requests in the [User forum](https://groups.google.com/g/openrefine) or as [Github issues](https://github.com/OpenRefine/OpenRefine/issues/new/choose) * Contribute your feature requests in the [User forum](https://groups.google.com/g/openrefine) or as [Github issues](https://github.com/OpenRefine/OpenRefine/issues/new/choose)
* Join the User Forum and/or the [Developer Forum](https://groups.google.com/g/openrefine-dev) * Join the User Forum and/or the [Developer Forum](https://groups.google.com/g/openrefine-dev)
* Share your successes and use cases with us, in the User forum * Share your successes and use cases with us, in the User forum

7
OpenRefine/docs/versioned_docs/version-3.5/technical-reference/build-test-run.md
View File

@ -230,6 +230,13 @@ Right click on the `server` subproject, click `Run as...` and `Run configuration
This will add a run configuration that you can then use to run OpenRefine from Eclipse. This will add a run configuration that you can then use to run OpenRefine from Eclipse.
## Code style in Eclipse
You can apply the supplied Eclipse code style (in `IDEs/eclipse/Refine.style.xml`) to make sure Eclipse lints your files according to the existing style.
Pull requests deviating from this style will fail in the CI.
You can manually apply the code style (regardless of your IDE) with the `mvn formatter:format` command.
## Testing in Eclipse {#testing-in-eclipse} ## Testing in Eclipse {#testing-in-eclipse}
You can run the server tests directly from Eclipse. To do that you need to have the TestNG launcher plugin installed, as well as the TestNG M2E plugin (for integration with Maven). If you don't have it, you can get it by [installing new software](https://help.eclipse.org/2020-03/index.jsp?topic=/org.eclipse.platform.doc.user/tasks/tasks-129.htm) from this update URL http://dl.bintray.com/testng-team/testng-eclipse-release/ You can run the server tests directly from Eclipse. To do that you need to have the TestNG launcher plugin installed, as well as the TestNG M2E plugin (for integration with Maven). If you don't have it, you can get it by [installing new software](https://help.eclipse.org/2020-03/index.jsp?topic=/org.eclipse.platform.doc.user/tasks/tasks-129.htm) from this update URL http://dl.bintray.com/testng-team/testng-eclipse-release/

2
OpenRefine/docs/versioned_docs/version-3.5/technical-reference/contributing.md
View File

@ -61,6 +61,8 @@ This describes the overall steps to your first code contribution in OpenRefine.
- Make changes to the code to fix the issue. If you are changing backend code, it would be great if you could also write a test in Java to demonstrate the fix. You can imitate existing tests for that. We currently do not have frontend tests. - Make changes to the code to fix the issue. If you are changing backend code, it would be great if you could also write a test in Java to demonstrate the fix. You can imitate existing tests for that. We currently do not have frontend tests.
- If you made Java changes, run linting to make sure they conform to our code style, with `mvn formatter:format`.
- commit your changes, using a message that contains one of the special words "closes" and "fixes" which are detected by Github, followed by the issue number, e.g. "closes #1234" or "fixes #1234", this will link the commit to the issue you are working on. - commit your changes, using a message that contains one of the special words "closes" and "fixes" which are detected by Github, followed by the issue number, e.g. "closes #1234" or "fixes #1234", this will link the commit to the issue you are working on.
- push your branch to your fork and create a pull request for it, explaining the approach you have used, any design decisions you have made. - push your branch to your fork and create a pull request for it, explaining the approach you have used, any design decisions you have made.

81
OpenRefine/docs/yarn.lock
View File

@ -1432,9 +1432,9 @@
to-fast-properties "^2.0.0" to-fast-properties "^2.0.0"
"@crowdin/cli@3": "@crowdin/cli@3":
version "3.7.2" version "3.7.7"
resolved "https://registry.yarnpkg.com/@crowdin/cli/-/cli-3.7.2.tgz#709597ead041359b213e0804d4c8b9f8f8a14dcd" resolved "https://registry.yarnpkg.com/@crowdin/cli/-/cli-3.7.7.tgz#0d2baf888b37aac1cd8d8cb484b0826fa7b82a26"
integrity sha512-dM7pQLHo10XnHvi0RAkOFAwqn/LB4NE42njn4dHSu8O30zLgsVbCEq4KvUajE2aFz0mDeAzMgxjEDJSssFUf2Q== integrity sha512-wN0NXLDymC9PgT3I6jMx/YLjcEtegAnFg7VaavzLfc1vFegD4lKlrvRmc0qV2IIiRMZLShEsS0QtYtAyhi/qew==
dependencies: dependencies:
shelljs "^0.8.4" shelljs "^0.8.4"
@ -2945,20 +2945,10 @@ caniuse-api@^3.0.0:
lodash.memoize "^4.1.2" lodash.memoize "^4.1.2"
lodash.uniq "^4.5.0" lodash.uniq "^4.5.0"
caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001214: caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001125, caniuse-lite@^1.0.30001214, caniuse-lite@^1.0.30001219, caniuse-lite@^1.0.30001272, caniuse-lite@^1.0.30001280:
version "1.0.30001219" version "1.0.30001303"
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001219.tgz#5bfa5d0519f41f993618bd318f606a4c4c16156b" resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001303.tgz"
integrity sha512-c0yixVG4v9KBc/tQ2rlbB3A/bgBFRvl8h8M4IeUbqCca4gsiCfvtaheUssbnux/Mb66Vjz7x8yYjDgYcNQOhyQ== integrity sha512-/Mqc1oESndUNszJP0kx0UaQU9kEv9nNtJ7Kn8AdA0mNnH8eR1cj0kG+NbNuC1Wq/b21eA8prhKRA3bbkjONegQ==
caniuse-lite@^1.0.30001125, caniuse-lite@^1.0.30001272, caniuse-lite@^1.0.30001280:
version "1.0.30001286"
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001286.tgz#3e9debad420419618cfdf52dc9b6572b28a8fff6"
integrity sha512-zaEMRH6xg8ESMi2eQ3R4eZ5qw/hJiVsO/HlLwniIwErij0JDr9P+8V4dtx1l+kLq6j3yy8l8W4fst1lBnat5wQ==
caniuse-lite@^1.0.30001219:
version "1.0.30001220"
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001220.tgz#c080e1c8eefb99f6cc9685da6313840bdbaf4c36"
integrity sha512-pjC2T4DIDyGAKTL4dMvGUQaMUHRmhvPpAgNNTa14jaBWHu+bLQgvpFqElxh9L4829Fdx0PlKiMp3wnYldRtECA==
ccount@^1.0.0, ccount@^1.0.3: ccount@^1.0.0, ccount@^1.0.3:
version "1.1.0" version "1.1.0"
@ -4499,9 +4489,9 @@ flux@^4.0.1:
fbjs "^3.0.0" fbjs "^3.0.0"
follow-redirects@^1.0.0, follow-redirects@^1.14.0: follow-redirects@^1.0.0, follow-redirects@^1.14.0:
version "1.14.3" version "1.14.7"
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.3.tgz#6ada78118d8d24caee595595accdc0ac6abd022e" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.7.tgz#2004c02eb9436eee9a21446a6477debf17e81685"
integrity sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw== integrity sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==
for-in@^1.0.2: for-in@^1.0.2:
version "1.0.2" version "1.0.2"
@ -4655,19 +4645,7 @@ glob-to-regexp@^0.4.1:
resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e"
integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==
glob@^7.0.0, glob@^7.1.3: glob@^7.0.0, glob@^7.0.3, glob@^7.1.3:
version "7.1.6"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6"
integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==
dependencies:
fs.realpath "^1.0.0"
inflight "^1.0.4"
inherits "2"
minimatch "^3.0.4"
once "^1.3.0"
path-is-absolute "^1.0.0"
glob@^7.0.3:
version "7.2.0" version "7.2.0"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023"
integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==
@ -5348,6 +5326,13 @@ is-core-module@^2.2.0:
dependencies: dependencies:
has "^1.0.3" has "^1.0.3"
is-core-module@^2.8.0:
version "2.8.1"
resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.8.1.tgz#f59fdfca701d5879d0a6b100a40aa1560ce27211"
integrity sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA==
dependencies:
has "^1.0.3"
is-data-descriptor@^0.1.4: is-data-descriptor@^0.1.4:
version "0.1.4" version "0.1.4"
resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56" resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56"
@ -6284,9 +6269,9 @@ nan@^2.12.1:
integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ== integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==
nanoid@^3.1.28, nanoid@^3.1.30: nanoid@^3.1.28, nanoid@^3.1.30:
version "3.1.30" version "3.2.0"
resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.30.tgz#63f93cc548d2a113dc5dfbc63bfa09e2b9b64362" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.2.0.tgz#62667522da6673971cca916a6d3eff3f415ff80c"
integrity sha512-zJpuPDwOv8D2zq2WRoMe1HsfZthVewpel9CAvTfc/2mBD1uUT/agc5f7GHGWXlYkFvi1mVxe4IjvP2HNrop7nQ== integrity sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA==
nanomatch@^1.2.9: nanomatch@^1.2.9:
version "1.2.13" version "1.2.13"
@ -6737,7 +6722,7 @@ path-key@^3.0.0, path-key@^3.1.0:
resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375"
integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==
path-parse@^1.0.6: path-parse@^1.0.6, path-parse@^1.0.7:
version "1.0.7" version "1.0.7"
resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735"
integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==
@ -7885,7 +7870,16 @@ resolve-url@^0.2.1:
resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a"
integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo= integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=
resolve@^1.1.6, resolve@^1.14.2, resolve@^1.3.2: resolve@^1.1.6:
version "1.21.0"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.21.0.tgz#b51adc97f3472e6a5cf4444d34bc9d6b9037591f"
integrity sha512-3wCbTpk5WJlyE4mSOtDLhqQmGFi0/TD9VPwmiolnk8U0wRgMEktqCXd3vy5buTO3tljvalNvKrjHEfrd2WpEKA==
dependencies:
is-core-module "^2.8.0"
path-parse "^1.0.7"
supports-preserve-symlinks-flag "^1.0.0"
resolve@^1.14.2, resolve@^1.3.2:
version "1.20.0" version "1.20.0"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975"
integrity sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A== integrity sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==
@ -8218,9 +8212,9 @@ shell-quote@1.7.2:
integrity sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg== integrity sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg==
shelljs@^0.8.4: shelljs@^0.8.4:
version "0.8.4" version "0.8.5"
resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.4.tgz#de7684feeb767f8716b326078a8a00875890e3c2" resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.5.tgz#de055408d8361bed66c669d2f000538ced8ee20c"
integrity sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ== integrity sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==
dependencies: dependencies:
glob "^7.0.0" glob "^7.0.0"
interpret "^1.0.0" interpret "^1.0.0"
@ -8606,6 +8600,11 @@ supports-color@^8.0.0:
dependencies: dependencies:
has-flag "^4.0.0" has-flag "^4.0.0"
supports-preserve-symlinks-flag@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09"
integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==
svg-parser@^2.0.2: svg-parser@^2.0.2:
version "2.0.4" version "2.0.4"
resolved "https://registry.yarnpkg.com/svg-parser/-/svg-parser-2.0.4.tgz#fdc2e29e13951736140b76cb122c8ee6630eb6b5" resolved "https://registry.yarnpkg.com/svg-parser/-/svg-parser-2.0.4.tgz#fdc2e29e13951736140b76cb122c8ee6630eb6b5"

6
OpenRefine/extensions/database/pom.xml
View File

@ -84,12 +84,12 @@
<dependency> <dependency>
<groupId>com.google.http-client</groupId> <groupId>com.google.http-client</groupId>
<artifactId>google-http-client-jackson2</artifactId> <artifactId>google-http-client-jackson2</artifactId>
<version>1.40.1</version> <version>1.41.2</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>mysql</groupId> <groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId> <artifactId>mysql-connector-java</artifactId>
<version>8.0.27</version> <version>8.0.28</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.jasypt</groupId> <groupId>org.jasypt</groupId>
@ -99,7 +99,7 @@
<dependency> <dependency>
<groupId>org.mariadb.jdbc</groupId> <groupId>org.mariadb.jdbc</groupId>
<artifactId>mariadb-java-client</artifactId> <artifactId>mariadb-java-client</artifactId>
<version>2.7.4</version> <version>2.7.5</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.xerial</groupId> <groupId>org.xerial</groupId>

366
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTestUtils.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database; package com.google.refine.extension.database;
import java.io.File; import java.io.File;
@ -18,92 +19,92 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class DBExtensionTestUtils { public class DBExtensionTestUtils {
private static final Logger logger = LoggerFactory.getLogger("DBExtensionTestUtils"); private static final Logger logger = LoggerFactory.getLogger("DBExtensionTestUtils");
private static final String MYSQL_DB_NAME = "mysql"; private static final String MYSQL_DB_NAME = "mysql";
private static final String DEFAULT_MYSQL_HOST = "127.0.0.1"; private static final String DEFAULT_MYSQL_HOST = "127.0.0.1";
private static final int DEFAULT_MYSQL_PORT = 3306; private static final int DEFAULT_MYSQL_PORT = 3306;
private static final String DEFAULT_MYSQL_USER = "root"; private static final String DEFAULT_MYSQL_USER = "root";
private static final String DEFAULT_MYSQL_PASSWORD = "secret"; private static final String DEFAULT_MYSQL_PASSWORD = "secret";
private static final String DEFAULT_MYSQL_DB_NAME = "testdb"; private static final String DEFAULT_MYSQL_DB_NAME = "testdb";
private static final String PGSQL_DB_NAME = "postgresql"; private static final String PGSQL_DB_NAME = "postgresql";
private static final String DEFAULT_PGSQL_HOST = "127.0.0.1"; private static final String DEFAULT_PGSQL_HOST = "127.0.0.1";
private static final int DEFAULT_PGSQL_PORT = 5432; private static final int DEFAULT_PGSQL_PORT = 5432;
private static final String DEFAULT_PGSQL_USER = "postgres"; private static final String DEFAULT_PGSQL_USER = "postgres";
private static final String DEFAULT_PGSQL_PASSWORD = ""; private static final String DEFAULT_PGSQL_PASSWORD = "";
private static final String DEFAULT_PGSQL_DB_NAME = "openrefine"; private static final String DEFAULT_PGSQL_DB_NAME = "openrefine";
private static final String DEFAULT_TEST_TABLE = "test_data"; private static final String DEFAULT_TEST_TABLE = "test_data";
private static final int SAMPLE_SIZE = 500000; private static final int SAMPLE_SIZE = 500000;
private static final int BATCH_SIZE = 1000; private static final int BATCH_SIZE = 1000;
private static Random rand = new Random(); private static Random rand = new Random();
private Map<Integer, Integer> mncMap; private Map<Integer, Integer> mncMap;
private Map<Integer, Integer> mccMap; private Map<Integer, Integer> mccMap;
/** /**
* Create Test Table with one row of Data * Create Test Table with one row of Data
* @param dbConfig DatabaseConfiguration to test *
* @param dbConfig
* DatabaseConfiguration to test
* @param tableName * @param tableName
* @throws DatabaseServiceException * @throws DatabaseServiceException
* @throws SQLException * @throws SQLException
*/ */
public static void initTestData(DatabaseConfiguration dbConfig, String tableName) public static void initTestData(DatabaseConfiguration dbConfig, String tableName)
throws DatabaseServiceException, SQLException { throws DatabaseServiceException, SQLException {
Statement stmt = null; Statement stmt = null;
Connection conn = null; Connection conn = null;
try { try {
DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType()); DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType());
conn = dbService.getConnection(dbConfig); conn = dbService.getConnection(dbConfig);
stmt = conn.createStatement(); stmt = conn.createStatement();
DatabaseMetaData dbm = conn.getMetaData(); DatabaseMetaData dbm = conn.getMetaData();
// check if "employee" table is there // check if "employee" table is there
ResultSet tables = dbm.getTables(null, null, tableName, null); ResultSet tables = dbm.getTables(null, null, tableName, null);
boolean dropTable = false; boolean dropTable = false;
if (tables.next()) { if (tables.next()) {
dropTable = true; dropTable = true;
//System.out.println("Drop Table Result::" + dropResult); // System.out.println("Drop Table Result::" + dropResult);
} }
tables.close(); tables.close();
if (dropTable) { if (dropTable) {
stmt.executeUpdate("DROP TABLE " + tableName); stmt.executeUpdate("DROP TABLE " + tableName);
} }
String createSQL = " CREATE TABLE " + tableName + " ( " String createSQL = " CREATE TABLE " + tableName + " ( "
+ " ID INT NOT NULL, " + " ID INT NOT NULL, "
+ " NAME VARCHAR (20) NOT NULL, " + " NAME VARCHAR (20) NOT NULL, "
+ " CITY VARCHAR (20) NOT NULL," + " CITY VARCHAR (20) NOT NULL,"
+ " PRIMARY KEY (ID) );"; + " PRIMARY KEY (ID) );";
stmt.executeUpdate(createSQL); stmt.executeUpdate(createSQL);
//System.out.println("Create Table Result::" + createResult); // System.out.println("Create Table Result::" + createResult);
String insertTableSQL = "INSERT INTO " + tableName String insertTableSQL = "INSERT INTO " + tableName
+ "(ID, NAME, CITY) " + "VALUES" + "(ID, NAME, CITY) " + "VALUES"
+ "(1,'frank lens','Dallas')"; + "(1,'frank lens','Dallas')";
stmt.executeUpdate(insertTableSQL); stmt.executeUpdate(insertTableSQL);
// System.out.println("Insert Data Result::" + insertResult); // System.out.println("Insert Data Result::" + insertResult);
logger.info("Database Test Init Data Created!!!"); logger.info("Database Test Init Data Created!!!");
} finally { } finally {
if(stmt != null) { if (stmt != null) {
try { try {
stmt.close(); stmt.close();
} catch (SQLException e) { } catch (SQLException e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
if(conn != null) { if (conn != null) {
try { try {
conn.close(); conn.close();
} catch (SQLException e) { } catch (SQLException e) {
@ -111,55 +112,53 @@ public class DBExtensionTestUtils {
} }
} }
} }
} }
public static void initTestData(DatabaseConfiguration dbConfig) public static void initTestData(DatabaseConfiguration dbConfig)
throws DatabaseServiceException, SQLException { throws DatabaseServiceException, SQLException {
Statement stmt = null; Statement stmt = null;
Connection conn = null; Connection conn = null;
try { try {
DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType()); DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType());
conn = dbService.getConnection(dbConfig); conn = dbService.getConnection(dbConfig);
stmt = conn.createStatement(); stmt = conn.createStatement();
DatabaseMetaData dbm = conn.getMetaData(); DatabaseMetaData dbm = conn.getMetaData();
// check if "employee" table is there // check if "employee" table is there
ResultSet tables = dbm.getTables(null, null, DEFAULT_TEST_TABLE, null); ResultSet tables = dbm.getTables(null, null, DEFAULT_TEST_TABLE, null);
boolean dropTable = false; boolean dropTable = false;
if (tables.next()) { if (tables.next()) {
dropTable = true; dropTable = true;
//System.out.println("Drop Table Result::" + dropResult); // System.out.println("Drop Table Result::" + dropResult);
} }
tables.close(); tables.close();
if (dropTable) { if (dropTable) {
stmt.executeUpdate("DROP TABLE " + DEFAULT_TEST_TABLE); stmt.executeUpdate("DROP TABLE " + DEFAULT_TEST_TABLE);
//System.out.println("Drop Table Result::" + dropResult); // System.out.println("Drop Table Result::" + dropResult);
} }
String createSQL = " CREATE TABLE TEST_DATA( " String createSQL = " CREATE TABLE TEST_DATA( "
+ " ID INT NOT NULL, " + " ID INT NOT NULL, "
+ " NAME VARCHAR (20) NOT NULL, " + " NAME VARCHAR (20) NOT NULL, "
+ " CITY VARCHAR (20) NOT NULL," + " CITY VARCHAR (20) NOT NULL,"
+ " PRIMARY KEY (ID) );"; + " PRIMARY KEY (ID) );";
stmt.executeUpdate(createSQL); stmt.executeUpdate(createSQL);
//System.out.println("Create Table Result::" + createResult); // System.out.println("Create Table Result::" + createResult);
String insertTableSQL = "INSERT INTO TEST_DATA" String insertTableSQL = "INSERT INTO TEST_DATA"
+ "(ID, NAME, CITY) " + "VALUES" + "(ID, NAME, CITY) " + "VALUES"
+ "(1,'frank lens','Dallas')"; + "(1,'frank lens','Dallas')";
stmt.executeUpdate(insertTableSQL); stmt.executeUpdate(insertTableSQL);
// System.out.println("Insert Data Result::" + insertResult); // System.out.println("Insert Data Result::" + insertResult);
logger.info("Database Test Init Data Created!!!"); logger.info("Database Test Init Data Created!!!");
} finally { } finally {
if(stmt != null) { if (stmt != null) {
try { try {
stmt.close(); stmt.close();
} catch (SQLException e) { } catch (SQLException e) {
@ -167,7 +166,7 @@ public class DBExtensionTestUtils {
e.printStackTrace(); e.printStackTrace();
} }
} }
if(conn != null) { if (conn != null) {
try { try {
conn.close(); conn.close();
} catch (SQLException e) { } catch (SQLException e) {
@ -176,31 +175,31 @@ public class DBExtensionTestUtils {
} }
} }
} }
} }
/** /**
* CREATE test data in MySQL * CREATE test data in MySQL Table name: test_data
* Table name: test_data *
* @param sampleSize * @param sampleSize
* @param batchSize * @param batchSize
* @throws DatabaseServiceException * @throws DatabaseServiceException
* @throws SQLException * @throws SQLException
*/ */
public void generateMySQLTestData(int sampleSize, int batchSize) public void generateMySQLTestData(int sampleSize, int batchSize)
throws DatabaseServiceException, SQLException { throws DatabaseServiceException, SQLException {
mncMap = new HashMap<Integer, Integer>(); mncMap = new HashMap<Integer, Integer>();
mccMap = new HashMap<Integer, Integer>(); mccMap = new HashMap<Integer, Integer>();
mccMap.put(0, 302); mccMap.put(0, 302);
mccMap.put(1, 311); mccMap.put(1, 311);
mccMap.put(2, 730); mccMap.put(2, 730);
mccMap.put(1, 622); mccMap.put(1, 622);
mncMap.put(0, 006); mncMap.put(0, 006);
mncMap.put(1, 140); mncMap.put(1, 140);
mncMap.put(2, 380); mncMap.put(2, 380);
mncMap.put(3, 710); mncMap.put(3, 710);
DatabaseConfiguration dc = new DatabaseConfiguration(); DatabaseConfiguration dc = new DatabaseConfiguration();
dc.setDatabaseHost(DEFAULT_MYSQL_HOST); dc.setDatabaseHost(DEFAULT_MYSQL_HOST);
dc.setDatabaseName(DEFAULT_MYSQL_DB_NAME); dc.setDatabaseName(DEFAULT_MYSQL_DB_NAME);
@ -209,75 +208,74 @@ public class DBExtensionTestUtils {
dc.setDatabaseType(MYSQL_DB_NAME); dc.setDatabaseType(MYSQL_DB_NAME);
dc.setDatabaseUser(DEFAULT_MYSQL_USER); dc.setDatabaseUser(DEFAULT_MYSQL_USER);
dc.setUseSSL(false); dc.setUseSSL(false);
String truncateTableSQL = "TRUNCATE test_data"; String truncateTableSQL = "TRUNCATE test_data";
String insertTableSQL = "INSERT INTO test_data(" String insertTableSQL = "INSERT INTO test_data("
+ "id, ue_id, start_time, end_date, bytes_upload, bytes_download, cell_id, mcc, mnc, lac, imei)" + "id, ue_id, start_time, end_date, bytes_upload, bytes_download, cell_id, mcc, mnc, lac, imei)"
+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);";
Connection conn = DatabaseService.get(MYSQL_DB_NAME).getConnection(dc); Connection conn = DatabaseService.get(MYSQL_DB_NAME).getConnection(dc);
Statement truncateStmt = conn.createStatement(); Statement truncateStmt = conn.createStatement();
int result = truncateStmt.executeUpdate(truncateTableSQL); int result = truncateStmt.executeUpdate(truncateTableSQL);
System.out.println("Truncate Table Result::" + result); System.out.println("Truncate Table Result::" + result);
truncateStmt.close(); truncateStmt.close();
conn.setAutoCommit(false); conn.setAutoCommit(false);
PreparedStatement stmt = conn.prepareStatement(insertTableSQL); PreparedStatement stmt = conn.prepareStatement(insertTableSQL);
int counter=1; int counter = 1;
for (int i = 0; i < sampleSize; i++) { for (int i = 0; i < sampleSize; i++) {
stmt.setLong(1, i); stmt.setLong(1, i);
stmt.setString(2, getNextUeId()); stmt.setString(2, getNextUeId());
stmt.setDate(3, getNextStartDate()); stmt.setDate(3, getNextStartDate());
stmt.setDate(4, getNextEndDate()); stmt.setDate(4, getNextEndDate());
stmt.setInt(5, rand.nextInt()); stmt.setInt(5, rand.nextInt());
stmt.setInt(6, rand.nextInt()); stmt.setInt(6, rand.nextInt());
stmt.setInt(7, rand.nextInt(10)); stmt.setInt(7, rand.nextInt(10));
stmt.setInt(8, getMCC()); stmt.setInt(8, getMCC());
stmt.setInt(9, getMNC()); stmt.setInt(9, getMNC());
stmt.setInt(10, rand.nextInt(100)); stmt.setInt(10, rand.nextInt(100));
stmt.setString(11, getNextIMEI()); stmt.setString(11, getNextIMEI());
stmt.addBatch(); stmt.addBatch();
//Execute batch of 1000 records // Execute batch of 1000 records
if(i%batchSize==0){ if (i % batchSize == 0) {
stmt.executeBatch(); stmt.executeBatch();
conn.commit(); conn.commit();
System.out.println("Batch "+(counter++)+" executed successfully"); System.out.println("Batch " + (counter++) + " executed successfully");
} }
} }
//execute final batch // execute final batch
stmt.executeBatch(); stmt.executeBatch();
System.out.println("Final Batch Executed "+(counter++)+" executed successfully"); System.out.println("Final Batch Executed " + (counter++) + " executed successfully");
conn.commit(); conn.commit();
conn.close(); conn.close();
} }
/** /**
* *
* @param sampleSize * @param sampleSize
* @param batchSize * @param batchSize
* @throws DatabaseServiceException * @throws DatabaseServiceException
* @throws SQLException * @throws SQLException
*/ */
public void generatePgSQLTestData(int sampleSize, int batchSize) throws DatabaseServiceException, SQLException { public void generatePgSQLTestData(int sampleSize, int batchSize) throws DatabaseServiceException, SQLException {
mncMap = new HashMap<Integer, Integer>(); mncMap = new HashMap<Integer, Integer>();
mccMap = new HashMap<Integer, Integer>(); mccMap = new HashMap<Integer, Integer>();
mccMap.put(0, 302); mccMap.put(0, 302);
mccMap.put(1, 311); mccMap.put(1, 311);
mccMap.put(2, 730); mccMap.put(2, 730);
mccMap.put(1, 622); mccMap.put(1, 622);
mncMap.put(0, 006); mncMap.put(0, 006);
mncMap.put(1, 140); mncMap.put(1, 140);
mncMap.put(2, 380); mncMap.put(2, 380);
mncMap.put(3, 710); mncMap.put(3, 710);
DatabaseConfiguration dc = new DatabaseConfiguration(); DatabaseConfiguration dc = new DatabaseConfiguration();
dc.setDatabaseHost(DEFAULT_PGSQL_HOST); dc.setDatabaseHost(DEFAULT_PGSQL_HOST);
dc.setDatabaseName(DEFAULT_PGSQL_DB_NAME); dc.setDatabaseName(DEFAULT_PGSQL_DB_NAME);
@ -286,115 +284,111 @@ public class DBExtensionTestUtils {
dc.setDatabaseType(PGSQL_DB_NAME); dc.setDatabaseType(PGSQL_DB_NAME);
dc.setDatabaseUser(DEFAULT_PGSQL_USER); dc.setDatabaseUser(DEFAULT_PGSQL_USER);
dc.setUseSSL(false); dc.setUseSSL(false);
String truncateTableSQL = "TRUNCATE public.test_data"; String truncateTableSQL = "TRUNCATE public.test_data";
String insertTableSQL = "INSERT INTO public.test_data(" String insertTableSQL = "INSERT INTO public.test_data("
+ "id, ue_id, start_time, end_date, bytes_upload, bytes_download, cell_id, mcc, mnc, lac, imei)" + "id, ue_id, start_time, end_date, bytes_upload, bytes_download, cell_id, mcc, mnc, lac, imei)"
+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);";
Connection conn = DatabaseService.get(PGSQL_DB_NAME).getConnection(dc); Connection conn = DatabaseService.get(PGSQL_DB_NAME).getConnection(dc);
Statement truncateStmt = conn.createStatement(); Statement truncateStmt = conn.createStatement();
int result = truncateStmt.executeUpdate(truncateTableSQL); int result = truncateStmt.executeUpdate(truncateTableSQL);
System.out.println("Truncate Table Result::" + result); System.out.println("Truncate Table Result::" + result);
truncateStmt.close(); truncateStmt.close();
conn.setAutoCommit(false); conn.setAutoCommit(false);
PreparedStatement stmt = conn.prepareStatement(insertTableSQL); PreparedStatement stmt = conn.prepareStatement(insertTableSQL);
int counter=1; int counter = 1;
for (int i = 0; i < sampleSize; i++) { for (int i = 0; i < sampleSize; i++) {
stmt.setLong(1, i); stmt.setLong(1, i);
stmt.setString(2, getNextUeId()); stmt.setString(2, getNextUeId());
stmt.setDate(3, getNextStartDate()); stmt.setDate(3, getNextStartDate());
stmt.setDate(4, getNextEndDate()); stmt.setDate(4, getNextEndDate());
stmt.setInt(5, rand.nextInt()); stmt.setInt(5, rand.nextInt());
stmt.setInt(6, rand.nextInt()); stmt.setInt(6, rand.nextInt());
stmt.setInt(7, rand.nextInt(10)); stmt.setInt(7, rand.nextInt(10));
stmt.setInt(8, getMCC()); stmt.setInt(8, getMCC());
stmt.setInt(9, getMNC()); stmt.setInt(9, getMNC());
stmt.setInt(10, rand.nextInt(100)); stmt.setInt(10, rand.nextInt(100));
stmt.setString(11, getNextIMEI()); stmt.setString(11, getNextIMEI());
stmt.addBatch(); stmt.addBatch();
//Execute batch of 1000 records // Execute batch of 1000 records
if(i%batchSize==0){ if (i % batchSize == 0) {
stmt.executeBatch(); stmt.executeBatch();
conn.commit(); conn.commit();
System.out.println("Batch "+(counter++)+" executed successfully"); System.out.println("Batch " + (counter++) + " executed successfully");
} }
} }
//execute final batch // execute final batch
stmt.executeBatch(); stmt.executeBatch();
System.out.println("Final Batch Executed "+(counter++)+" executed successfully"); System.out.println("Final Batch Executed " + (counter++) + " executed successfully");
conn.commit(); conn.commit();
conn.close(); conn.close();
} }
private String getNextIMEI() { private String getNextIMEI() {
int n = 1000000000 + rand.nextInt(900000000); int n = 1000000000 + rand.nextInt(900000000);
return "" + n; return "" + n;
} }
private int getMNC() {
private int getMNC() {
return mncMap.get(rand.nextInt(3)); return mncMap.get(rand.nextInt(3));
} }
private int getMCC() { private int getMCC() {
return mccMap.get(rand.nextInt(3)); return mccMap.get(rand.nextInt(3));
} }
private Date getNextEndDate() { private Date getNextEndDate() {
return new Date(System.currentTimeMillis() + 1); return new Date(System.currentTimeMillis() + 1);
} }
private Date getNextStartDate() { private Date getNextStartDate() {
return new Date(System.currentTimeMillis()); return new Date(System.currentTimeMillis());
} }
private String getNextUeId() { private String getNextUeId() {
int n = 300000000 + rand.nextInt(900000000); int n = 300000000 + rand.nextInt(900000000);
return "" + n; return "" + n;
} }
public static void main(String[] args) throws DatabaseServiceException, SQLException { public static void main(String[] args) throws DatabaseServiceException, SQLException {
DBExtensionTestUtils testUtil = new DBExtensionTestUtils(); DBExtensionTestUtils testUtil = new DBExtensionTestUtils();
testUtil.generatePgSQLTestData(SAMPLE_SIZE, BATCH_SIZE); testUtil.generatePgSQLTestData(SAMPLE_SIZE, BATCH_SIZE);
// testUtil.generateMySQLTestData(); // testUtil.generateMySQLTestData();
} }
public static void cleanUpTestData(DatabaseConfiguration dbConfig) { public static void cleanUpTestData(DatabaseConfiguration dbConfig) {
Statement stmt = null; Statement stmt = null;
Connection conn = null; Connection conn = null;
try { try {
DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType()); DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType());
conn = dbService.getConnection(dbConfig); conn = dbService.getConnection(dbConfig);
stmt = conn.createStatement(); stmt = conn.createStatement();
DatabaseMetaData dbm = conn.getMetaData(); DatabaseMetaData dbm = conn.getMetaData();
// check if "employee" table is there // check if "employee" table is there
ResultSet tables = dbm.getTables(null, null, DEFAULT_TEST_TABLE, null); ResultSet tables = dbm.getTables(null, null, DEFAULT_TEST_TABLE, null);
if (tables.next()) { if (tables.next()) {
stmt.executeUpdate("DROP TABLE " + DEFAULT_TEST_TABLE); stmt.executeUpdate("DROP TABLE " + DEFAULT_TEST_TABLE);
//System.out.println("Drop Table Result::" + dropResult); // System.out.println("Drop Table Result::" + dropResult);
} }
logger.info("Database Test Cleanup Done"); logger.info("Database Test Cleanup Done");
} catch (DatabaseServiceException e) { } catch (DatabaseServiceException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
@ -402,7 +396,7 @@ public class DBExtensionTestUtils {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} finally { } finally {
if(stmt != null) { if (stmt != null) {
try { try {
stmt.close(); stmt.close();
} catch (SQLException e) { } catch (SQLException e) {
@ -410,7 +404,7 @@ public class DBExtensionTestUtils {
e.printStackTrace(); e.printStackTrace();
} }
} }
if(conn != null) { if (conn != null) {
try { try {
conn.close(); conn.close();
} catch (SQLException e) { } catch (SQLException e) {
@ -419,9 +413,9 @@ public class DBExtensionTestUtils {
} }
} }
} }
} }
public static File createTempDirectory(String name) public static File createTempDirectory(String name)
throws IOException { throws IOException {
File dir = File.createTempFile(name, ""); File dir = File.createTempFile(name, "");
@ -429,7 +423,7 @@ public class DBExtensionTestUtils {
dir.mkdir(); dir.mkdir();
return dir; return dir;
} }
public static String getJDBCUrl(DatabaseConfiguration dbConfig) { public static String getJDBCUrl(DatabaseConfiguration dbConfig) {
Map<String, Object> substitutes = new HashMap<String, Object>(); Map<String, Object> substitutes = new HashMap<String, Object>();
substitutes.put("dbType", dbConfig.getDatabaseType()); substitutes.put("dbType", dbConfig.getDatabaseType());

15
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTests.java
View File

@ -38,22 +38,21 @@ import java.util.Properties;
import org.slf4j.Logger; import org.slf4j.Logger;
public class DBExtensionTests { public class DBExtensionTests {
protected final String MYSQL_DB_NAME = "mysql"; protected final String MYSQL_DB_NAME = "mysql";
protected final String DEFAULT_MYSQL_HOST = "127.0.0.1"; protected final String DEFAULT_MYSQL_HOST = "127.0.0.1";
protected final String DEFAULT_MYSQL_PORT = "3306"; protected final String DEFAULT_MYSQL_PORT = "3306";
protected final String DEFAULT_MYSQL_USER = "root"; protected final String DEFAULT_MYSQL_USER = "root";
protected final String DEFAULT_MYSQL_PASSWORD = "secret"; protected final String DEFAULT_MYSQL_PASSWORD = "secret";
protected final String DEFAULT_MYSQL_DB_NAME = "testdb"; protected final String DEFAULT_MYSQL_DB_NAME = "testdb";
protected final String PGSQL_DB_NAME = "postgresql"; protected final String PGSQL_DB_NAME = "postgresql";
protected final String DEFAULT_PGSQL_HOST = "127.0.0.1"; protected final String DEFAULT_PGSQL_HOST = "127.0.0.1";
protected final String DEFAULT_PGSQL_PORT = "5432"; protected final String DEFAULT_PGSQL_PORT = "5432";
protected final String DEFAULT_PGSQL_USER = "postgres"; protected final String DEFAULT_PGSQL_USER = "postgres";
protected final String DEFAULT_PGSQL_PASSWORD = ""; protected final String DEFAULT_PGSQL_PASSWORD = "";
protected final String DEFAULT_PGSQL_DB_NAME = "testdb"; protected final String DEFAULT_PGSQL_DB_NAME = "testdb";
protected final String MARIA_DB_NAME = "mariadb"; protected final String MARIA_DB_NAME = "mariadb";
protected final String DEFAULT_MARIADB_HOST = "127.0.0.1"; protected final String DEFAULT_MARIADB_HOST = "127.0.0.1";
protected final String DEFAULT_MARIADB_PORT = "3306"; protected final String DEFAULT_MARIADB_PORT = "3306";
@ -63,11 +62,11 @@ public class DBExtensionTests {
protected final String SQLITE_DB_NAME = "sqlite"; protected final String SQLITE_DB_NAME = "sqlite";
protected final String DEFAULT_SQLITE_DB_NAME = "extension_test_db.sqlite"; protected final String DEFAULT_SQLITE_DB_NAME = "extension_test_db.sqlite";
protected final String DEFAULT_TEST_TABLE = "test_data"; protected final String DEFAULT_TEST_TABLE = "test_data";
protected Properties properties; protected Properties properties;
protected Logger logger; protected Logger logger;
} }

121
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DatabaseImportControllerTest.java
View File

@ -1,5 +1,5 @@
package com.google.refine.extension.database;
package com.google.refine.extension.database;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@ -34,50 +34,48 @@ import com.google.refine.io.FileProjectManager;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.util.ParsingUtilities; import com.google.refine.util.ParsingUtilities;
@Test(groups = { "requiresMySQL" }) @Test(groups = { "requiresMySQL" })
public class DatabaseImportControllerTest extends DBExtensionTests{ public class DatabaseImportControllerTest extends DBExtensionTests {
@Mock @Mock
private HttpServletRequest request; private HttpServletRequest request;
@Mock @Mock
private HttpServletResponse response; private HttpServletResponse response;
private Project project; private Project project;
private ProjectMetadata metadata; private ProjectMetadata metadata;
private ImportingJob job; private ImportingJob job;
private RefineServlet servlet; private RefineServlet servlet;
private String JSON_OPTION = "{\"mode\":\"row-based\"}}"; private String JSON_OPTION = "{\"mode\":\"row-based\"}}";
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
private String query; private String query;
//System under test // System under test
private DatabaseImportController SUT = null; private DatabaseImportController SUT = null;
@BeforeMethod @BeforeMethod
public void setUp() throws IOException { public void setUp() throws IOException {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
File dir = DBExtensionTestUtils.createTempDirectory("OR_DBExtension_Test_WorkspaceDir"); File dir = DBExtensionTestUtils.createTempDirectory("OR_DBExtension_Test_WorkspaceDir");
FileProjectManager.initialize(dir); FileProjectManager.initialize(dir);
servlet = new RefineDbServletStub(); servlet = new RefineDbServletStub();
ImportingManager.initialize(servlet); ImportingManager.initialize(servlet);
project = new Project(); project = new Project();
metadata = new ProjectMetadata(); metadata = new ProjectMetadata();
job = ImportingManager.createJob(); job = ImportingManager.createJob();
metadata.setName("Database Import Test Project"); metadata.setName("Database Import Test Project");
ProjectManager.singleton.registerProject(project, metadata); ProjectManager.singleton.registerProject(project, metadata);
SUT = new DatabaseImportController(); SUT = new DatabaseImportController();
} }
@AfterMethod @AfterMethod
public void tearDown() { public void tearDown() {
SUT = null; SUT = null;
@ -87,9 +85,8 @@ public class DatabaseImportControllerTest extends DBExtensionTests{
metadata = null; metadata = null;
ImportingManager.disposeJob(job.id); ImportingManager.disposeJob(job.id);
job = null; job = null;
//options = null; // options = null;
} }
@Test @Test
public void testDoGet() { public void testDoGet() {
@ -98,11 +95,11 @@ public class DatabaseImportControllerTest extends DBExtensionTests{
try { try {
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
SUT.doGet(request, response); SUT.doGet(request, response);
String result = sw.getBuffer().toString().trim(); String result = sw.getBuffer().toString().trim();
ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class);
String code = json.get("status").asText(); String code = json.get("status").asText();
String message = json.get("message").asText(); String message = json.get("message").asText();
Assert.assertNotNull(code); Assert.assertNotNull(code);
@ -110,24 +107,23 @@ public class DatabaseImportControllerTest extends DBExtensionTests{
Assert.assertEquals(code, "error"); Assert.assertEquals(code, "error");
Assert.assertEquals(message, "GET not implemented"); Assert.assertEquals(message, "GET not implemented");
} catch (Exception e) { } catch (Exception e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }
} }
@Test @Test
public void testDoPostInvalidSubCommand() throws IOException, ServletException { public void testDoPostInvalidSubCommand() throws IOException, ServletException {
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(request.getQueryString()).thenReturn( when(request.getQueryString()).thenReturn(
"http://127.0.0.1:3333/command/core/importing-controller?controller=database/database-import-controller&subCommand=invalid-sub-command"); "http://127.0.0.1:3333/command/core/importing-controller?controller=database/database-import-controller&subCommand=invalid-sub-command");
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
//test // test
SUT.doPost(request, response); SUT.doPost(request, response);
String result = sw.getBuffer().toString().trim(); String result = sw.getBuffer().toString().trim();
ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class);
@ -138,43 +134,40 @@ public class DatabaseImportControllerTest extends DBExtensionTests{
Assert.assertEquals(code, "error"); Assert.assertEquals(code, "error");
Assert.assertEquals(message, "No such sub command"); Assert.assertEquals(message, "No such sub command");
} }
@Test @Test
public void testDoPostInitializeParser() throws ServletException, IOException { public void testDoPostInitializeParser() throws ServletException, IOException {
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(request.getQueryString()).thenReturn( when(request.getQueryString()).thenReturn(
"http://127.0.0.1:3333/command/core/importing-controller?controller=database/database-import-controller&subCommand=initialize-parser-ui"); "http://127.0.0.1:3333/command/core/importing-controller?controller=database/database-import-controller&subCommand=initialize-parser-ui");
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
SUT.doPost(request, response); SUT.doPost(request, response);
String result = sw.getBuffer().toString().trim(); String result = sw.getBuffer().toString().trim();
ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class);
String status = json.get("status").asText(); String status = json.get("status").asText();
//System.out.println("json::" + json); // System.out.println("json::" + json);
Assert.assertEquals(status, "ok"); Assert.assertEquals(status, "ok");
} }
@Test @Test
public void testDoPostParsePreview() throws IOException, ServletException { public void testDoPostParsePreview() throws IOException, ServletException {
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
long jobId = job.id; long jobId = job.id;
when(request.getQueryString()).thenReturn( when(request.getQueryString()).thenReturn(
"http://127.0.0.1:3333/command/core/importing-controller?controller=database%2Fdatabase-import-controller&jobID=" "http://127.0.0.1:3333/command/core/importing-controller?controller=database%2Fdatabase-import-controller&jobID="
+ jobId + "&subCommand=parse-preview"); + jobId + "&subCommand=parse-preview");
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType()); when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType());
when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost());
when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort());
@ -183,31 +176,30 @@ public class DatabaseImportControllerTest extends DBExtensionTests{
when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName());
when(request.getParameter("query")).thenReturn(query); when(request.getParameter("query")).thenReturn(query);
when(request.getParameter("options")).thenReturn(JSON_OPTION); when(request.getParameter("options")).thenReturn(JSON_OPTION);
SUT.doPost(request, response); SUT.doPost(request, response);
String result = sw.getBuffer().toString().trim(); String result = sw.getBuffer().toString().trim();
ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class);
String status = json.get("status").asText(); String status = json.get("status").asText();
//System.out.println("json::" + json); // System.out.println("json::" + json);
Assert.assertEquals(status, "ok"); Assert.assertEquals(status, "ok");
} }
@Test @Test
public void testDoPostCreateProject() throws IOException, ServletException { public void testDoPostCreateProject() throws IOException, ServletException {
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
long jobId = job.id; long jobId = job.id;
when(request.getQueryString()).thenReturn( when(request.getQueryString()).thenReturn(
"http://127.0.0.1:3333/command/core/importing-controller?controller=database%2Fdatabase-import-controller&jobID=" "http://127.0.0.1:3333/command/core/importing-controller?controller=database%2Fdatabase-import-controller&jobID="
+ jobId + "&subCommand=create-project"); + jobId + "&subCommand=create-project");
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType()); when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType());
when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost());
when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort());
@ -216,27 +208,26 @@ public class DatabaseImportControllerTest extends DBExtensionTests{
when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName());
when(request.getParameter("query")).thenReturn(query); when(request.getParameter("query")).thenReturn(query);
when(request.getParameter("options")).thenReturn(JSON_OPTION); when(request.getParameter("options")).thenReturn(JSON_OPTION);
SUT.doPost(request, response); SUT.doPost(request, response);
String result = sw.getBuffer().toString().trim(); String result = sw.getBuffer().toString().trim();
ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class);
String status = json.get("status").asText(); String status = json.get("status").asText();
//System.out.println("json::" + json); // System.out.println("json::" + json);
Assert.assertEquals(status, "ok"); Assert.assertEquals(status, "ok");
} }
@BeforeTest @BeforeTest
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" })
public void beforeTest( public void beforeTest(
@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, @Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration(); testDbConfig = new DatabaseConfiguration();
testDbConfig.setDatabaseHost(mySqlDbHost); testDbConfig.setDatabaseHost(mySqlDbHost);
testDbConfig.setDatabaseName(mySqlDbName); testDbConfig.setDatabaseName(mySqlDbName);
@ -246,13 +237,11 @@ public class DatabaseImportControllerTest extends DBExtensionTests{
testDbConfig.setDatabaseUser(mySqlDbUser); testDbConfig.setDatabaseUser(mySqlDbUser);
testDbConfig.setUseSSL(false); testDbConfig.setUseSSL(false);
query = "SELECT count(*) FROM " + mySqlTestTable; query = "SELECT count(*) FROM " + mySqlTestTable;
//testTable = mySqlTestTable; // testTable = mySqlTestTable;
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
} }
} }

82
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/DatabaseServiceTest.java
View File

@ -1,5 +1,5 @@
package com.google.refine.extension.database;
package com.google.refine.extension.database;
import java.sql.Connection; import java.sql.Connection;
import java.util.List; import java.util.List;
@ -18,22 +18,17 @@ import com.google.refine.extension.database.model.DatabaseRow;
import com.google.refine.extension.database.mysql.MySQLDatabaseService; import com.google.refine.extension.database.mysql.MySQLDatabaseService;
import com.google.refine.extension.database.pgsql.PgSQLDatabaseService; import com.google.refine.extension.database.pgsql.PgSQLDatabaseService;
public class DatabaseServiceTest extends DBExtensionTests {
public class DatabaseServiceTest extends DBExtensionTests{
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
private String testTable; private String testTable;
@BeforeTest @BeforeTest
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" })
public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
testDbConfig = new DatabaseConfiguration(); testDbConfig = new DatabaseConfiguration();
testDbConfig.setDatabaseHost(mySqlDbHost); testDbConfig.setDatabaseHost(mySqlDbHost);
testDbConfig.setDatabaseName(mySqlDbName); testDbConfig.setDatabaseName(mySqlDbName);
@ -42,18 +37,16 @@ public class DatabaseServiceTest extends DBExtensionTests{
testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME);
testDbConfig.setDatabaseUser(mySqlDbUser); testDbConfig.setDatabaseUser(mySqlDbUser);
testDbConfig.setUseSSL(false); testDbConfig.setUseSSL(false);
testTable = mySqlTestTable; testTable = mySqlTestTable;
// DBExtensionTestUtils.initTestData(testDbConfig); // DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance());
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance());
} }
@Test @Test
public void testGetDatabaseUrl() { public void testGetDatabaseUrl() {
DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType());
@ -61,32 +54,31 @@ public class DatabaseServiceTest extends DBExtensionTests{
Assert.assertNotNull(dbUrl); Assert.assertNotNull(dbUrl);
Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig));
} }
@Test(groups = { "requiresPgSQL" })
@Test(groups = {"requiresPgSQL"})
public void testGetPgSQLDBService() { public void testGetPgSQLDBService() {
DatabaseService dbService = DatabaseService.get(PgSQLDatabaseService.DB_NAME); DatabaseService dbService = DatabaseService.get(PgSQLDatabaseService.DB_NAME);
Assert.assertNotNull(dbService); Assert.assertNotNull(dbService);
Assert.assertEquals(dbService.getClass(), PgSQLDatabaseService.class); Assert.assertEquals(dbService.getClass(), PgSQLDatabaseService.class);
} }
@Test(groups = {"requiresMySQL"}) @Test(groups = { "requiresMySQL" })
public void testGetMySQLDBService() { public void testGetMySQLDBService() {
DatabaseService dbService = DatabaseService.get(MySQLDatabaseService.DB_NAME); DatabaseService dbService = DatabaseService.get(MySQLDatabaseService.DB_NAME);
Assert.assertNotNull(dbService); Assert.assertNotNull(dbService);
Assert.assertEquals(dbService.getClass(), MySQLDatabaseService.class); Assert.assertEquals(dbService.getClass(), MySQLDatabaseService.class);
} }
@Test(groups = {"requiresMariaDB"}) @Test(groups = { "requiresMariaDB" })
public void testGetMariaDBSQLDBService() { public void testGetMariaDBSQLDBService() {
DatabaseService dbService = DatabaseService.get(MariaDBDatabaseService.DB_NAME); DatabaseService dbService = DatabaseService.get(MariaDBDatabaseService.DB_NAME);
Assert.assertNotNull(dbService); Assert.assertNotNull(dbService);
Assert.assertEquals(dbService.getClass(), MariaDBDatabaseService.class); Assert.assertEquals(dbService.getClass(), MariaDBDatabaseService.class);
} }
@Test(groups = {"requiresSQLite"}) @Test(groups = { "requiresSQLite" })
public void testGetSQLiteDBService() { public void testGetSQLiteDBService() {
DatabaseService dbService = DatabaseService.get(SQLiteDatabaseService.DB_NAME); DatabaseService dbService = DatabaseService.get(SQLiteDatabaseService.DB_NAME);
@ -94,28 +86,28 @@ public class DatabaseServiceTest extends DBExtensionTests{
Assert.assertEquals(dbService.getClass(), SQLiteDatabaseService.class); Assert.assertEquals(dbService.getClass(), SQLiteDatabaseService.class);
} }
@Test(groups = {"requiresMySQL"}) @Test(groups = { "requiresMySQL" })
public void testGetConnection() throws DatabaseServiceException { public void testGetConnection() throws DatabaseServiceException {
DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType());
Connection conn = dbService.getConnection(testDbConfig); Connection conn = dbService.getConnection(testDbConfig);
Assert.assertNotNull(conn); Assert.assertNotNull(conn);
} }
@Test(groups = {"requiresMySQL"}) @Test(groups = { "requiresMySQL" })
public void testTestConnection() throws DatabaseServiceException { public void testTestConnection() throws DatabaseServiceException {
DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType());
boolean result = dbService.testConnection(testDbConfig); boolean result = dbService.testConnection(testDbConfig);
Assert.assertEquals(result, true); Assert.assertEquals(result, true);
} }
@Test(groups = {"requiresMySQL"}) @Test(groups = { "requiresMySQL" })
public void testConnect() throws DatabaseServiceException { public void testConnect() throws DatabaseServiceException {
DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType());
DatabaseInfo databaseInfo = dbService.connect(testDbConfig); DatabaseInfo databaseInfo = dbService.connect(testDbConfig);
Assert.assertNotNull(databaseInfo); Assert.assertNotNull(databaseInfo);
} }
@Test(groups = {"requiresMySQL"}) @Test(groups = { "requiresMySQL" })
public void testExecuteQuery() throws DatabaseServiceException { public void testExecuteQuery() throws DatabaseServiceException {
DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType());
DatabaseInfo databaseInfo = dbService.testQuery(testDbConfig, DatabaseInfo databaseInfo = dbService.testQuery(testDbConfig,
@ -132,24 +124,24 @@ public class DatabaseServiceTest extends DBExtensionTests{
Assert.assertEquals(limitQuery, "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";"); Assert.assertEquals(limitQuery, "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";");
} }
@Test(groups = {"requiresMySQL"}) @Test(groups = { "requiresMySQL" })
public void testGetColumns() throws DatabaseServiceException { public void testGetColumns() throws DatabaseServiceException {
List<DatabaseColumn> dbColumns; List<DatabaseColumn> dbColumns;
DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType());
dbColumns = dbService.getColumns(testDbConfig,"SELECT * FROM " + testTable); dbColumns = dbService.getColumns(testDbConfig, "SELECT * FROM " + testTable);
Assert.assertNotNull(dbColumns); Assert.assertNotNull(dbColumns);
int cols = dbColumns.size(); int cols = dbColumns.size();
Assert.assertEquals(cols, 10); Assert.assertEquals(cols, 10);
} }
@Test(groups = {"requiresMySQL"}) @Test(groups = { "requiresMySQL" })
public void testGetRows() throws DatabaseServiceException { public void testGetRows() throws DatabaseServiceException {
DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType());
List<DatabaseRow> dbRows = dbService.getRows(testDbConfig, List<DatabaseRow> dbRows = dbService.getRows(testDbConfig,
"SELECT * FROM " + testTable); "SELECT * FROM " + testTable);
Assert.assertNotNull(dbRows); Assert.assertNotNull(dbRows);
Assert.assertEquals(dbRows.size(), 1); Assert.assertEquals(dbRows.size(), 1);
} }

11
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitMariaDBTestDatabase.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database; package com.google.refine.extension.database;
import java.sql.SQLException; import java.sql.SQLException;
@ -18,10 +19,10 @@ public class InitMariaDBTestDatabase extends DBExtensionTests {
@BeforeSuite @BeforeSuite
@Parameters({ "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariadbTestTable" }) @Parameters({ "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariadbTestTable" })
public void beforeSuite( public void beforeSuite(
@Optional(DEFAULT_MARIADB_NAME) String mariadbDbName, @Optional(DEFAULT_MARIADB_HOST) String mariadbDbHost, @Optional(DEFAULT_MARIADB_NAME) String mariadbDbName, @Optional(DEFAULT_MARIADB_HOST) String mariadbDbHost,
@Optional(DEFAULT_MARIADB_PORT) String mariadbDbPort, @Optional(DEFAULT_MARIADB_USER) String mariadbDbUser, @Optional(DEFAULT_MARIADB_PORT) String mariadbDbPort, @Optional(DEFAULT_MARIADB_USER) String mariadbDbUser,
@Optional(DEFAULT_MARIADB_PASSWORD) String mariadbDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariadbTestTable) @Optional(DEFAULT_MARIADB_PASSWORD) String mariadbDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariadbTestTable)
throws DatabaseServiceException, SQLException { throws DatabaseServiceException, SQLException {
mariadbDbConfig = new DatabaseConfiguration(); mariadbDbConfig = new DatabaseConfiguration();
mariadbDbConfig.setDatabaseHost(mariadbDbHost); mariadbDbConfig.setDatabaseHost(mariadbDbHost);
@ -34,7 +35,7 @@ public class InitMariaDBTestDatabase extends DBExtensionTests {
DBExtensionTestUtils.initTestData(mariadbDbConfig); DBExtensionTestUtils.initTestData(mariadbDbConfig);
} }
@AfterSuite @AfterSuite
public void afterSuite() { public void afterSuite() {
DBExtensionTestUtils.cleanUpTestData(mariadbDbConfig); DBExtensionTestUtils.cleanUpTestData(mariadbDbConfig);

21
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitMySQLTestDatabase.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database; package com.google.refine.extension.database;
import java.sql.SQLException; import java.sql.SQLException;
@ -12,19 +13,19 @@ import com.google.refine.extension.database.mysql.MySQLDatabaseService;
@Test(groups = { "requiresMySQL" }) @Test(groups = { "requiresMySQL" })
public class InitMySQLTestDatabase extends DBExtensionTests { public class InitMySQLTestDatabase extends DBExtensionTests {
private DatabaseConfiguration mysqlDbConfig; private DatabaseConfiguration mysqlDbConfig;
@BeforeSuite @BeforeSuite
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" }) @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" })
public void beforeSuite( public void beforeSuite(
@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, @Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable)
throws DatabaseServiceException, SQLException { throws DatabaseServiceException, SQLException {
//System.out.println("@BeforeSuite\n"); // System.out.println("@BeforeSuite\n");
mysqlDbConfig = new DatabaseConfiguration(); mysqlDbConfig = new DatabaseConfiguration();
mysqlDbConfig.setDatabaseHost(mySqlDbHost); mysqlDbConfig.setDatabaseHost(mySqlDbHost);
mysqlDbConfig.setDatabaseName(mySqlDbName); mysqlDbConfig.setDatabaseName(mySqlDbName);
mysqlDbConfig.setDatabasePassword(mySqlDbPassword); mysqlDbConfig.setDatabasePassword(mySqlDbPassword);
@ -32,10 +33,10 @@ public class InitMySQLTestDatabase extends DBExtensionTests {
mysqlDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); mysqlDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME);
mysqlDbConfig.setDatabaseUser(mySqlDbUser); mysqlDbConfig.setDatabaseUser(mySqlDbUser);
mysqlDbConfig.setUseSSL(false); mysqlDbConfig.setUseSSL(false);
DBExtensionTestUtils.initTestData(mysqlDbConfig); DBExtensionTestUtils.initTestData(mysqlDbConfig);
} }
@AfterSuite @AfterSuite
public void afterSuite() { public void afterSuite() {
DBExtensionTestUtils.cleanUpTestData(mysqlDbConfig); DBExtensionTestUtils.cleanUpTestData(mysqlDbConfig);

15
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitPostgresTestDatabase.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database; package com.google.refine.extension.database;
import java.sql.SQLException; import java.sql.SQLException;
@ -18,11 +19,11 @@ public class InitPostgresTestDatabase extends DBExtensionTests {
@BeforeSuite @BeforeSuite
@Parameters({ "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable" }) @Parameters({ "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable" })
public void beforeSuite( public void beforeSuite(
@Optional(DEFAULT_PGSQL_DB_NAME) String pgSqlDbName, @Optional(DEFAULT_PGSQL_HOST) String pgSqlDbHost, @Optional(DEFAULT_PGSQL_DB_NAME) String pgSqlDbName, @Optional(DEFAULT_PGSQL_HOST) String pgSqlDbHost,
@Optional(DEFAULT_PGSQL_PORT) String pgSqlDbPort, @Optional(DEFAULT_PGSQL_USER) String pgSqlDbUser, @Optional(DEFAULT_PGSQL_PORT) String pgSqlDbPort, @Optional(DEFAULT_PGSQL_USER) String pgSqlDbUser,
@Optional(DEFAULT_PGSQL_PASSWORD) String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String pgSqlTestTable) @Optional(DEFAULT_PGSQL_PASSWORD) String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String pgSqlTestTable)
throws DatabaseServiceException, SQLException { throws DatabaseServiceException, SQLException {
pgsqlDbConfig = new DatabaseConfiguration(); pgsqlDbConfig = new DatabaseConfiguration();
pgsqlDbConfig.setDatabaseHost(pgSqlDbHost); pgsqlDbConfig.setDatabaseHost(pgSqlDbHost);
pgsqlDbConfig.setDatabaseName(pgSqlDbName); pgsqlDbConfig.setDatabaseName(pgSqlDbName);
@ -31,10 +32,10 @@ public class InitPostgresTestDatabase extends DBExtensionTests {
pgsqlDbConfig.setDatabaseType(PgSQLDatabaseService.DB_NAME); pgsqlDbConfig.setDatabaseType(PgSQLDatabaseService.DB_NAME);
pgsqlDbConfig.setDatabaseUser(pgSqlDbUser); pgsqlDbConfig.setDatabaseUser(pgSqlDbUser);
pgsqlDbConfig.setUseSSL(false); pgsqlDbConfig.setUseSSL(false);
DBExtensionTestUtils.initTestData(pgsqlDbConfig); DBExtensionTestUtils.initTestData(pgsqlDbConfig);
} }
@AfterSuite @AfterSuite
public void afterSuite() { public void afterSuite() {
DBExtensionTestUtils.cleanUpTestData(pgsqlDbConfig); DBExtensionTestUtils.cleanUpTestData(pgsqlDbConfig);

7
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/InitSQLiteTestDatabase.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database; package com.google.refine.extension.database;
import java.sql.SQLException; import java.sql.SQLException;
@ -14,17 +15,17 @@ public class InitSQLiteTestDatabase extends DBExtensionTests {
private DatabaseConfiguration sqliteDbConfig; private DatabaseConfiguration sqliteDbConfig;
@BeforeSuite @BeforeSuite
@Parameters({ "sqliteDbName", "sqliteTestTable"}) @Parameters({ "sqliteDbName", "sqliteTestTable" })
public void beforeSuite( public void beforeSuite(
@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable) @Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable)
throws DatabaseServiceException, SQLException { throws DatabaseServiceException, SQLException {
sqliteDbConfig = new DatabaseConfiguration(); sqliteDbConfig = new DatabaseConfiguration();
sqliteDbConfig.setDatabaseName(sqliteDbName); sqliteDbConfig.setDatabaseName(sqliteDbName);
DBExtensionTestUtils.initTestData(sqliteDbConfig); DBExtensionTestUtils.initTestData(sqliteDbConfig);
} }
@AfterSuite @AfterSuite
public void afterSuite() { public void afterSuite() {
DBExtensionTestUtils.cleanUpTestData(sqliteDbConfig); DBExtensionTestUtils.cleanUpTestData(sqliteDbConfig);

39
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/SimpleTextEncryptorTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database; package com.google.refine.extension.database;
import org.testng.Assert; import org.testng.Assert;
@ -5,24 +6,24 @@ import org.testng.annotations.Test;
public class SimpleTextEncryptorTest { public class SimpleTextEncryptorTest {
@Test @Test
public void encrypt() { public void encrypt() {
SimpleTextEncryptor textEncryptor = new SimpleTextEncryptor("WEWssa!@d445d"); SimpleTextEncryptor textEncryptor = new SimpleTextEncryptor("WEWssa!@d445d");
String password = "testpass"; String password = "testpass";
String encPass = textEncryptor.encrypt(password); String encPass = textEncryptor.encrypt(password);
Assert.assertNotNull(encPass); Assert.assertNotNull(encPass);
Assert.assertNotEquals(encPass, password); Assert.assertNotEquals(encPass, password);
}
@Test }
public void decrypt() {
SimpleTextEncryptor textEncryptor = new SimpleTextEncryptor("OOEWssa!@d445d"); @Test
String password = "testpass"; public void decrypt() {
String encPass = textEncryptor.encrypt(password); SimpleTextEncryptor textEncryptor = new SimpleTextEncryptor("OOEWssa!@d445d");
Assert.assertNotNull(encPass); String password = "testpass";
Assert.assertNotEquals(encPass, password); String encPass = textEncryptor.encrypt(password);
String decPass = textEncryptor.decrypt(encPass); Assert.assertNotNull(encPass);
Assert.assertEquals(decPass, password); Assert.assertNotEquals(encPass, password);
} String decPass = textEncryptor.decrypt(encPass);
Assert.assertEquals(decPass, password);
}
} }

51
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/ConnectCommandTest.java
View File

@ -27,10 +27,8 @@ import com.google.refine.extension.database.DatabaseService;
import com.google.refine.extension.database.mysql.MySQLDatabaseService; import com.google.refine.extension.database.mysql.MySQLDatabaseService;
import com.google.refine.util.ParsingUtilities; import com.google.refine.util.ParsingUtilities;
@Test(groups = { "requiresMySQL" }) @Test(groups = { "requiresMySQL" })
public class ConnectCommandTest extends DBExtensionTests { public class ConnectCommandTest extends DBExtensionTests {
@Mock @Mock
private HttpServletRequest request; private HttpServletRequest request;
@ -39,17 +37,16 @@ public class ConnectCommandTest extends DBExtensionTests {
private HttpServletResponse response; private HttpServletResponse response;
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
// private String testTable; // private String testTable;
@BeforeTest @BeforeTest
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" })
public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration(); testDbConfig = new DatabaseConfiguration();
testDbConfig.setDatabaseHost(mySqlDbHost); testDbConfig.setDatabaseHost(mySqlDbHost);
testDbConfig.setDatabaseName(mySqlDbName); testDbConfig.setDatabaseName(mySqlDbName);
@ -58,15 +55,14 @@ public class ConnectCommandTest extends DBExtensionTests {
testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME);
testDbConfig.setDatabaseUser(mySqlDbUser); testDbConfig.setDatabaseUser(mySqlDbUser);
testDbConfig.setUseSSL(false); testDbConfig.setUseSSL(false);
//testTable = mySqlTestTable; // testTable = mySqlTestTable;
//DBExtensionTestUtils.initTestData(testDbConfig); // DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
} }
@Test @Test
public void testDoPost() throws IOException, ServletException { public void testDoPost() throws IOException, ServletException {
@ -77,37 +73,38 @@ public class ConnectCommandTest extends DBExtensionTests {
when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword());
when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName());
when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken());
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
ConnectCommand connectCommand = new ConnectCommand(); ConnectCommand connectCommand = new ConnectCommand();
connectCommand.doPost(request, response); connectCommand.doPost(request, response);
String result = sw.getBuffer().toString().trim(); String result = sw.getBuffer().toString().trim();
ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class);
String code = json.get("code").asText(); String code = json.get("code").asText();
Assert.assertEquals(code, "ok"); Assert.assertEquals(code, "ok");
String databaseInfo = json.get("databaseInfo").asText(); String databaseInfo = json.get("databaseInfo").asText();
Assert.assertNotNull(databaseInfo); Assert.assertNotNull(databaseInfo);
} }
@Test @Test
public void testCsrfProtection() throws ServletException, IOException { public void testCsrfProtection() throws ServletException, IOException {
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
ConnectCommand connectCommand = new ConnectCommand(); ConnectCommand connectCommand = new ConnectCommand();
connectCommand.doPost(request, response); connectCommand.doPost(request, response);
Assert.assertEquals( Assert.assertEquals(
ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", ObjectNode.class), ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}",
ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class)); ObjectNode.class),
ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class));
} }
} }

69
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/ExecuteQueryCommandTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database.cmd; package com.google.refine.extension.database.cmd;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@ -28,40 +29,38 @@ import com.google.refine.util.ParsingUtilities;
@Test(groups = { "requiresMySQL" }) @Test(groups = { "requiresMySQL" })
public class ExecuteQueryCommandTest extends DBExtensionTests { public class ExecuteQueryCommandTest extends DBExtensionTests {
@Mock @Mock
private HttpServletRequest request; private HttpServletRequest request;
@Mock @Mock
private HttpServletResponse response; private HttpServletResponse response;
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
private String testTable; private String testTable;
@BeforeTest
@BeforeTest @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" })
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
@Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
MockitoAnnotations.initMocks(this);
MockitoAnnotations.initMocks(this); testDbConfig = new DatabaseConfiguration();
testDbConfig = new DatabaseConfiguration(); testDbConfig.setDatabaseHost(mySqlDbHost);
testDbConfig.setDatabaseHost(mySqlDbHost); testDbConfig.setDatabaseName(mySqlDbName);
testDbConfig.setDatabaseName(mySqlDbName); testDbConfig.setDatabasePassword(mySqlDbPassword);
testDbConfig.setDatabasePassword(mySqlDbPassword); testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort));
testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME);
testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); testDbConfig.setDatabaseUser(mySqlDbUser);
testDbConfig.setDatabaseUser(mySqlDbUser); testDbConfig.setUseSSL(false);
testDbConfig.setUseSSL(false);
testTable = mySqlTestTable;
testTable = mySqlTestTable;
// DBExtensionTestUtils.initTestData(testDbConfig); // DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
} }
@Test @Test
public void testDoPost() throws IOException, ServletException { public void testDoPost() throws IOException, ServletException {
@ -74,20 +73,19 @@ public class ExecuteQueryCommandTest extends DBExtensionTests {
when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName());
when(request.getParameter("queryString")).thenReturn("SELECT count(*) FROM " + testTable); when(request.getParameter("queryString")).thenReturn("SELECT count(*) FROM " + testTable);
when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken());
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
ExecuteQueryCommand executeQueryCommand = new ExecuteQueryCommand(); ExecuteQueryCommand executeQueryCommand = new ExecuteQueryCommand();
executeQueryCommand.doPost(request, response); executeQueryCommand.doPost(request, response);
String result = sw.getBuffer().toString().trim(); String result = sw.getBuffer().toString().trim();
ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class);
String code = json.get("code").asText(); String code = json.get("code").asText();
Assert.assertEquals(code, "ok"); Assert.assertEquals(code, "ok");
@ -97,15 +95,16 @@ public class ExecuteQueryCommandTest extends DBExtensionTests {
@Test @Test
public void testCsrfProtection() throws ServletException, IOException { public void testCsrfProtection() throws ServletException, IOException {
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
ConnectCommand connectCommand = new ConnectCommand(); ConnectCommand connectCommand = new ConnectCommand();
connectCommand.doPost(request, response); connectCommand.doPost(request, response);
Assert.assertEquals( Assert.assertEquals(
ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", ObjectNode.class), ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}",
ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class)); ObjectNode.class),
ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class));
} }
} }

198
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/SavedConnectionCommandTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database.cmd; package com.google.refine.extension.database.cmd;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@ -43,8 +44,8 @@ import com.google.refine.io.FileProjectManager;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.util.ParsingUtilities; import com.google.refine.util.ParsingUtilities;
public class SavedConnectionCommandTest extends DBExtensionTests{ public class SavedConnectionCommandTest extends DBExtensionTests {
@Mock @Mock
private HttpServletRequest request; private HttpServletRequest request;
@ -52,37 +53,36 @@ public class SavedConnectionCommandTest extends DBExtensionTests{
private HttpServletResponse response; private HttpServletResponse response;
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
private Project project; private Project project;
private ProjectMetadata metadata; private ProjectMetadata metadata;
//private ImportingJob job; // private ImportingJob job;
private RefineServlet servlet; private RefineServlet servlet;
// private String JSON_OPTION = "{\"mode\":\"row-based\"}}"; // private String JSON_OPTION = "{\"mode\":\"row-based\"}}";
// System under test
//System under test
private SavedConnectionCommand SUT = null; private SavedConnectionCommand SUT = null;
@BeforeMethod @BeforeMethod
public void setUp() throws IOException { public void setUp() throws IOException {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
File dir = DBExtensionTestUtils.createTempDirectory("OR_DBExtension_Test_WorkspaceDir"); File dir = DBExtensionTestUtils.createTempDirectory("OR_DBExtension_Test_WorkspaceDir");
FileProjectManager.initialize(dir); FileProjectManager.initialize(dir);
servlet = new RefineDbServletStub(); servlet = new RefineDbServletStub();
ImportingManager.initialize(servlet); ImportingManager.initialize(servlet);
project = new Project(); project = new Project();
metadata = new ProjectMetadata(); metadata = new ProjectMetadata();
//job = ImportingManager.createJob(); // job = ImportingManager.createJob();
metadata.setName("Save DB Config Test Project"); metadata.setName("Save DB Config Test Project");
ProjectManager.singleton.registerProject(project, metadata); ProjectManager.singleton.registerProject(project, metadata);
SUT = new SavedConnectionCommand(); SUT = new SavedConnectionCommand();
} }
@AfterMethod @AfterMethod
public void tearDown() { public void tearDown() {
SUT = null; SUT = null;
@ -90,35 +90,33 @@ public class SavedConnectionCommandTest extends DBExtensionTests{
response = null; response = null;
project = null; project = null;
metadata = null; metadata = null;
// ImportingManager.disposeJob(job.id); // ImportingManager.disposeJob(job.id);
// job = null; // job = null;
//options = null; // options = null;
} }
@BeforeTest @BeforeTest
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" })
public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
// MockitoAnnotations.initMocks(this); // MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration(); testDbConfig = new DatabaseConfiguration();
testDbConfig.setDatabaseHost(mySqlDbHost); testDbConfig.setDatabaseHost(mySqlDbHost);
testDbConfig.setDatabaseName(mySqlDbName); testDbConfig.setDatabaseName(mySqlDbName);
testDbConfig.setDatabasePassword(mySqlDbPassword); testDbConfig.setDatabasePassword(mySqlDbPassword);
testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort));
testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME);
testDbConfig.setDatabaseUser(mySqlDbUser); testDbConfig.setDatabaseUser(mySqlDbUser);
testDbConfig.setUseSSL(false); testDbConfig.setUseSSL(false);
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
} }
private void saveDatabaseConfiguration(String savedDbName) { private void saveDatabaseConfiguration(String savedDbName) {
when(request.getParameter("connectionName")).thenReturn(savedDbName); when(request.getParameter("connectionName")).thenReturn(savedDbName);
when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME);
when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost());
@ -129,7 +127,7 @@ public class SavedConnectionCommandTest extends DBExtensionTests{
when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken());
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
try { try {
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
@ -139,12 +137,12 @@ public class SavedConnectionCommandTest extends DBExtensionTests{
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }
} }
@Test @Test
public void testDoPost() throws IOException, ServletException { public void testDoPost() throws IOException, ServletException {
when(request.getParameter("connectionName")).thenReturn("test-db-name"); when(request.getParameter("connectionName")).thenReturn("test-db-name");
when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME);
when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost());
@ -153,36 +151,35 @@ public class SavedConnectionCommandTest extends DBExtensionTests{
when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword());
when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName());
when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken());
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
SUT.doPost(request, response); SUT.doPost(request, response);
String result = sw.getBuffer().toString().trim(); String result = sw.getBuffer().toString().trim();
assertNotNull(result); assertNotNull(result);
assertFalse(result.isEmpty(), "Valid response Message expected!"); assertFalse(result.isEmpty(), "Valid response Message expected!");
ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class);
// System.out.println("json:" + json); // System.out.println("json:" + json);
ArrayNode savedConnections = (ArrayNode) json.get("savedConnections"); ArrayNode savedConnections = (ArrayNode) json.get("savedConnections");
Assert.assertNotNull(savedConnections); Assert.assertNotNull(savedConnections);
int len = savedConnections.size(); int len = savedConnections.size();
Assert.assertEquals(len, 1); Assert.assertEquals(len, 1);
} }
@Test @Test
public void testDoGet() throws IOException, ServletException { public void testDoGet() throws IOException, ServletException {
String testDbName = "testLocalDb"; String testDbName = "testLocalDb";
//add saved connection // add saved connection
saveDatabaseConfiguration(testDbName); saveDatabaseConfiguration(testDbName);
when(request.getParameter("connectionName")).thenReturn(testDbName); when(request.getParameter("connectionName")).thenReturn(testDbName);
when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME);
when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost());
@ -191,22 +188,22 @@ public class SavedConnectionCommandTest extends DBExtensionTests{
when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword());
when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName());
when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken());
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
SUT.doGet(request, response); SUT.doGet(request, response);
ObjectNode json = ParsingUtilities.mapper.readValue(sw.getBuffer().toString().trim(), ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(sw.getBuffer().toString().trim(), ObjectNode.class);
ArrayNode savedConnections = (ArrayNode) json.get("savedConnections"); ArrayNode savedConnections = (ArrayNode) json.get("savedConnections");
Assert.assertNotNull(savedConnections); Assert.assertNotNull(savedConnections);
Assert.assertEquals(savedConnections.size(), 1); Assert.assertEquals(savedConnections.size(), 1);
ObjectNode sc = (ObjectNode)savedConnections.get(0); ObjectNode sc = (ObjectNode) savedConnections.get(0);
String connName = sc.get("connectionName").asText(); String connName = sc.get("connectionName").asText();
Assert.assertEquals(connName, testDbName); Assert.assertEquals(connName, testDbName);
} }
@ -215,14 +212,13 @@ public class SavedConnectionCommandTest extends DBExtensionTests{
public void testDoPut() throws IOException, ServletException { public void testDoPut() throws IOException, ServletException {
String testDbName = "testLocalDb"; String testDbName = "testLocalDb";
saveDatabaseConfiguration(testDbName); saveDatabaseConfiguration(testDbName);
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
//modify database config // modify database config
String newHost = "localhost"; String newHost = "localhost";
when(request.getParameter("connectionName")).thenReturn(testDbName); when(request.getParameter("connectionName")).thenReturn(testDbName);
when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME);
@ -232,16 +228,16 @@ public class SavedConnectionCommandTest extends DBExtensionTests{
when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword());
when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName());
when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken());
SUT.doPut(request, response); SUT.doPut(request, response);
ObjectNode json = ParsingUtilities.mapper.readValue(sw.getBuffer().toString().trim(), ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(sw.getBuffer().toString().trim(), ObjectNode.class);
ArrayNode savedConnections = (ArrayNode) json.get("savedConnections"); ArrayNode savedConnections = (ArrayNode) json.get("savedConnections");
Assert.assertNotNull(savedConnections); Assert.assertNotNull(savedConnections);
Assert.assertEquals(savedConnections.size(), 1); Assert.assertEquals(savedConnections.size(), 1);
ObjectNode sc = (ObjectNode)savedConnections.get(0); ObjectNode sc = (ObjectNode) savedConnections.get(0);
String newDbHost = sc.get("databaseHost").asText(); String newDbHost = sc.get("databaseHost").asText();
Assert.assertEquals(newDbHost, newHost); Assert.assertEquals(newDbHost, newHost);
} }
@ -250,64 +246,62 @@ public class SavedConnectionCommandTest extends DBExtensionTests{
public void testDoDeleteValidConnectionName() { public void testDoDeleteValidConnectionName() {
String testDbName = "testLocalDb"; String testDbName = "testLocalDb";
saveDatabaseConfiguration(testDbName); saveDatabaseConfiguration(testDbName);
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
try { try {
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
when(request.getParameter("connectionName")).thenReturn(testDbName); when(request.getParameter("connectionName")).thenReturn(testDbName);
SUT.doDelete(request, response); SUT.doDelete(request, response);
ObjectNode json = ParsingUtilities.mapper.readValue(sw.getBuffer().toString().trim(), ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(sw.getBuffer().toString().trim(), ObjectNode.class);
ArrayNode savedConnections = (ArrayNode) json.get("savedConnections"); ArrayNode savedConnections = (ArrayNode) json.get("savedConnections");
Assert.assertNotNull(savedConnections); Assert.assertNotNull(savedConnections);
Assert.assertEquals(savedConnections.size(), 0); Assert.assertEquals(savedConnections.size(), 0);
} catch (Exception e) { } catch (Exception e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }
} }
@Test @Test
public void testDoDeleteInValidConnectionName() { public void testDoDeleteInValidConnectionName() {
String testDbName = "testLocalDb"; String testDbName = "testLocalDb";
saveDatabaseConfiguration(testDbName); saveDatabaseConfiguration(testDbName);
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
try { try {
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
when(request.getParameter("connectionName")).thenReturn("noDbName"); when(request.getParameter("connectionName")).thenReturn("noDbName");
SUT.doDelete(request, response); SUT.doDelete(request, response);
ObjectNode json = ParsingUtilities.mapper.createObjectNode(); ObjectNode json = ParsingUtilities.mapper.createObjectNode();
Assert.assertNotNull(json); Assert.assertNotNull(json);
} catch (Exception e) { } catch (Exception e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }
} }
/** /**
* Added to check XSS invalid tokens * Added to check XSS invalid tokens
*
* @throws IOException * @throws IOException
* @throws ServletException * @throws ServletException
*/ */
@Test @Test
public void testDoPostInvalidConnectionName() throws IOException, ServletException { public void testDoPostInvalidConnectionName() throws IOException, ServletException {
when(request.getParameter("connectionName")).thenReturn("<img></img>"); when(request.getParameter("connectionName")).thenReturn("<img></img>");
when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME);
when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost());
when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort());
@ -315,29 +309,29 @@ public class SavedConnectionCommandTest extends DBExtensionTests{
when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword());
when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName());
when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken());
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
SUT.doPost(request, response); SUT.doPost(request, response);
verify(response, times(1)).sendError(HttpStatus.SC_BAD_REQUEST, "Connection Name is Invalid. Expecting [a-zA-Z0-9._-]"); verify(response, times(1)).sendError(HttpStatus.SC_BAD_REQUEST, "Connection Name is Invalid. Expecting [a-zA-Z0-9._-]");
} }
@Test @Test
public void testCsrfProtection() throws ServletException, IOException { public void testCsrfProtection() throws ServletException, IOException {
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
SUT.doPost(request, response); SUT.doPost(request, response);
Assert.assertEquals( Assert.assertEquals(
ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", ObjectNode.class), ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}",
ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class)); ObjectNode.class),
ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class));
} }
} }

81
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/TestConnectCommandTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database.cmd; package com.google.refine.extension.database.cmd;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@ -26,10 +27,9 @@ import com.google.refine.extension.database.DatabaseService;
import com.google.refine.extension.database.mysql.MySQLDatabaseService; import com.google.refine.extension.database.mysql.MySQLDatabaseService;
import com.google.refine.util.ParsingUtilities; import com.google.refine.util.ParsingUtilities;
@Test(groups = { "requiresMySQL" }) @Test(groups = { "requiresMySQL" })
public class TestConnectCommandTest extends DBExtensionTests{ public class TestConnectCommandTest extends DBExtensionTests {
@Mock @Mock
private HttpServletRequest request; private HttpServletRequest request;
@ -38,37 +38,34 @@ public class TestConnectCommandTest extends DBExtensionTests{
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
// private String testTable; // private String testTable;
@BeforeTest
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"})
public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration();
testDbConfig.setDatabaseHost(mySqlDbHost);
testDbConfig.setDatabaseName(mySqlDbName);
testDbConfig.setDatabasePassword(mySqlDbPassword);
testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort));
testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME);
testDbConfig.setDatabaseUser(mySqlDbUser);
testDbConfig.setUseSSL(false);
//testTable = mySqlTestTable;
// DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
}
@BeforeTest
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" })
public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration();
testDbConfig.setDatabaseHost(mySqlDbHost);
testDbConfig.setDatabaseName(mySqlDbName);
testDbConfig.setDatabasePassword(mySqlDbPassword);
testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort));
testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME);
testDbConfig.setDatabaseUser(mySqlDbUser);
testDbConfig.setUseSSL(false);
// testTable = mySqlTestTable;
// DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
}
@Test @Test
public void testDoPost() throws IOException, ServletException { public void testDoPost() throws IOException, ServletException {
when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME);
when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost());
when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort());
@ -76,7 +73,6 @@ public class TestConnectCommandTest extends DBExtensionTests{
when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword());
when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName());
when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken()); when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken());
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
@ -84,29 +80,30 @@ public class TestConnectCommandTest extends DBExtensionTests{
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
TestConnectCommand connectCommand = new TestConnectCommand(); TestConnectCommand connectCommand = new TestConnectCommand();
connectCommand.doPost(request, response); connectCommand.doPost(request, response);
String result = sw.getBuffer().toString().trim(); String result = sw.getBuffer().toString().trim();
ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class);
String code = json.get("code").asText(); String code = json.get("code").asText();
Assert.assertEquals(code, "ok"); Assert.assertEquals(code, "ok");
} }
@Test @Test
public void testCsrfProtection() throws ServletException, IOException { public void testCsrfProtection() throws ServletException, IOException {
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
ConnectCommand connectCommand = new ConnectCommand(); ConnectCommand connectCommand = new ConnectCommand();
connectCommand.doPost(request, response); connectCommand.doPost(request, response);
Assert.assertEquals( Assert.assertEquals(
ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", ObjectNode.class), ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}",
ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class)); ObjectNode.class),
ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class));
} }
} }

71
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/cmd/TestQueryCommandTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database.cmd; package com.google.refine.extension.database.cmd;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@ -28,41 +29,38 @@ import com.google.refine.util.ParsingUtilities;
@Test(groups = { "requiresMySQL" }) @Test(groups = { "requiresMySQL" })
public class TestQueryCommandTest extends DBExtensionTests { public class TestQueryCommandTest extends DBExtensionTests {
@Mock @Mock
private HttpServletRequest request; private HttpServletRequest request;
@Mock @Mock
private HttpServletResponse response; private HttpServletResponse response;
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
private String testTable; private String testTable;
@BeforeTest
@BeforeTest @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" })
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
@Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
MockitoAnnotations.initMocks(this);
MockitoAnnotations.initMocks(this); testDbConfig = new DatabaseConfiguration();
testDbConfig = new DatabaseConfiguration(); testDbConfig.setDatabaseHost(mySqlDbHost);
testDbConfig.setDatabaseHost(mySqlDbHost); testDbConfig.setDatabaseName(mySqlDbName);
testDbConfig.setDatabaseName(mySqlDbName); testDbConfig.setDatabasePassword(mySqlDbPassword);
testDbConfig.setDatabasePassword(mySqlDbPassword); testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort));
testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME);
testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); testDbConfig.setDatabaseUser(mySqlDbUser);
testDbConfig.setDatabaseUser(mySqlDbUser); testDbConfig.setUseSSL(false);
testDbConfig.setUseSSL(false);
testTable = mySqlTestTable;
testTable = mySqlTestTable; // DBExtensionTestUtils.initTestData(testDbConfig);
//DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
}
}
@Test @Test
public void testDoPost() throws IOException, ServletException { public void testDoPost() throws IOException, ServletException {
@ -82,12 +80,12 @@ public class TestQueryCommandTest extends DBExtensionTests {
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
TestQueryCommand executeQueryCommand = new TestQueryCommand(); TestQueryCommand executeQueryCommand = new TestQueryCommand();
executeQueryCommand.doPost(request, response); executeQueryCommand.doPost(request, response);
String result = sw.getBuffer().toString().trim(); String result = sw.getBuffer().toString().trim();
ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class); ObjectNode json = ParsingUtilities.mapper.readValue(result, ObjectNode.class);
String code = json.get("code").asText(); String code = json.get("code").asText();
Assert.assertEquals(code, "ok"); Assert.assertEquals(code, "ok");
@ -95,19 +93,20 @@ public class TestQueryCommandTest extends DBExtensionTests {
Assert.assertNotNull(queryResult); Assert.assertNotNull(queryResult);
} }
@Test @Test
public void testCsrfProtection() throws ServletException, IOException { public void testCsrfProtection() throws ServletException, IOException {
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
when(response.getWriter()).thenReturn(pw); when(response.getWriter()).thenReturn(pw);
TestQueryCommand connectCommand = new TestQueryCommand(); TestQueryCommand connectCommand = new TestQueryCommand();
connectCommand.doPost(request, response); connectCommand.doPost(request, response);
Assert.assertEquals( Assert.assertEquals(
ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", ObjectNode.class), ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}",
ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class)); ObjectNode.class),
ParsingUtilities.mapper.readValue(sw.toString(), ObjectNode.class));
} }
} }

53
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManagerTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database.mariadb; package com.google.refine.extension.database.mariadb;
import java.sql.Connection; import java.sql.Connection;
@ -17,18 +18,15 @@ import com.google.refine.extension.database.DatabaseServiceException;
@Test(groups = { "requiresMariaDB" }) @Test(groups = { "requiresMariaDB" })
public class MariaDBConnectionManagerTest extends DBExtensionTests { public class MariaDBConnectionManagerTest extends DBExtensionTests {
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
@BeforeTest @BeforeTest
@Parameters({ "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariaTestTable"}) @Parameters({ "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariaTestTable" })
public void beforeTest(@Optional(DEFAULT_MARIADB_NAME) String mariaDbName, @Optional(DEFAULT_MARIADB_HOST) String mariaDbHost, public void beforeTest(@Optional(DEFAULT_MARIADB_NAME) String mariaDbName, @Optional(DEFAULT_MARIADB_HOST) String mariaDbHost,
@Optional(DEFAULT_MARIADB_PORT) String mariaDbPort, @Optional(DEFAULT_MARIADB_USER) String mariaDbUser, @Optional(DEFAULT_MARIADB_PORT) String mariaDbPort, @Optional(DEFAULT_MARIADB_USER) String mariaDbUser,
@Optional(DEFAULT_MARIADB_PASSWORD) String mariaDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariaDbTestTable) { @Optional(DEFAULT_MARIADB_PASSWORD) String mariaDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariaDbTestTable) {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration(); testDbConfig = new DatabaseConfiguration();
@ -39,38 +37,37 @@ public class MariaDBConnectionManagerTest extends DBExtensionTests {
testDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME); testDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME);
testDbConfig.setDatabaseUser(mariaDbUser); testDbConfig.setDatabaseUser(mariaDbUser);
testDbConfig.setUseSSL(false); testDbConfig.setUseSSL(false);
// testTable = mariaDbTestTable; // testTable = mariaDbTestTable;
// DBExtensionTestUtils.initTestData(testDbConfig); // DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance());
} }
@Test @Test
public void testTestConnection() throws DatabaseServiceException { public void testTestConnection() throws DatabaseServiceException {
boolean conn = MariaDBConnectionManager.getInstance().testConnection(testDbConfig); boolean conn = MariaDBConnectionManager.getInstance().testConnection(testDbConfig);
Assert.assertEquals(conn, true); Assert.assertEquals(conn, true);
} }
@Test @Test
public void testGetConnection() throws DatabaseServiceException { public void testGetConnection() throws DatabaseServiceException {
Connection conn = MariaDBConnectionManager.getInstance().getConnection(testDbConfig, true); Connection conn = MariaDBConnectionManager.getInstance().getConnection(testDbConfig, true);
Assert.assertNotNull(conn); Assert.assertNotNull(conn);
} }
@Test @Test
public void testShutdown() throws DatabaseServiceException, SQLException { public void testShutdown() throws DatabaseServiceException, SQLException {
Connection conn = MariaDBConnectionManager.getInstance().getConnection(testDbConfig, true); Connection conn = MariaDBConnectionManager.getInstance().getConnection(testDbConfig, true);
Assert.assertNotNull(conn); Assert.assertNotNull(conn);
MariaDBConnectionManager.getInstance().shutdown(); MariaDBConnectionManager.getInstance().shutdown();
if(conn != null) { if (conn != null) {
Assert.assertEquals(conn.isClosed(), true); Assert.assertEquals(conn.isClosed(), true);
} }
} }
} }

147
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseServiceTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database.mariadb; package com.google.refine.extension.database.mariadb;
import java.sql.Connection; import java.sql.Connection;
@ -20,21 +21,18 @@ import com.google.refine.extension.database.model.DatabaseInfo;
import com.google.refine.extension.database.model.DatabaseRow; import com.google.refine.extension.database.model.DatabaseRow;
@Test(groups = { "requiresMariaDB" }) @Test(groups = { "requiresMariaDB" })
public class MariaDBDatabaseServiceTest extends DBExtensionTests{ public class MariaDBDatabaseServiceTest extends DBExtensionTests {
private DatabaseConfiguration testDbConfig;
private DatabaseConfiguration testDbConfig; private String testTable;
private String testTable;
@BeforeTest @BeforeTest
@Parameters({ "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariadbTestTable"}) @Parameters({ "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariadbTestTable" })
public void beforeTest(@Optional(DEFAULT_MARIADB_NAME) String mariaDbName, @Optional(DEFAULT_MARIADB_HOST) String mariaDbHost, public void beforeTest(@Optional(DEFAULT_MARIADB_NAME) String mariaDbName, @Optional(DEFAULT_MARIADB_HOST) String mariaDbHost,
@Optional(DEFAULT_MARIADB_PORT) String mariaDbPort, @Optional(DEFAULT_MARIADB_USER) String mariaDbUser, @Optional(DEFAULT_MARIADB_PORT) String mariaDbPort, @Optional(DEFAULT_MARIADB_USER) String mariaDbUser,
@Optional(DEFAULT_MARIADB_PASSWORD) String mariaDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariaDbTestTable) { @Optional(DEFAULT_MARIADB_PASSWORD) String mariaDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariaDbTestTable) {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration(); testDbConfig = new DatabaseConfiguration();
@ -45,89 +43,88 @@ public class MariaDBDatabaseServiceTest extends DBExtensionTests{
testDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME); testDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME);
testDbConfig.setDatabaseUser(mariaDbUser); testDbConfig.setDatabaseUser(mariaDbUser);
testDbConfig.setUseSSL(false); testDbConfig.setUseSSL(false);
testTable = mariaDbTestTable; testTable = mariaDbTestTable;
//DBExtensionTestUtils.initTestData(testDbConfig); // DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance());
} }
@Test @Test
public void testGetDatabaseUrl() { public void testGetDatabaseUrl() {
MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService)DatabaseService.get(MariaDBDatabaseService.DB_NAME); MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME);
String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig); String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig);
//System.out.println("dbUrl:" + dbUrl); // System.out.println("dbUrl:" + dbUrl);
Assert.assertNotNull(dbUrl); Assert.assertNotNull(dbUrl);
Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig));
} }
@Test @Test
public void testGetConnection() throws DatabaseServiceException { public void testGetConnection() throws DatabaseServiceException {
MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService)DatabaseService.get(MariaDBDatabaseService.DB_NAME); MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME);
Connection conn = pgSqlService.getConnection(testDbConfig); Connection conn = pgSqlService.getConnection(testDbConfig);
Assert.assertNotNull(conn);
}
@Test Assert.assertNotNull(conn);
public void testTestConnection() throws DatabaseServiceException { }
MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService)DatabaseService.get(MariaDBDatabaseService.DB_NAME);
boolean result = pgSqlService.testConnection(testDbConfig);
Assert.assertEquals(result, true);
}
@Test @Test
public void testConnect() throws DatabaseServiceException { public void testTestConnection() throws DatabaseServiceException {
MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService)DatabaseService.get(MariaDBDatabaseService.DB_NAME); MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME);
boolean result = pgSqlService.testConnection(testDbConfig);
Assert.assertEquals(result, true);
}
@Test
public void testConnect() throws DatabaseServiceException {
MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME);
DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig); DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig);
Assert.assertNotNull(databaseInfo); Assert.assertNotNull(databaseInfo);
} }
@Test
public void testExecuteQuery() throws DatabaseServiceException {
MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService
.get(MariaDBDatabaseService.DB_NAME);
DatabaseInfo databaseInfo = pgSqlService.testQuery(testDbConfig, "SELECT * FROM " + testTable);
Assert.assertNotNull(databaseInfo); @Test
} public void testExecuteQuery() throws DatabaseServiceException {
MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService
.get(MariaDBDatabaseService.DB_NAME);
DatabaseInfo databaseInfo = pgSqlService.testQuery(testDbConfig, "SELECT * FROM " + testTable);
@Test Assert.assertNotNull(databaseInfo);
public void testBuildLimitQuery() { }
MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME);
String limitQuery = pgSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable);
Assert.assertNotNull(limitQuery);
Assert.assertEquals(limitQuery, "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";");
}
@Test @Test
public void testGetRows() throws DatabaseServiceException { public void testBuildLimitQuery() {
MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME);
.get(MariaDBDatabaseService.DB_NAME); String limitQuery = pgSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable);
List<DatabaseRow> dbRows = pgSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable); Assert.assertNotNull(limitQuery);
Assert.assertEquals(limitQuery, "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";");
}
Assert.assertNotNull(dbRows); @Test
} public void testGetRows() throws DatabaseServiceException {
MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService
.get(MariaDBDatabaseService.DB_NAME);
List<DatabaseRow> dbRows = pgSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable);
@Test Assert.assertNotNull(dbRows);
public void testGetInstance() { }
MariaDBDatabaseService instance = MariaDBDatabaseService.getInstance();
Assert.assertNotNull(instance);
}
@Test @Test
public void testGetColumns() throws DatabaseServiceException { public void testGetInstance() {
List<DatabaseColumn> dbColumns; MariaDBDatabaseService instance = MariaDBDatabaseService.getInstance();
Assert.assertNotNull(instance);
}
MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService @Test
.get(MariaDBDatabaseService.DB_NAME); public void testGetColumns() throws DatabaseServiceException {
List<DatabaseColumn> dbColumns;
dbColumns = pgSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable); MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService
.get(MariaDBDatabaseService.DB_NAME);
Assert.assertNotNull(dbColumns); dbColumns = pgSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable);
}
Assert.assertNotNull(dbColumns);
}
} }

53
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mysql/MySQLConnectionManagerTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database.mysql; package com.google.refine.extension.database.mysql;
import java.sql.Connection; import java.sql.Connection;
@ -17,16 +18,15 @@ import com.google.refine.extension.database.DatabaseServiceException;
@Test(groups = { "requiresMySQL" }) @Test(groups = { "requiresMySQL" })
public class MySQLConnectionManagerTest extends DBExtensionTests { public class MySQLConnectionManagerTest extends DBExtensionTests {
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
@BeforeTest @BeforeTest
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" })
public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration(); testDbConfig = new DatabaseConfiguration();
@ -37,40 +37,39 @@ public class MySQLConnectionManagerTest extends DBExtensionTests {
testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME);
testDbConfig.setDatabaseUser(mySqlDbUser); testDbConfig.setDatabaseUser(mySqlDbUser);
testDbConfig.setUseSSL(false); testDbConfig.setUseSSL(false);
//testTable = mySqlTestTable; // testTable = mySqlTestTable;
// DBExtensionTestUtils.initTestData(testDbConfig); // DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
} }
@Test @Test
public void testTestConnection() throws DatabaseServiceException { public void testTestConnection() throws DatabaseServiceException {
boolean conn = MySQLConnectionManager.getInstance().testConnection(testDbConfig); boolean conn = MySQLConnectionManager.getInstance().testConnection(testDbConfig);
Assert.assertEquals(conn, true); Assert.assertEquals(conn, true);
} }
@Test @Test
public void testGetConnection() throws DatabaseServiceException { public void testGetConnection() throws DatabaseServiceException {
Connection conn = MySQLConnectionManager.getInstance().getConnection(testDbConfig, true); Connection conn = MySQLConnectionManager.getInstance().getConnection(testDbConfig, true);
Assert.assertNotNull(conn); Assert.assertNotNull(conn);
} }
@Test @Test
public void testShutdown() throws DatabaseServiceException, SQLException { public void testShutdown() throws DatabaseServiceException, SQLException {
Connection conn = MySQLConnectionManager.getInstance().getConnection(testDbConfig, true); Connection conn = MySQLConnectionManager.getInstance().getConnection(testDbConfig, true);
Assert.assertNotNull(conn); Assert.assertNotNull(conn);
MySQLConnectionManager.getInstance().shutdown(); MySQLConnectionManager.getInstance().shutdown();
if(conn != null) { if (conn != null) {
Assert.assertEquals(conn.isClosed(), true); Assert.assertEquals(conn.isClosed(), true);
} }
} }
} }

138
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/mysql/MySQLDatabaseServiceTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database.mysql; package com.google.refine.extension.database.mysql;
import java.sql.Connection; import java.sql.Connection;
@ -19,19 +20,18 @@ import com.google.refine.extension.database.model.DatabaseColumn;
import com.google.refine.extension.database.model.DatabaseInfo; import com.google.refine.extension.database.model.DatabaseInfo;
import com.google.refine.extension.database.model.DatabaseRow; import com.google.refine.extension.database.model.DatabaseRow;
@Test(groups = { "requiresMySQL" }) @Test(groups = { "requiresMySQL" })
public class MySQLDatabaseServiceTest extends DBExtensionTests{ public class MySQLDatabaseServiceTest extends DBExtensionTests {
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
private String testTable; private String testTable;
@BeforeTest @BeforeTest
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable" })
public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration(); testDbConfig = new DatabaseConfiguration();
@ -42,92 +42,88 @@ public class MySQLDatabaseServiceTest extends DBExtensionTests{
testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME);
testDbConfig.setDatabaseUser(mySqlDbUser); testDbConfig.setDatabaseUser(mySqlDbUser);
testDbConfig.setUseSSL(false); testDbConfig.setUseSSL(false);
testTable = mySqlTestTable; testTable = mySqlTestTable;
//DBExtensionTestUtils.initTestData(testDbConfig); // DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
} }
@Test @Test
public void testGetDatabaseUrl() { public void testGetDatabaseUrl() {
MySQLDatabaseService pgSqlService = (MySQLDatabaseService)DatabaseService.get(MySQLDatabaseService.DB_NAME); MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME);
String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig); String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig);
//System.out.println("dbUrl:" + dbUrl); // System.out.println("dbUrl:" + dbUrl);
Assert.assertNotNull(dbUrl); Assert.assertNotNull(dbUrl);
Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig));
} }
@Test @Test
public void testGetConnection() throws DatabaseServiceException { public void testGetConnection() throws DatabaseServiceException {
MySQLDatabaseService pgSqlService = (MySQLDatabaseService)DatabaseService.get(MySQLDatabaseService.DB_NAME); MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME);
Connection conn = pgSqlService.getConnection(testDbConfig); Connection conn = pgSqlService.getConnection(testDbConfig);
Assert.assertNotNull(conn);
}
@Test Assert.assertNotNull(conn);
public void testTestConnection() throws DatabaseServiceException { }
MySQLDatabaseService pgSqlService = (MySQLDatabaseService)DatabaseService.get(MySQLDatabaseService.DB_NAME);
boolean result = pgSqlService.testConnection(testDbConfig);
Assert.assertEquals(result, true);
}
@Test @Test
public void testConnect() throws DatabaseServiceException { public void testTestConnection() throws DatabaseServiceException {
MySQLDatabaseService pgSqlService = (MySQLDatabaseService)DatabaseService.get(MySQLDatabaseService.DB_NAME); MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME);
boolean result = pgSqlService.testConnection(testDbConfig);
Assert.assertEquals(result, true);
}
@Test
public void testConnect() throws DatabaseServiceException {
MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME);
DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig); DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig);
Assert.assertNotNull(databaseInfo); Assert.assertNotNull(databaseInfo);
} }
@Test
public void testExecuteQuery() throws DatabaseServiceException {
MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService @Test
.get(MySQLDatabaseService.DB_NAME); public void testExecuteQuery() throws DatabaseServiceException {
DatabaseInfo databaseInfo = pgSqlService.testQuery(testDbConfig, "SELECT * FROM " + testTable);
Assert.assertNotNull(databaseInfo); MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService
} .get(MySQLDatabaseService.DB_NAME);
DatabaseInfo databaseInfo = pgSqlService.testQuery(testDbConfig, "SELECT * FROM " + testTable);
@Test Assert.assertNotNull(databaseInfo);
public void testBuildLimitQuery() { }
MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME);
String limitQuery = pgSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable);
Assert.assertNotNull(limitQuery);
Assert.assertEquals(limitQuery, "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";");
}
@Test @Test
public void testGetRows() throws DatabaseServiceException { public void testBuildLimitQuery() {
MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME);
.get(MySQLDatabaseService.DB_NAME); String limitQuery = pgSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable);
List<DatabaseRow> dbRows = pgSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable); Assert.assertNotNull(limitQuery);
Assert.assertEquals(limitQuery, "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";");
}
Assert.assertNotNull(dbRows); @Test
} public void testGetRows() throws DatabaseServiceException {
MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService
.get(MySQLDatabaseService.DB_NAME);
List<DatabaseRow> dbRows = pgSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable);
@Test Assert.assertNotNull(dbRows);
public void testGetInstance() { }
MySQLDatabaseService instance = MySQLDatabaseService.getInstance();
Assert.assertNotNull(instance);
}
@Test @Test
public void testGetColumns() throws DatabaseServiceException { public void testGetInstance() {
List<DatabaseColumn> dbColumns; MySQLDatabaseService instance = MySQLDatabaseService.getInstance();
MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService Assert.assertNotNull(instance);
.get(MySQLDatabaseService.DB_NAME); }
dbColumns = pgSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable); @Test
public void testGetColumns() throws DatabaseServiceException {
List<DatabaseColumn> dbColumns;
MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService
.get(MySQLDatabaseService.DB_NAME);
Assert.assertNotNull(dbColumns); dbColumns = pgSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable);
}
Assert.assertNotNull(dbColumns);
}
} }

63
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/pgsql/PgSQLConnectionManagerTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database.pgsql; package com.google.refine.extension.database.pgsql;
import java.sql.Connection; import java.sql.Connection;
@ -15,21 +16,19 @@ import com.google.refine.extension.database.DatabaseConfiguration;
import com.google.refine.extension.database.DatabaseService; import com.google.refine.extension.database.DatabaseService;
import com.google.refine.extension.database.DatabaseServiceException; import com.google.refine.extension.database.DatabaseServiceException;
@Test(groups = { "requiresPgSQL" }) @Test(groups = { "requiresPgSQL" })
public class PgSQLConnectionManagerTest extends DBExtensionTests { public class PgSQLConnectionManagerTest extends DBExtensionTests {
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
@BeforeTest @BeforeTest
@Parameters({ "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable"}) @Parameters({ "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable" })
public void beforeTest(@Optional(DEFAULT_PGSQL_DB_NAME) String pgSqlDbName, @Optional(DEFAULT_PGSQL_HOST) String pgSqlDbHost, public void beforeTest(@Optional(DEFAULT_PGSQL_DB_NAME) String pgSqlDbName, @Optional(DEFAULT_PGSQL_HOST) String pgSqlDbHost,
@Optional(DEFAULT_PGSQL_PORT) String pgSqlDbPort, @Optional(DEFAULT_PGSQL_USER) String pgSqlDbUser, @Optional(DEFAULT_PGSQL_PORT) String pgSqlDbPort, @Optional(DEFAULT_PGSQL_USER) String pgSqlDbUser,
@Optional(DEFAULT_PGSQL_PASSWORD) String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String pgSqlTestTable) { @Optional(DEFAULT_PGSQL_PASSWORD) String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String pgSqlTestTable) {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration(); testDbConfig = new DatabaseConfiguration();
testDbConfig.setDatabaseHost(pgSqlDbHost); testDbConfig.setDatabaseHost(pgSqlDbHost);
testDbConfig.setDatabaseName(pgSqlDbName); testDbConfig.setDatabaseName(pgSqlDbName);
@ -38,42 +37,40 @@ public class PgSQLConnectionManagerTest extends DBExtensionTests {
testDbConfig.setDatabaseType(PgSQLDatabaseService.DB_NAME); testDbConfig.setDatabaseType(PgSQLDatabaseService.DB_NAME);
testDbConfig.setDatabaseUser(pgSqlDbUser); testDbConfig.setDatabaseUser(pgSqlDbUser);
testDbConfig.setUseSSL(false); testDbConfig.setUseSSL(false);
//testTable = mySqlTestTable;
//DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance());
}
// testTable = mySqlTestTable;
// DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance());
}
@Test @Test
public void testTestConnection() throws DatabaseServiceException { public void testTestConnection() throws DatabaseServiceException {
boolean isConnected = PgSQLConnectionManager.getInstance().testConnection(testDbConfig); boolean isConnected = PgSQLConnectionManager.getInstance().testConnection(testDbConfig);
Assert.assertEquals(isConnected, true); Assert.assertEquals(isConnected, true);
} }
@Test @Test
public void testGetConnection() throws DatabaseServiceException { public void testGetConnection() throws DatabaseServiceException {
Connection conn = PgSQLConnectionManager.getInstance().getConnection(testDbConfig, true); Connection conn = PgSQLConnectionManager.getInstance().getConnection(testDbConfig, true);
Assert.assertNotNull(conn); Assert.assertNotNull(conn);
} }
@Test @Test
public void testShutdown() throws DatabaseServiceException, SQLException { public void testShutdown() throws DatabaseServiceException, SQLException {
Connection conn = PgSQLConnectionManager.getInstance().getConnection(testDbConfig, true); Connection conn = PgSQLConnectionManager.getInstance().getConnection(testDbConfig, true);
Assert.assertNotNull(conn); Assert.assertNotNull(conn);
PgSQLConnectionManager.getInstance().shutdown(); PgSQLConnectionManager.getInstance().shutdown();
if(conn != null) { if (conn != null) {
Assert.assertEquals(conn.isClosed(), true); Assert.assertEquals(conn.isClosed(), true);
} }
} }
} }

50
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/pgsql/PgSQLDatabaseServiceTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.database.pgsql; package com.google.refine.extension.database.pgsql;
import java.sql.Connection; import java.sql.Connection;
@ -20,18 +21,17 @@ import com.google.refine.extension.database.model.DatabaseInfo;
import com.google.refine.extension.database.model.DatabaseRow; import com.google.refine.extension.database.model.DatabaseRow;
@Test(groups = { "requiresPgSQL" }) @Test(groups = { "requiresPgSQL" })
public class PgSQLDatabaseServiceTest extends DBExtensionTests{ public class PgSQLDatabaseServiceTest extends DBExtensionTests {
private DatabaseConfiguration testDbConfig; private DatabaseConfiguration testDbConfig;
private String testTable; private String testTable;
@BeforeTest @BeforeTest
@Parameters({ "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable"}) @Parameters({ "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable" })
public void beforeTest(@Optional(DEFAULT_PGSQL_DB_NAME) String pgSqlDbName, @Optional(DEFAULT_PGSQL_HOST) String pgSqlDbHost, public void beforeTest(@Optional(DEFAULT_PGSQL_DB_NAME) String pgSqlDbName, @Optional(DEFAULT_PGSQL_HOST) String pgSqlDbHost,
@Optional(DEFAULT_PGSQL_PORT) String pgSqlDbPort, @Optional(DEFAULT_PGSQL_USER) String pgSqlDbUser, @Optional(DEFAULT_PGSQL_PORT) String pgSqlDbPort, @Optional(DEFAULT_PGSQL_USER) String pgSqlDbUser,
@Optional(DEFAULT_PGSQL_PASSWORD) String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String pgSqlTestTable) { @Optional(DEFAULT_PGSQL_PASSWORD) String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String pgSqlTestTable) {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration(); testDbConfig = new DatabaseConfiguration();
testDbConfig.setDatabaseHost(pgSqlDbHost); testDbConfig.setDatabaseHost(pgSqlDbHost);
@ -41,38 +41,35 @@ public class PgSQLDatabaseServiceTest extends DBExtensionTests{
testDbConfig.setDatabaseType(PgSQLDatabaseService.DB_NAME); testDbConfig.setDatabaseType(PgSQLDatabaseService.DB_NAME);
testDbConfig.setDatabaseUser(pgSqlDbUser); testDbConfig.setDatabaseUser(pgSqlDbUser);
testDbConfig.setUseSSL(false); testDbConfig.setUseSSL(false);
testTable = pgSqlTestTable; testTable = pgSqlTestTable;
//DBExtensionTestUtils.initTestData(testDbConfig); // DBExtensionTestUtils.initTestData(testDbConfig);
DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance());
} }
@Test @Test
public void testGetDatabaseUrl() { public void testGetDatabaseUrl() {
PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService)DatabaseService.get(PgSQLDatabaseService.DB_NAME); PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService.get(PgSQLDatabaseService.DB_NAME);
String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig); String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig);
Assert.assertNotNull(dbUrl); Assert.assertNotNull(dbUrl);
Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig));
} }
@Test @Test
public void testGetConnection() throws DatabaseServiceException { public void testGetConnection() throws DatabaseServiceException {
PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService)DatabaseService.get(PgSQLDatabaseService.DB_NAME); PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService.get(PgSQLDatabaseService.DB_NAME);
Connection conn = pgSqlService.getConnection(testDbConfig); Connection conn = pgSqlService.getConnection(testDbConfig);
Assert.assertNotNull(conn); Assert.assertNotNull(conn);
} }
@Test @Test
public void testTestConnection() throws DatabaseServiceException { public void testTestConnection() throws DatabaseServiceException {
PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService)DatabaseService.get(PgSQLDatabaseService.DB_NAME); PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService.get(PgSQLDatabaseService.DB_NAME);
boolean result = pgSqlService.testConnection(testDbConfig); boolean result = pgSqlService.testConnection(testDbConfig);
Assert.assertEquals(result, true); Assert.assertEquals(result, true);
} }
@ -80,11 +77,11 @@ public class PgSQLDatabaseServiceTest extends DBExtensionTests{
@Test @Test
public void testConnect() throws DatabaseServiceException { public void testConnect() throws DatabaseServiceException {
PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService)DatabaseService.get(PgSQLDatabaseService.DB_NAME); PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService.get(PgSQLDatabaseService.DB_NAME);
DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig); DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig);
Assert.assertNotNull(databaseInfo); Assert.assertNotNull(databaseInfo);
} }
@Test @Test
public void testExecuteQuery() throws DatabaseServiceException { public void testExecuteQuery() throws DatabaseServiceException {
@ -131,5 +128,4 @@ public class PgSQLDatabaseServiceTest extends DBExtensionTests{
} }
} }

4
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManagerTest.java
View File

@ -49,8 +49,8 @@ public class SQLiteConnectionManagerTest extends DBExtensionTests {
@BeforeTest @BeforeTest
@Parameters({ "sqliteDbName", "sqliteTestTable" }) @Parameters({ "sqliteDbName", "sqliteTestTable" })
public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName,
@Optional(DEFAULT_TEST_TABLE) String sqliteTestTable) @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable)
throws DatabaseServiceException, SQLException { throws DatabaseServiceException, SQLException {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);

22
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseServiceTest.java
View File

@ -51,8 +51,8 @@ public class SQLiteDatabaseServiceTest extends DBExtensionTests {
@BeforeTest @BeforeTest
@Parameters({ "sqliteDbName", "sqliteTestTable" }) @Parameters({ "sqliteDbName", "sqliteTestTable" })
public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName,
@Optional(DEFAULT_TEST_TABLE) String sqliteTestTable) @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable)
throws DatabaseServiceException, SQLException { throws DatabaseServiceException, SQLException {
MockitoAnnotations.initMocks(this); MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration(); testDbConfig = new DatabaseConfiguration();
@ -77,7 +77,7 @@ public class SQLiteDatabaseServiceTest extends DBExtensionTests {
@Test @Test
public void testGetDatabaseUrl() { public void testGetDatabaseUrl() {
SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME); .get(SQLiteDatabaseService.DB_NAME);
String dbUrl = sqLiteDatabaseService.getDatabaseUrl(testDbConfig); String dbUrl = sqLiteDatabaseService.getDatabaseUrl(testDbConfig);
Assert.assertNotNull(dbUrl); Assert.assertNotNull(dbUrl);
@ -88,7 +88,7 @@ public class SQLiteDatabaseServiceTest extends DBExtensionTests {
public void testGetConnection() throws DatabaseServiceException { public void testGetConnection() throws DatabaseServiceException {
SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME); .get(SQLiteDatabaseService.DB_NAME);
Connection conn = sqLiteDatabaseService.getConnection(testDbConfig); Connection conn = sqLiteDatabaseService.getConnection(testDbConfig);
Assert.assertNotNull(conn); Assert.assertNotNull(conn);
@ -97,7 +97,7 @@ public class SQLiteDatabaseServiceTest extends DBExtensionTests {
@Test @Test
public void testTestConnection() throws DatabaseServiceException { public void testTestConnection() throws DatabaseServiceException {
SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME); .get(SQLiteDatabaseService.DB_NAME);
boolean result = sqLiteDatabaseService.testConnection(testDbConfig); boolean result = sqLiteDatabaseService.testConnection(testDbConfig);
Assert.assertTrue(result); Assert.assertTrue(result);
@ -107,7 +107,7 @@ public class SQLiteDatabaseServiceTest extends DBExtensionTests {
public void testConnect() throws DatabaseServiceException { public void testConnect() throws DatabaseServiceException {
SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME); .get(SQLiteDatabaseService.DB_NAME);
DatabaseInfo databaseInfo = sqLiteDatabaseService.connect(testDbConfig); DatabaseInfo databaseInfo = sqLiteDatabaseService.connect(testDbConfig);
Assert.assertNotNull(databaseInfo); Assert.assertNotNull(databaseInfo);
} }
@ -115,7 +115,7 @@ public class SQLiteDatabaseServiceTest extends DBExtensionTests {
@Test @Test
public void testExecuteQuery() throws DatabaseServiceException { public void testExecuteQuery() throws DatabaseServiceException {
SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME); .get(SQLiteDatabaseService.DB_NAME);
DatabaseInfo databaseInfo = sqLiteDatabaseService.testQuery(testDbConfig, "SELECT * FROM " + testTable); DatabaseInfo databaseInfo = sqLiteDatabaseService.testQuery(testDbConfig, "SELECT * FROM " + testTable);
Assert.assertNotNull(databaseInfo); Assert.assertNotNull(databaseInfo);
@ -124,17 +124,17 @@ public class SQLiteDatabaseServiceTest extends DBExtensionTests {
@Test @Test
public void testBuildLimitQuery() { public void testBuildLimitQuery() {
SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME); .get(SQLiteDatabaseService.DB_NAME);
String limitQuery = sqliteSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable); String limitQuery = sqliteSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable);
Assert.assertNotNull(limitQuery); Assert.assertNotNull(limitQuery);
Assert.assertEquals(limitQuery, Assert.assertEquals(limitQuery,
"SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";"); "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";");
} }
@Test @Test
public void testGetRows() throws DatabaseServiceException { public void testGetRows() throws DatabaseServiceException {
SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME); .get(SQLiteDatabaseService.DB_NAME);
List<DatabaseRow> dbRows = sqliteSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable); List<DatabaseRow> dbRows = sqliteSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable);
Assert.assertNotNull(dbRows); Assert.assertNotNull(dbRows);
@ -151,7 +151,7 @@ public class SQLiteDatabaseServiceTest extends DBExtensionTests {
List<DatabaseColumn> dbColumns; List<DatabaseColumn> dbColumns;
SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME); .get(SQLiteDatabaseService.DB_NAME);
dbColumns = sqliteSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable); dbColumns = sqliteSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable);

22
OpenRefine/extensions/database/tests/src/com/google/refine/extension/database/stub/RefineDbServletStub.java
View File

@ -50,18 +50,18 @@ import com.google.refine.commands.Command;
public class RefineDbServletStub extends RefineServlet { public class RefineDbServletStub extends RefineServlet {
private static File tempDir = null; private static File tempDir = null;
//requirement of extending HttpServlet, not required for testing // requirement of extending HttpServlet, not required for testing
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public void wrapService(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException{ public void wrapService(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
super.service(request, response); super.service(request, response);
} }
public String wrapGetCommandName(HttpServletRequest request){ public String wrapGetCommandName(HttpServletRequest request) {
return super.getCommandKey(request); return super.getCommandKey(request);
} }
@Override @Override
public File getTempDir() { public File getTempDir() {
if (tempDir == null) { if (tempDir == null) {
@ -69,27 +69,29 @@ public class RefineDbServletStub extends RefineServlet {
tempDir = File.createTempFile("refine-test-dir", ""); tempDir = File.createTempFile("refine-test-dir", "");
tempDir.deleteOnExit(); tempDir.deleteOnExit();
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException("Failed to create temp directory",e); throw new RuntimeException("Failed to create temp directory", e);
} }
} }
return tempDir; return tempDir;
} }
//-------------------helper methods-------------- // -------------------helper methods--------------
/** /**
* Helper method for inserting a mock object * Helper method for inserting a mock object
*
* @param commandName * @param commandName
* @param command * @param command
*/ */
public void insertCommand(String commandName, Command command ){ public void insertCommand(String commandName, Command command) {
registerOneCommand("core/" + commandName, command); registerOneCommand("core/" + commandName, command);
} }
/** /**
* Helper method for clearing up after testing * Helper method for clearing up after testing
*
* @param commandName * @param commandName
*/ */
public void removeCommand( String commandName ){ public void removeCommand(String commandName) {
unregisterCommand("core/" + commandName); unregisterCommand("core/" + commandName);
} }
} }

2
OpenRefine/extensions/gdata/pom.xml
View File

@ -80,7 +80,7 @@
<dependency> <dependency>
<groupId>com.google.apis</groupId> <groupId>com.google.apis</groupId>
<artifactId>google-api-services-drive</artifactId> <artifactId>google-api-services-drive</artifactId>
<version>v3-rev20211107-1.32.1</version> <version>v3-rev20220110-1.32.1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.apis</groupId> <groupId>com.google.apis</groupId>

39
OpenRefine/extensions/gdata/tests/src/com/google/refine/extension/gdata/GoogleAPIExtensionTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.gdata; package com.google.refine.extension.gdata;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
@ -8,30 +9,30 @@ import static org.testng.Assert.*;
public class GoogleAPIExtensionTest { public class GoogleAPIExtensionTest {
@Test @Test
public void extractSpreadSheetIdTest() { public void extractSpreadSheetIdTest() {
// GoogleAPIExtension will call ProjectManager.singleton.getPreferenceStore() during class initialization, // GoogleAPIExtension will call ProjectManager.singleton.getPreferenceStore() during class initialization,
// which will cause NullPointerException if this line is omitted. // which will cause NullPointerException if this line is omitted.
ProjectManager.singleton = mock(ProjectManager.class); ProjectManager.singleton = mock(ProjectManager.class);
String spreadSheetId = "16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0"; String spreadSheetId = "16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0";
String url1 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0/edit#gid=0"; String url1 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0/edit#gid=0";
assertEquals(GoogleAPIExtension.extractSpreadSheetId(url1), spreadSheetId); assertEquals(GoogleAPIExtension.extractSpreadSheetId(url1), spreadSheetId);
String url2 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0/"; String url2 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0/";
assertEquals(GoogleAPIExtension.extractSpreadSheetId(url2), spreadSheetId); assertEquals(GoogleAPIExtension.extractSpreadSheetId(url2), spreadSheetId);
String url3 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0?foo=bar"; String url3 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0?foo=bar";
assertEquals(GoogleAPIExtension.extractSpreadSheetId(url3), spreadSheetId); assertEquals(GoogleAPIExtension.extractSpreadSheetId(url3), spreadSheetId);
String url4 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0/?foo=bar"; String url4 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0/?foo=bar";
assertEquals(GoogleAPIExtension.extractSpreadSheetId(url4), spreadSheetId); assertEquals(GoogleAPIExtension.extractSpreadSheetId(url4), spreadSheetId);
String url5 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0#foo"; String url5 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0#foo";
assertEquals(GoogleAPIExtension.extractSpreadSheetId(url5), spreadSheetId); assertEquals(GoogleAPIExtension.extractSpreadSheetId(url5), spreadSheetId);
String url6 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0"; String url6 = "https://docs.google.com/spreadsheets/d/16L0JfpBWPfBJTqKtm-YU5-UBWLpkwXII-IRLMLnoKw0";
assertEquals(GoogleAPIExtension.extractSpreadSheetId(url6), spreadSheetId); assertEquals(GoogleAPIExtension.extractSpreadSheetId(url6), spreadSheetId);
} }
} }

1
OpenRefine/extensions/gdata/tests/src/com/google/refine/extension/gdata/SpreadsheetSerializerTests.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.gdata; package com.google.refine.extension.gdata;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;

37
OpenRefine/extensions/gdata/tests/src/com/google/refine/extension/gdata/UploadCommandTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.extension.gdata; package com.google.refine.extension.gdata;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
@ -20,7 +21,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.commands.Command; import com.google.refine.commands.Command;
import com.google.refine.util.ParsingUtilities; import com.google.refine.util.ParsingUtilities;
class UploadCommandStub extends UploadCommand { class UploadCommandStub extends UploadCommand {
public byte[] getIcon() throws IOException { public byte[] getIcon() throws IOException {
@ -29,33 +29,34 @@ class UploadCommandStub extends UploadCommand {
} }
public class UploadCommandTest { public class UploadCommandTest {
protected HttpServletRequest request = null; protected HttpServletRequest request = null;
protected HttpServletResponse response = null; protected HttpServletResponse response = null;
protected Command command = null; protected Command command = null;
protected StringWriter writer = null; protected StringWriter writer = null;
@BeforeMethod
@BeforeMethod public void setUpRequestResponse() {
public void setUpRequestResponse() { request = mock(HttpServletRequest.class);
request = mock(HttpServletRequest.class); response = mock(HttpServletResponse.class);
response = mock(HttpServletResponse.class); writer = new StringWriter();
writer = new StringWriter(); command = new UploadCommand();
command = new UploadCommand(); try {
try {
when(response.getWriter()).thenReturn(new PrintWriter(writer)); when(response.getWriter()).thenReturn(new PrintWriter(writer));
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
@Test
public void testCsrfProtection() throws ServletException, IOException {
command.doPost(request, response);
Assert.assertEquals(
ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", ObjectNode.class),
ParsingUtilities.mapper.readValue(writer.toString(), ObjectNode.class));
} @Test
public void testCsrfProtection() throws ServletException, IOException {
command.doPost(request, response);
Assert.assertEquals(
ParsingUtilities.mapper.readValue("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}",
ObjectNode.class),
ParsingUtilities.mapper.readValue(writer.toString(), ObjectNode.class));
}
@Test @Test
public void testIconRead() throws IOException { public void testIconRead() throws IOException {

15
OpenRefine/extensions/jython/tests/src/com/google/refine/jython/JythonAttributeTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.jython; package com.google.refine.jython;
import java.util.Properties; import java.util.Properties;
@ -11,7 +12,7 @@ import com.google.refine.model.Cell;
import com.google.refine.model.Row; import com.google.refine.model.Row;
public class JythonAttributeTest { public class JythonAttributeTest {
class MyFieldObject implements HasFields { class MyFieldObject implements HasFields {
@Override @Override
@ -26,25 +27,25 @@ public class JythonAttributeTest {
public boolean fieldAlsoHasFields(String name) { public boolean fieldAlsoHasFields(String name) {
return true; return true;
} }
} }
@Test @Test
public void testWrappedObjectsHaveAttributes() { public void testWrappedObjectsHaveAttributes() {
Row row = new Row(2); Row row = new Row(2);
row.setCell(0, new Cell("sunshine", null)); row.setCell(0, new Cell("sunshine", null));
row.setCell(1, new Cell("hammock", null)); row.setCell(1, new Cell("hammock", null));
Properties props = new Properties(); Properties props = new Properties();
MyFieldObject obj = new MyFieldObject(); MyFieldObject obj = new MyFieldObject();
JythonHasFieldsWrapper wrapper = new JythonHasFieldsWrapper(obj, props); JythonHasFieldsWrapper wrapper = new JythonHasFieldsWrapper(obj, props);
Assert.assertEquals(wrapper.__findattr__("sunshine").toString(), "hammock"); Assert.assertEquals(wrapper.__findattr__("sunshine").toString(), "hammock");
props.put("cell", obj); props.put("cell", obj);
Evaluable eval = new JythonEvaluable("return cell.sunshine"); Evaluable eval = new JythonEvaluable("return cell.sunshine");
String result = (String)eval.evaluate(props).toString(); String result = (String) eval.evaluate(props).toString();
Assert.assertEquals(result, "hammock"); Assert.assertEquals(result, "hammock");
} }
} }

101
OpenRefine/extensions/jython/tests/src/com/google/refine/jython/JythonEvaluableTest.java
View File

@ -1,3 +1,4 @@
package com.google.refine.jython; package com.google.refine.jython;
import java.io.File; import java.io.File;
@ -21,62 +22,62 @@ import com.google.refine.model.Row;
*/ */
public class JythonEvaluableTest { public class JythonEvaluableTest {
// Reproduces the situation when result is a PyObject // Reproduces the situation when result is a PyObject
// Version with a test case which only calls the existing evaluate method // Version with a test case which only calls the existing evaluate method
@Test @Test
public void unwrapPyObjectTest(){ public void unwrapPyObjectTest() {
Properties props = new Properties(); Properties props = new Properties();
Project project = new Project(); Project project = new Project();
Row row = new Row(2); Row row = new Row(2);
row.setCell(0, new Cell("one", null)); row.setCell(0, new Cell("one", null));
row.setCell(0, new Cell("1", null)); row.setCell(0, new Cell("1", null));
props.put("columnName", "number"); props.put("columnName", "number");
props.put("true", "true"); props.put("true", "true");
props.put("false", "false"); props.put("false", "false");
props.put("rowIndex", "0"); props.put("rowIndex", "0");
props.put("value", 1); props.put("value", 1);
props.put("project", project); props.put("project", project);
props.put("call", "number"); props.put("call", "number");
props.put("PI", "3.141592654"); props.put("PI", "3.141592654");
props.put("cells", new CellTuple(project, row)); props.put("cells", new CellTuple(project, row));
String funcExpression ="class Foo(object):\n" + String funcExpression = "class Foo(object):\n" +
" bar = 1\n" + " bar = 1\n" +
"\n" + "\n" +
"return Foo()"; "return Foo()";
JythonEvaluable eval1 = new JythonEvaluable(funcExpression); JythonEvaluable eval1 = new JythonEvaluable(funcExpression);
PyObject po = (PyObject) eval1.evaluate(props); PyObject po = (PyObject) eval1.evaluate(props);
Assert.assertEquals(po.__getattr__("bar").toString(),"1"); Assert.assertEquals(po.__getattr__("bar").toString(), "1");
} }
@Test @Test
public void testJythonConcurrent(){ public void testJythonConcurrent() {
Properties props = new Properties(); Properties props = new Properties();
Project project = new Project(); Project project = new Project();
Row row = new Row(2); Row row = new Row(2);
row.setCell(0, new Cell("one", null)); row.setCell(0, new Cell("one", null));
row.setCell(0, new Cell("1", null)); row.setCell(0, new Cell("1", null));
props.put("columnName", "number"); props.put("columnName", "number");
props.put("true", "true"); props.put("true", "true");
props.put("false", "false"); props.put("false", "false");
props.put("rowIndex", "0"); props.put("rowIndex", "0");
props.put("value", 1); props.put("value", 1);
props.put("project", project); props.put("project", project);
props.put("call", "number"); props.put("call", "number");
props.put("PI", "3.141592654"); props.put("PI", "3.141592654");
props.put("cells", new CellTuple(project, row)); props.put("cells", new CellTuple(project, row));
Evaluable eval1 = new JythonEvaluable("a = value\nreturn a * 2"); Evaluable eval1 = new JythonEvaluable("a = value\nreturn a * 2");
Long value1 = (Long) eval1.evaluate(props); Long value1 = (Long) eval1.evaluate(props);
// create some unrelated evaluable // create some unrelated evaluable
new JythonEvaluable("a = value\nreturn a * 10"); new JythonEvaluable("a = value\nreturn a * 10");
// repeat same previous test // repeat same previous test
Long value2 = (Long) eval1.evaluate(props); Long value2 = (Long) eval1.evaluate(props);
Assert.assertEquals(value1, value2); Assert.assertEquals(value1, value2);
} }
} }

11
OpenRefine/extensions/phonetic/tests/src/org/openrefine/phonetic/keyers/BeiderMorseKeyerTest.java
View File

@ -1,3 +1,4 @@
package org.openrefine.phonetic.keyers; package org.openrefine.phonetic.keyers;
import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertEquals;
@ -6,20 +7,20 @@ import org.testng.annotations.Test;
import com.google.refine.clustering.binning.Keyer; import com.google.refine.clustering.binning.Keyer;
public class BeiderMorseKeyerTest { public class BeiderMorseKeyerTest {
Keyer keyer = new BeiderMorseKeyer(); Keyer keyer = new BeiderMorseKeyer();
@Test @Test
public void testKey() { public void testKey() {
assertEquals(keyer.key("Alphonse"), "YlfYnzi"); assertEquals(keyer.key("Alphonse"), "YlfYnzi");
} }
@Test @Test
public void testAccents() { public void testAccents() {
assertEquals(keyer.key("Éléonore"), "ilionor"); assertEquals(keyer.key("Éléonore"), "ilionor");
} }
@Test @Test
public void testEmpty() { public void testEmpty() {
assertEquals(keyer.key(""), ""); assertEquals(keyer.key(""), "");

9
OpenRefine/extensions/phonetic/tests/src/org/openrefine/phonetic/keyers/DaitchMokotoffKeyerTest.java
View File

@ -1,3 +1,4 @@
package org.openrefine.phonetic.keyers; package org.openrefine.phonetic.keyers;
import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertEquals;
@ -7,19 +8,19 @@ import org.testng.annotations.Test;
import com.google.refine.clustering.binning.Keyer; import com.google.refine.clustering.binning.Keyer;
public class DaitchMokotoffKeyerTest { public class DaitchMokotoffKeyerTest {
protected Keyer keyer = new DaitchMokotoffKeyer(); protected Keyer keyer = new DaitchMokotoffKeyer();
@Test @Test
public void testDaitchMokotoff() { public void testDaitchMokotoff() {
assertEquals(keyer.key("Alphonse"), "087640"); assertEquals(keyer.key("Alphonse"), "087640");
} }
@Test @Test
public void testAccents() { public void testAccents() {
assertEquals(keyer.key("Éléonore"), "086900"); assertEquals(keyer.key("Éléonore"), "086900");
} }
@Test @Test
public void testEmpty() { public void testEmpty() {
assertEquals(keyer.key(""), "000000"); assertEquals(keyer.key(""), "000000");

2
OpenRefine/extensions/sample/pom.xml
View File

@ -5,7 +5,7 @@
<groupId>org.openrefine</groupId> <groupId>org.openrefine</groupId>
<artifactId>sample</artifactId> <artifactId>sample</artifactId>
<packaging>jar</packaging> <packaging>jar</packaging>
<version>3.0-SNAPSHOT</version> <version>3.6-SNAPSHOT</version>
<name>OpenRefine - Sample extension</name> <name>OpenRefine - Sample extension</name>
<description>Example extension provided for demonstration purposes</description> <description>Example extension provided for demonstration purposes</description>

2
OpenRefine/extensions/wikidata/module/MOD-INF/controller.js
View File

@ -58,12 +58,14 @@ function init() {
"scripts/ajv.min.js", "scripts/ajv.min.js",
"scripts/wikidata-manifest-v1.0.js", "scripts/wikidata-manifest-v1.0.js",
"scripts/wikibase-manifest-schema-v1.js", "scripts/wikibase-manifest-schema-v1.js",
"scripts/wikibase-manifest-schema-v2.js",
"scripts/wikibase-manager.js", "scripts/wikibase-manager.js",
"scripts/menu-bar-extension.js", "scripts/menu-bar-extension.js",
"scripts/warningsrenderer.js", "scripts/warningsrenderer.js",
"scripts/langsuggest.js", "scripts/langsuggest.js",
"scripts/bettersuggest.js", "scripts/bettersuggest.js",
"scripts/previewrenderer.js", "scripts/previewrenderer.js",
"scripts/wikibase-suggest.js",
"scripts/schema-alignment.js", "scripts/schema-alignment.js",
"scripts/wikidata-extension-manager.js", "scripts/wikidata-extension-manager.js",
"scripts/dialogs/manage-account-dialog.js", "scripts/dialogs/manage-account-dialog.js",

7
OpenRefine/extensions/wikidata/module/langs/translation-en.json
View File

@ -27,7 +27,7 @@
"wikibase-addition/cancel": "Cancel", "wikibase-addition/cancel": "Cancel",
"wikibase-addition/invalid-manifest": "Invalid Wikibase manifest.", "wikibase-addition/invalid-manifest": "Invalid Wikibase manifest.",
"wikibase-schema/dialog-header": "Align to Wikibase", "wikibase-schema/dialog-header": "Align to Wikibase",
"wikibase-schema/dialog-explanation": "You are currently working against <a href=\"$1\" target=\"_blank\">$2</a>. You should have your data reconciled to reconciliation services linked to $2 (such as <a href=\"$3\" target=\"_blank\">$3</a>) first. The schema below specifies how your tabular data will be transformed into $2 edits. You can drag and drop the column names below in most input boxes: for each row, edits will be generated with the values in these columns.", "wikibase-schema/dialog-explanation": "You are currently working against <a href=\"$1\" target=\"_blank\">$2</a>. You should have your data reconciled to reconciliation services linked to $2 first. The schema below specifies how your tabular data will be transformed into $2 edits. You can drag and drop the column names below in most input boxes: for each row, edits will be generated with the values in these columns.",
"wikibase-schema/preview-explanation": "This tab shows the first edits (out of $1) that will be made once you upload the changes to <a href=\"$2\" target=\"_blank\">$3</a>. You can use facets to inspect the edits on particular items.", "wikibase-schema/preview-explanation": "This tab shows the first edits (out of $1) that will be made once you upload the changes to <a href=\"$2\" target=\"_blank\">$3</a>. You can use facets to inspect the edits on particular items.",
"wikibase-schema/schema-tab-header": "Schema", "wikibase-schema/schema-tab-header": "Schema",
"wikibase-schema/warnings-tab-header": "Issues", "wikibase-schema/warnings-tab-header": "Issues",
@ -58,7 +58,7 @@
"wikibase-schema/description-override": "Description (override if present)", "wikibase-schema/description-override": "Description (override if present)",
"wikibase-schema/override-term": "override if present", "wikibase-schema/override-term": "override if present",
"wikibase-schema/alias": "Alias", "wikibase-schema/alias": "Alias",
"wikibase-schema/item-or-reconciled-column": "type item or drag reconciled column here", "wikibase-schema/entity-or-reconciled-column": "type entity or drag reconciled column here",
"wikibase-schema/amount": "amount", "wikibase-schema/amount": "amount",
"wikibase-schema/unit": "unit", "wikibase-schema/unit": "unit",
"wikibase-schema/full-url": "full URL including the protocol", "wikibase-schema/full-url": "full URL including the protocol",
@ -77,6 +77,7 @@
"wikibase-schema/discard-schema-changes-alt": "Discard the changes made to the schema.", "wikibase-schema/discard-schema-changes-alt": "Discard the changes made to the schema.",
"wikibase-schema/incomplete-schema-could-not-be-saved": "Your schema is incomplete so it cannot be saved yet.", "wikibase-schema/incomplete-schema-could-not-be-saved": "Your schema is incomplete so it cannot be saved yet.",
"wikibase-schema/unsaved-warning": "You have made unsaved changes to your Wikibase schema. Close anyway?", "wikibase-schema/unsaved-warning": "You have made unsaved changes to your Wikibase schema. Close anyway?",
"wikibase-schema/incompatible-site-iri-reconciled-column": "This column is reconciled to the identifier space $1, but this input requires a column reconciled to $2.",
"wikibase-preview/new-id": "new item", "wikibase-preview/new-id": "new item",
"wikibase-account/dialog-header": "$1 account", "wikibase-account/dialog-header": "$1 account",
"wikibase-account/explain-log-in": "Logging in to <a href=\"$1\" target=\"_blank\">$2</a> lets you to upload edits directly from OpenRefine.", "wikibase-account/explain-log-in": "Logging in to <a href=\"$1\" target=\"_blank\">$2</a> lets you to upload edits directly from OpenRefine.",
@ -192,7 +193,7 @@
"warnings-messages/non-printable-characters/title": "Non-printable characters in strings.", "warnings-messages/non-printable-characters/title": "Non-printable characters in strings.",
"warnings-messages/non-printable-characters/body": "Strings such as <span class=\"wb-issue-preformat\">{example_string}</span> contain non-printable characters.", "warnings-messages/non-printable-characters/body": "Strings such as <span class=\"wb-issue-preformat\">{example_string}</span> contain non-printable characters.",
"warnings-messages/invalid-identifier-space/title": "Invalid identifier space for reconciled cells.", "warnings-messages/invalid-identifier-space/title": "Invalid identifier space for reconciled cells.",
"warnings-messages/invalid-identifier-space/body": "Some reconciled cells such as <span class=\"wb-issue-preformat\">{example_cell}</span> were ignored because they are not reconciled to {wikibase_name}.", "warnings-messages/invalid-identifier-space/body": "Some reconciled cells such as <span class=\"wb-issue-preformat\">{example_cell}</span> were ignored because they are not reconciled to <span class=\"wb-issue-preformat\">{expected_site_iri}</span>.",
"warnings-messages/ignored-language/title": "Invalid language identifiers.", "warnings-messages/ignored-language/title": "Invalid language identifiers.",
"warnings-messages/ignored-language/body": "Some language identifiers are invalid, such as <span class=\"wb-issue-preformat\">{example_value}</span>. See the <a href=\"https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Schema_alignment#Languages\" target=\"_blank\">allowed values</a>.", "warnings-messages/ignored-language/body": "Some language identifiers are invalid, such as <span class=\"wb-issue-preformat\">{example_value}</span>. See the <a href=\"https://www.wikidata.org/wiki/Wikidata:Tools/OpenRefine/Editing/Schema_alignment#Languages\" target=\"_blank\">allowed values</a>.",
"warnings-messages/ignored-date/title": "Invalid date formats.", "warnings-messages/ignored-date/title": "Invalid date formats.",

2
OpenRefine/extensions/wikidata/module/langs/translation-fr.json
View File

@ -6,7 +6,7 @@
"wikibase-extension/export-to-qs": "Exporter vers QuickStatements", "wikibase-extension/export-to-qs": "Exporter vers QuickStatements",
"wikibase-extension/quickstatements-export-name": "QuickStatements", "wikibase-extension/quickstatements-export-name": "QuickStatements",
"wikibase-schema/dialog-header": "Aligner sur la Wikibase", "wikibase-schema/dialog-header": "Aligner sur la Wikibase",
"wikibase-schema/dialog-explanation": "Vous travaillez actuellement contre <a href=\"$1\" target=\"_blank\">$2</a>. D'abord, vous devriez réconcilier vos données avec les services de réconciliation associés à $2 (tels que <a href=\"$3\" target=\"_blank\">$3</a>). Le schéma ci-dessous spécifie comment vos données tabulaires seront transformées en $2 modifications. Vous pouvez glisser-déposer les noms des colonnes ci-dessous dans la plupart des champs de saisie : pour chaque ligne, des modifications seront générées avec les valeurs dans ces colonnes.", "wikibase-schema/dialog-explanation": "Vous travaillez actuellement contre <a href=\"$1\" target=\"_blank\">$2</a>. D'abord, vous devriez réconcilier vos données avec les services de réconciliation associés à $2. Le schéma ci-dessous spécifie comment vos données tabulaires seront transformées en $2 modifications. Vous pouvez glisser-déposer les noms des colonnes ci-dessous dans la plupart des champs de saisie : pour chaque ligne, des modifications seront générées avec les valeurs dans ces colonnes.",
"wikibase-schema/preview-explanation": "Cet onglet affiche les premières modifications (sur $1) qui seront effectuées une fois que vous aurez importé les modifications dans <a href=\"$2\" target=\"_blank\">$3</a>. Vous pouvez utiliser des facettes pour inspecter les modifications sur des éléments particuliers.", "wikibase-schema/preview-explanation": "Cet onglet affiche les premières modifications (sur $1) qui seront effectuées une fois que vous aurez importé les modifications dans <a href=\"$2\" target=\"_blank\">$3</a>. Vous pouvez utiliser des facettes pour inspecter les modifications sur des éléments particuliers.",
"wikibase-schema/schema-tab-header": "Schéma", "wikibase-schema/schema-tab-header": "Schéma",
"wikibase-schema/warnings-tab-header": "Problèmes", "wikibase-schema/warnings-tab-header": "Problèmes",

8
OpenRefine/extensions/wikidata/module/langs/translation-nl.json
View File

@ -85,5 +85,11 @@
"warnings-messages/property-found-in-qualifier/body": "U heeft {property_entity} als kwalificatie gebruikt, maar deze is daar niet voor ontworpen.", "warnings-messages/property-found-in-qualifier/body": "U heeft {property_entity} als kwalificatie gebruikt, maar deze is daar niet voor ontworpen.",
"warnings-messages/property-found-in-reference/title": "{property_entity} gebruikt als bron.", "warnings-messages/property-found-in-reference/title": "{property_entity} gebruikt als bron.",
"warnings-messages/property-found-in-reference/body": "U heeft {property_entity} gebruikt als bron, maar deze is daar niet voor ontworpen.", "warnings-messages/property-found-in-reference/body": "U heeft {property_entity} gebruikt als bron, maar deze is daar niet voor ontworpen.",
"warnings-messages/missing-mandatory-qualifiers/title": "{statement_property_entity} mist een {missing_property_entity}-kwalificatie." "warnings-messages/missing-mandatory-qualifiers/title": "{statement_property_entity} mist een {missing_property_entity}-kwalificatie.",
"wikibase-extension/qs-file": "QuickStatements-bestand",
"wikibase-extension/wikibase-schema": "Wikibase schema",
"wikibase-extension/select-wikibase-instance": "Selecteer Wikibase instance",
"wikibase-extension/wikibase-edits": "Wikibase bewerkingen…",
"wikibase-extension/export-wikibase-schema": "Wikibase schema exporteren",
"wikibase-management/dialog-header": "Selecteer Wikibase instance"
} }

42
OpenRefine/extensions/wikidata/module/scripts/dialogs/wikibase-dialog.js
View File

@ -36,12 +36,12 @@ WikibaseDialog.populateDialog = function () {
if (wikibaseName.toLowerCase() === WikibaseManager.getSelectedWikibaseName().toLowerCase()) { if (wikibaseName.toLowerCase() === WikibaseManager.getSelectedWikibaseName().toLowerCase()) {
item += "<td><a class=\"wikibase-dialog-selector-remove wikibase-selected\" onclick=\"void(0)\"></a></td>"; item += "<td><a class=\"wikibase-dialog-selector-remove wikibase-selected\" onclick=\"void(0)\"></a></td>";
} else { } else {
item += "<td><a class=\"wikibase-dialog-selector-remove\" onclick=\"WikibaseDialog.removeWikibase(event, '" + wikibaseName + "')\"></a></td>"; item += "<td><a class=\"wikibase-dialog-selector-remove\" onclick=\"WikibaseDialog.removeWikibase(event, '" + wikibaseName + "')\"></a></td>";
} }
item += "</tr>"; item += "</tr>";
WikibaseDialog.elmts.wikibaseList.append(item); WikibaseDialog.elmts.wikibaseList.append(item);
} }
}, wikibaseName); }, wikibaseName);
} }
}; };
@ -86,9 +86,14 @@ WikibaseDialog.addWikibaseManifest = function () {
} }
WikibaseManager.addWikibase(manifest); WikibaseManager.addWikibase(manifest);
let lang = $.i18n('core-recon/wd-recon-lang');
let reconEndpoint = manifest.reconciliation.endpoint.replace("${lang}", lang); // pre-register reconciliation services mentioned by this manifest
ReconciliationManager.getOrRegisterServiceFromUrl(reconEndpoint, function () {}, true); for (let reconEndpoint of WikibaseManager.getReconciliationEndpoints(manifest)) {
let lang = $.i18n('core-recon/wd-recon-lang');
let endpoint = reconEndpoint.replace("${lang}", lang);
ReconciliationManager.getOrRegisterServiceFromUrl(endpoint, function () {}, true);
}
DialogSystem.dismissUntil(level - 1); DialogSystem.dismissUntil(level - 1);
WikibaseDialog.populateDialog(); WikibaseDialog.populateDialog();
}; };
@ -113,15 +118,28 @@ WikibaseDialog.validateManifest = function (manifest) {
if (!WikibaseDialog.ajv) { if (!WikibaseDialog.ajv) {
WikibaseDialog.ajv = new Ajv(); WikibaseDialog.ajv = new Ajv();
WikibaseDialog.validateWikibaseManifestV1 = WikibaseDialog.ajv.compile(WikibaseManifestSchemaV1); WikibaseDialog.validateWikibaseManifestV1 = WikibaseDialog.ajv.compile(WikibaseManifestSchemaV1);
WikibaseDialog.validateWikibaseManifestV2 = WikibaseDialog.ajv.compile(WikibaseManifestSchemaV2);
} }
if (WikibaseDialog.validateWikibaseManifestV1(manifest)) { if (manifest.version !== undefined && manifest.version.startsWith('1.')) {
return true; if (WikibaseDialog.validateWikibaseManifestV1(manifest)) {
return true;
} else {
let errMsg = WikibaseDialog.ajv.errorsText(WikibaseDialog.validateWikibaseManifestV1.errors, {
dataVar: "manifest"
});
alert(errMsg);
return false;
}
} else { } else {
let errMsg = WikibaseDialog.ajv.errorsText(WikibaseDialog.validateWikibaseManifestV1.errors, { if (WikibaseDialog.validateWikibaseManifestV2(manifest)) {
dataVar: "manifest" return true;
}); } else {
alert(errMsg); let errMsg = WikibaseDialog.ajv.errorsText(WikibaseDialog.validateWikibaseManifestV2.errors, {
return false; dataVar: "manifest"
});
alert(errMsg);
return false;
}
} }
}; };

177
OpenRefine/extensions/wikidata/module/scripts/schema-alignment.js
View File

@ -121,8 +121,7 @@ SchemaAlignment._rerenderTabs = function() {
var schemaElmts = this._schemaElmts = DOM.bind(schemaTab); var schemaElmts = this._schemaElmts = DOM.bind(schemaTab);
schemaElmts.dialogExplanation.html($.i18n('wikibase-schema/dialog-explanation', schemaElmts.dialogExplanation.html($.i18n('wikibase-schema/dialog-explanation',
WikibaseManager.getSelectedWikibaseMainPage(), WikibaseManager.getSelectedWikibaseMainPage(),
WikibaseManager.getSelectedWikibaseName(), WikibaseManager.getSelectedWikibaseName()));
WikibaseManager.getSelectedWikibaseReconEndpoint().replace("${lang}", "en")));
this._plusButton($.i18n('wikibase-schema/add-item-button'), schemaElmts.addItemButton); this._plusButton($.i18n('wikibase-schema/add-item-button'), schemaElmts.addItemButton);
schemaElmts.addItemButton.click(function(e) { schemaElmts.addItemButton.click(function(e) {
SchemaAlignment._addItem(); SchemaAlignment._addItem();
@ -162,17 +161,24 @@ SchemaAlignment._rerenderTabs = function() {
previewElmts.invalidSchemaWarningPreview.text($.i18n('wikibase-schema/invalid-schema-warning-preview')); previewElmts.invalidSchemaWarningPreview.text($.i18n('wikibase-schema/invalid-schema-warning-preview'));
this._previewPanes = $(".schema-alignment-dialog-preview"); this._previewPanes = $(".schema-alignment-dialog-preview");
var reconServiceURL = WikibaseManager.getSelectedWikibaseReconEndpoint() // add all recon services for all the entity types of the Wikibase instance
.replace("${lang}", $.i18n("core-recon/wd-recon-lang")); var entityTypes = WikibaseManager.getSelectedWikibaseAvailableEntityTypes();
ReconciliationManager.getOrRegisterServiceFromUrl(reconServiceURL, function (service) { SchemaAlignment._reconService = {};
SchemaAlignment._reconService = service; for (let entityType of entityTypes) {
var reconServiceTemplate = WikibaseManager.getSelectedWikibaseReconEndpoint(entityType);
if (reconServiceTemplate != null) {
var reconServiceURL = reconServiceTemplate.replace("${lang}", $.i18n("core-recon/wd-recon-lang"));
ReconciliationManager.getOrRegisterServiceFromUrl(reconServiceURL, function (service) {
SchemaAlignment._reconService[entityType] = service;
}, false);
}
}
// Load the existing schema // Load the existing schema
SchemaAlignment._reset(theProject.overlayModels.wikibaseSchema); SchemaAlignment._reset(theProject.overlayModels.wikibaseSchema);
// Perform initial preview // Perform initial preview
SchemaAlignment.preview(); SchemaAlignment.preview();
}, false);
}; };
SchemaAlignment.onWikibaseChange = function() { SchemaAlignment.onWikibaseChange = function() {
@ -190,9 +196,11 @@ SchemaAlignment.updateColumns = function() {
for (var i = 0; i < columns.length; i++) { for (var i = 0; i < columns.length; i++) {
var column = columns[i]; var column = columns[i];
var reconConfig = column.reconConfig; var reconConfig = column.reconConfig;
// make sure the column was reconciled to the target Wikibase // make sure the column was reconciled.
// TODO we could potentially ignore any reconciliation to a siteIRI not
// mentioned in the manifest…
var cell = SchemaAlignment._createDraggableColumn(column.name, var cell = SchemaAlignment._createDraggableColumn(column.name,
reconConfig && reconConfig.identifierSpace === WikibaseManager.getSelectedWikibaseSiteIri() && column.reconStats); reconConfig && column.reconStats ? reconConfig.identifierSpace : null);
this._columnArea.append(cell); this._columnArea.append(cell);
} }
@ -327,9 +335,13 @@ SchemaAlignment._changesCleared = function() {
.addClass('disabled'); .addClass('disabled');
}; };
SchemaAlignment._createDraggableColumn = function(name, reconciled) { SchemaAlignment._createDraggableColumn = function(name, reconciledSiteIRI) {
var cell = $("<div></div>").addClass('wbs-draggable-column').text(name); var cell = $("<div></div>").addClass('wbs-draggable-column').text(name);
if (reconciled) { cell.data({
'columnName': name,
'reconciledSiteIRI': reconciledSiteIRI
});
if (reconciledSiteIRI !== null) {
cell.addClass('wbs-reconciled-column'); cell.addClass('wbs-reconciled-column');
} else { } else {
cell.addClass('wbs-unreconciled-column'); cell.addClass('wbs-unreconciled-column');
@ -380,7 +392,11 @@ SchemaAlignment._addItem = function(json) {
e.preventDefault(); e.preventDefault();
}); });
var inputContainer = $('<div></div>').addClass('wbs-item-input').appendTo(item); var inputContainer = $('<div></div>').addClass('wbs-item-input').appendTo(item);
SchemaAlignment._initField(inputContainer, "wikibase-item", subject); // TODO temporary solution to pick another entity type than item
// depending on the Wikibase manifest. To be removed in favour of proper support
// for multiple entity types per Wikibase
let defaultEntityType = WikibaseManager.getSelectedWikibaseDefaultEntityType();
SchemaAlignment._initField(inputContainer, "wikibase-"+defaultEntityType, subject);
var right = $('<div></div>').addClass('wbs-item-contents').appendTo(item); var right = $('<div></div>').addClass('wbs-item-contents').appendTo(item);
// Terms // Terms
@ -908,28 +924,29 @@ SchemaAlignment._initPropertyField = function(inputContainer, targetContainer, i
var input = $('<input></input>').appendTo(inputContainer); var input = $('<input></input>').appendTo(inputContainer);
input.attr("placeholder", $.i18n('wikibase-schema/property-placeholder')); input.attr("placeholder", $.i18n('wikibase-schema/property-placeholder'));
if (this._reconService !== null) { var endpoint = WikibaseManager.getSelectedWikibaseApiForEntityType('property');
var endpoint = this._reconService.suggest.property; var suggestConfig = {
var suggestConfig = $.extend({}, endpoint); mediawiki_endpoint: endpoint,
suggestConfig.key = null; entity_type: 'property',
suggestConfig.query_param_name = "prefix"; language: $.i18n("core-recon/wd-recon-lang"),
view_url: WikibaseManager.getSelectedWikibaseSiteIriForEntityType('property')+'{{id}}'
input.suggestP(suggestConfig).bind("fb-select", function(evt, data) { };
// Fetch the type of this property and add the appropriate target value type
SchemaAlignment._getPropertyType(data.id, function(datatype) { input.suggestWikibase(suggestConfig).bind("fb-select", function(evt, data) {
inputContainer.data("jsonValue", { SchemaAlignment._getPropertyType(data.id, function(datatype) {
type : "wbpropconstant", inputContainer.data("jsonValue", {
pid : data.id, type : "wbpropconstant",
label: data.name, pid : data.id,
datatype: datatype, label: data.name,
}); datatype: datatype,
SchemaAlignment._addStatement(targetContainer, datatype, null);
var addValueButtons = targetContainer.parent().find('.wbs-add-statement');
var removeGroupButton = targetContainer.parent().find('.wbs-remove-statement-group');
removeGroupButton.hide();
addValueButtons.show();
}); });
SchemaAlignment._hasChanged(); SchemaAlignment._addStatement(targetContainer, datatype, null);
var addValueButtons = targetContainer.parent().find('.wbs-add-statement');
var removeGroupButton = targetContainer.parent().find('.wbs-remove-statement-group');
removeGroupButton.hide();
addValueButtons.show();
});
SchemaAlignment._hasChanged();
}).bind("fb-textchange", function(evt, data) { }).bind("fb-textchange", function(evt, data) {
inputContainer.data("jsonValue", null); inputContainer.data("jsonValue", null);
targetContainer.find('.wbs-statement').remove(); targetContainer.find('.wbs-statement').remove();
@ -938,9 +955,8 @@ SchemaAlignment._initPropertyField = function(inputContainer, targetContainer, i
addValueButtons.hide(); addValueButtons.hide();
removeGroupButton.show(); removeGroupButton.show();
}); });
// adds tweaks to display the validation status more clearly, like in Wikidata // adds tweaks to display the validation status more clearly, like in Wikidata
fixSuggestInput(input); fixSuggestInput(input);
}
// Init with the provided initial value. // Init with the provided initial value.
if (initialValue) { if (initialValue) {
@ -960,52 +976,34 @@ SchemaAlignment._initField = function(inputContainer, mode, initialValue, change
changedCallback = SchemaAlignment._hasChanged; changedCallback = SchemaAlignment._hasChanged;
} }
if (this._reconService !== null && (mode === "wikibase-item" || mode === "unit")) { // the mode for Wikibase entities has the form "wikibase-{entitytype}", such as "wikibase-item" or "wikibase-property"
if (mode === "wikibase-item") { if (this._reconService !== null && (mode.startsWith("wikibase-") || mode === "unit")) {
input.attr("placeholder", $.i18n('wikibase-schema/item-or-reconciled-column')); var entityType = null;
if (mode.startsWith("wikibase-")) {
input.attr("placeholder", $.i18n('wikibase-schema/entity-or-reconciled-column'));
entityType = mode.slice("wikibase-".length);
} else { } else {
input.attr("placeholder", $.i18n('wikibase-schema/unit')); input.attr("placeholder", $.i18n('wikibase-schema/unit'));
entityType = "item";
} }
var endpoint = null; var endpoint = WikibaseManager.getSelectedWikibaseApiForEntityType(entityType);
endpoint = this._reconService.suggest.entity; var suggestConfig = {
var suggestConfig = $.extend({}, endpoint); mediawiki_endpoint: endpoint,
suggestConfig.key = null; entity_type: entityType,
suggestConfig.query_param_name = "prefix"; language: $.i18n("core-recon/wd-recon-lang"),
if ('view' in this._reconService && 'url' in this._reconService.view && !('view_url' in endpoint)) { view_url: WikibaseManager.getSelectedWikibaseSiteIriForEntityType(entityType)+'{{id}'
suggestConfig.view_url = this._reconService.view.url; };
}
input.suggestWikibase(suggestConfig).bind("fb-select", function(evt, data) {
input.suggest(suggestConfig).bind("fb-select", function(evt, data) {
inputContainer.data("jsonValue", { inputContainer.data("jsonValue", {
type : "wbitemconstant", type : "wbentityidvalueconstant",
qid : data.id, id : data.id,
label: data.name, label: data.name,
}); });
changedCallback(); changedCallback();
}); });
// adds tweaks to display the validation status more clearly, like in Wikidata // adds tweaks to display the validation status more clearly, like in Wikidata
fixSuggestInput(input); fixSuggestInput(input);
} else if (this._reconService !== null && mode === "wikibase-property") {
var endpoint = null;
endpoint = this._reconService.suggest.property;
var suggestConfig = $.extend({}, endpoint);
suggestConfig.key = null;
suggestConfig.query_param_name = "prefix";
input.suggestP(suggestConfig).bind("fb-select", function(evt, data) {
inputContainer.data("jsonValue", {
type : "wbpropconstant",
pid : data.id,
label: data.name,
datatype: "not-important",
});
changedCallback();
});
// adds tweaks to display the validation status more clearly, like in Wikidata
fixSuggestInput(input);
} else if (mode === "time") { } else if (mode === "time") {
input.attr("placeholder", "YYYY(-MM(-DD))"); input.attr("placeholder", "YYYY(-MM(-DD))");
var propagateValue = function(val) { var propagateValue = function(val) {
@ -1176,9 +1174,15 @@ SchemaAlignment._initField = function(inputContainer, mode, initialValue, change
// Make it droppable // Make it droppable
var acceptClass = ".wbs-draggable-column"; var acceptClass = ".wbs-draggable-column";
var wbVariableType = "wbstringvariable"; var wbVariableType = "wbstringvariable";
if (mode === "wikibase-item" || mode === "unit") { var entityType = null;
if (mode.startsWith("wikibase-") || mode === "unit") {
acceptClass = ".wbs-reconciled-column"; acceptClass = ".wbs-reconciled-column";
wbVariableType = "wbitemvariable"; wbVariableType = "wbentityvariable";
if (mode === "unit") {
entityType = "item";
} else {
entityType = mode.slice("wikibase-".length);
}
} else if (mode === "time") { } else if (mode === "time") {
wbVariableType = "wbdatevariable"; wbVariableType = "wbdatevariable";
} else if (mode === "globe-coordinate") { } else if (mode === "globe-coordinate") {
@ -1194,6 +1198,16 @@ SchemaAlignment._initField = function(inputContainer, mode, initialValue, change
accept: acceptClass, accept: acceptClass,
}).on("drop", function (evt, ui) { }).on("drop", function (evt, ui) {
var column = ui.draggable.clone(); var column = ui.draggable.clone();
if (entityType !== null) {
// check that the siteIRI of the reconciled column matches
// the expected one.
var expectedSiteIRI = WikibaseManager.getSelectedWikibaseSiteIriForEntityType(entityType);
var actualSiteIRI = ui.draggable.data('reconciledSiteIRI');
if (actualSiteIRI !== expectedSiteIRI) {
alert($.i18n('wikibase-schema/incompatible-site-iri-reconciled-column', actualSiteIRI, expectedSiteIRI));
return false;
}
}
acceptDraggableColumn(column); acceptDraggableColumn(column);
inputContainer.data("jsonValue", { inputContainer.data("jsonValue", {
type : wbVariableType, type : wbVariableType,
@ -1210,10 +1224,12 @@ SchemaAlignment._initField = function(inputContainer, mode, initialValue, change
// Init with the provided initial value. // Init with the provided initial value.
if (initialValue) { if (initialValue) {
if (initialValue.type === "wbitemconstant" || initialValue.type === "wbpropconstant") { if (initialValue.type === "wbentityidvalueconstant" ||
/* item for backwards-compatibility purposes */
initialValue.type === "wbitemconstant" || initialValue.type === "wbpropconstant") {
input.val(initialValue.label); input.val(initialValue.label);
input.addClass("wbs-validated-input"); input.addClass("wbs-validated-input");
} else if (initialValue.type == "wbitemvariable") { } else if (initialValue.type == "wbentityvariable") {
var cell = SchemaAlignment._createDraggableColumn(initialValue.columnName, true); var cell = SchemaAlignment._createDraggableColumn(initialValue.columnName, true);
acceptDraggableColumn(cell); acceptDraggableColumn(cell);
} else if (initialValue.type === "wbstringconstant" || } else if (initialValue.type === "wbstringconstant" ||
@ -1279,6 +1295,7 @@ SchemaAlignment.getJSON = function() {
return { return {
itemDocuments: list, itemDocuments: list,
siteIri: WikibaseManager.getSelectedWikibaseSiteIri(), siteIri: WikibaseManager.getSelectedWikibaseSiteIri(),
entityTypeSiteIri: Object.fromEntries(WikibaseManager.getSelectedWikibaseAvailableEntityTypes().map(et => [et, WikibaseManager.getSelectedWikibaseSiteIriForEntityType(et)])),
mediaWikiApiEndpoint: WikibaseManager.getSelectedWikibaseApi() mediaWikiApiEndpoint: WikibaseManager.getSelectedWikibaseApi()
}; };
} else { } else {

86
OpenRefine/extensions/wikidata/module/scripts/wikibase-manager.js
View File

@ -24,6 +24,18 @@ WikibaseManager.getSelectedWikibaseApi = function () {
return WikibaseManager.getSelectedWikibase().mediawiki.api; return WikibaseManager.getSelectedWikibase().mediawiki.api;
}; };
WikibaseManager.getSelectedWikibaseApiForEntityType = function (entityType) {
let manifest = WikibaseManager.getSelectedWikibase();
// version 1
if (manifest.version.split('.')[0] === '1') {
return manifest.wikibase.site_iri;
} else { // version 2 or above
let record = manifest.entity_types[entityType];
let api = record === undefined ? undefined : record.mediawiki_api;
return api === undefined ? manifest.mediawiki.api : api;
}
};
WikibaseManager.getSelectedWikibaseName = function () { WikibaseManager.getSelectedWikibaseName = function () {
return WikibaseManager.selected; return WikibaseManager.selected;
}; };
@ -47,13 +59,81 @@ WikibaseManager.getSelectedWikibaseEditGroupsURLSchema = function() {
/** /**
* Returns the default reconciliation service URL of the Wikibase, * Returns the default reconciliation service URL of the Wikibase,
* such as "https://wikidata.reconci.link/${lang}/api". * for a given entity type, such as "https://wikidata.reconci.link/${lang}/api".
* *
* Notice that there is a "${lang}" variable in the URL, which should * Notice that there is a "${lang}" variable in the URL, which should
* be replaced with the actual language code. * be replaced with the actual language code.
*/ */
WikibaseManager.getSelectedWikibaseReconEndpoint = function () { WikibaseManager.getSelectedWikibaseReconEndpoint = function (entityType) {
return WikibaseManager.getSelectedWikibase().reconciliation.endpoint; let manifest = WikibaseManager.getSelectedWikibase();
// version 1
if (manifest.version.split('.')[0] === '1') {
if (entityType === 'item') {
return manifest.reconciliation.endpoint;
}
return null;
} else { // version 2 or above
let record = manifest.entity_types[entityType];
return record === undefined ? undefined : record.reconciliation_endpoint;
}
};
WikibaseManager.getReconciliationEndpoints = function (manifest) {
// version 1
if (manifest.version.split('.')[0] === '1') {
return [manifest.reconciliation.endpoint];
} else { // version 2 or above
return Object.keys(manifest.entity_types)
.map(k => manifest.entity_types[k].reconciliation_endpoint)
.filter(endpoint => endpoint != null);
}
};
WikibaseManager.getSelectedWikibaseSiteIriForEntityType = function (entityType) {
let manifest = WikibaseManager.getSelectedWikibase();
// version 1
if (manifest.version.split('.')[0] === '1') {
return manifest.wikibase.site_iri;
} else { // version 2 or above
let record = manifest.entity_types[entityType];
return record === undefined ? undefined : record.site_iri;
}
};
WikibaseManager.getSelectedWikibaseAvailableEntityTypes = function () {
let manifest = WikibaseManager.getSelectedWikibase();
// version 1
if (manifest.version.split('.')[0] === '1') {
return ['item', 'property'];
} else { // version 2 or above
return Object.keys(manifest.entity_types);
}
};
/**
* TODO temporary function to be removed once we have proper support
* for multiple entity types. This one just guesses which item-like
* entity type we should let the user edit, based on the manifest.
* - for Wikidata it returns 'item'
* - for Commons it returns 'mediainfo'
*/
WikibaseManager.getSelectedWikibaseDefaultEntityType = function () {
let manifest = WikibaseManager.getSelectedWikibase();
// version 1
if (manifest.version.split('.')[0] === '1') {
return 'item';
} else { // version 2 or above
for (let entityType of ['item', 'property', 'mediainfo']) {
if (manifest.entity_types[entityType] !== undefined &&
manifest.entity_types[entityType].site_iri === manifest.wikibase.site_iri) {
return entityType;
}
}
return 'item'; // by default, as a fallback
}
}; };
WikibaseManager.selectWikibase = function (wikibaseName) { WikibaseManager.selectWikibase = function (wikibaseName) {

2
OpenRefine/extensions/wikidata/module/scripts/wikibase-manifest-schema-v1.js
View File

@ -1,6 +1,6 @@
const WikibaseManifestSchemaV1 = { const WikibaseManifestSchemaV1 = {
"$schema": "http://json-schema.org/draft-07/schema#", "$schema": "http://json-schema.org/draft-07/schema#",
"$id": "TBD", "$id": "https://openrefine.org/schemas/wikibase-manifest-v1.json",
"type": "object", "type": "object",
"description": "The schema validates Wikibase manifests with version 1.x. The manifest contains configurations of basic information (e.g. URL of the main page), extensions (e.g. OAuth extension) or external services (e.g. Reconciliation service) of a Wikibase", "description": "The schema validates Wikibase manifests with version 1.x. The manifest contains configurations of basic information (e.g. URL of the main page), extensions (e.g. OAuth extension) or external services (e.g. Reconciliation service) of a Wikibase",
"properties": { "properties": {

142
OpenRefine/extensions/wikidata/module/scripts/wikibase-manifest-schema-v2.js Normal file
View File

@ -0,0 +1,142 @@
const WikibaseManifestSchemaV2 = {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://openrefine.org/schemas/wikibase-manifest-v2.json",
"type": "object",
"description": "The schema validates Wikibase manifests with version 2.x. The manifest contains configurations of basic information (e.g. URL of the main page), extensions (e.g. OAuth extension) or external services (e.g. Reconciliation service) of a Wikibase",
"properties": {
"version": {
"type": "string",
"pattern": "^2\\.[0-9]+$",
"description": "The version of the Wikibase manifest, in the format of 2.x"
},
"mediawiki": {
"type": "object",
"description": "The configurations of the MediaWiki engine",
"properties": {
"name": {
"type": "string",
"description": "The name of the Wikibase, such as 'Wikidata'"
},
"root": {
"type": "string",
"format": "url",
"pattern": "^.*/$",
"description": "The URL of the root of the Wikibase, such as 'https://www.wikidata.org/wiki/'. The trailing slash cannot be omitted"
},
"main_page": {
"type": "string",
"format": "url",
"description": "The URL of the main page of the Wikibase, such as 'https://www.wikidata.org/wiki/Wikidata:Main_Page'"
},
"api": {
"type": "string",
"format": "url",
"description": "The MediaWiki API endpoint of the Wikibase, such as 'https://www.wikidata.org/w/api.php'"
}
},
"required": ["name", "root", "main_page", "api"]
},
"wikibase": {
"type": "object",
"description": "The configurations of the Wikibase extension",
"properties": {
"site_iri": {
"type": "string",
"format": "url",
"pattern": "^.*/$",
"description": "The IRI of the Wikibase, such as 'http://www.wikidata.org/entity/'. This should match the IRI prefixes used in RDF serialization. Be careful about using 'http' or 'https', because any variation will break comparisons at various places. The trailing slash cannot be omitted"
},
"maxlag": {
"type": "integer",
"description": "The default maxlag of this Wikibase. For Wikidata, the default value is 5 (seconds)"
},
"properties": {
"type": "object",
"properties": {
"instance_of": {
"type": "string",
"description": "The 'instance of' qid of the Wikibase ('P31' for Wikidata)"
},
"subclass_of": {
"type": "string",
"description": "The 'subclass of' qid of the Wikibase ('P279' for Wikidata)"
}
},
"required": ["instance_of", "subclass_of"]
},
"constraints": {
"type": "object",
"description": "Constraints related qids and pids, not required since the constraints extension may not be installed",
"properties": {
"property_constraint_pid": {
"type": "string",
"description": "The property constraint pid of the Wikibase ('P2302' for Wikidata)"
}
},
"patternProperties": {
"^.*$": {
"type": "string",
"description": "If a pid/qid is missing, constraint checks depending on it will be skipped"
}
},
"required": ["property_constraint_pid"]
}
},
"required": ["site_iri", "maxlag", "properties"]
},
"oauth": {
"type": "object",
"description": "The configurations of the OAuth extension. Not required. Configuring this if and only if the OAuth extension is installed",
"properties": {
"registration_page": {
"type": "string",
"format": "url",
"description": "The url of the OAuth consumer registration page, 'https://meta.wikimedia.org/wiki/Special:OAuthConsumerRegistration/propose' for Wikidata"
}
},
"required": ["registration_page"]
},
"entity_types": {
"type": "object",
"description": "The available entity types on this Wikibase, with particular settings for each of them. The keys of this object are strings such as 'item', 'property', 'mediainfo' or 'lexeme'",
"patternProperties": {
"^.*$": {
"type": "object",
"description": "The settings for the given entity type.",
"properties": {
"reconciliation_endpoint": {
"type": "string",
"format": "url",
"pattern": "^.*\\${lang}.*$",
"description": "The default reconciliation API endpoint of the Wikibase for this entity type, the endpoint should include the language variable '${lang}', such as 'https://wikidata.reconci.link/${lang}/api'"
},
"site_iri": {
"type": "string",
"format": "url",
"description": "The RDF prefix in which URIs for entities of this type are minted. If entities of this type are edited locally, this should be the global site_iri of the Wikibase. If those entities are drawn from another Wikibase instance, it should be the site_iri of that instance."
},
"mediawiki_api": {
"type": "string",
"format": "url",
"description": "The MediaWiki API endpoint of the Wikibase, such as 'https://www.wikidata.org/w/api.php'"
}
},
"required": ["site_iri"]
}
}
},
"editgroups": {
"type": "object",
"description": "The configurations of the EditGroups service of the Wikibase",
"properties": {
"url_schema": {
"type": "string",
"pattern": "^.*\\${batch_id}.*$",
"description": "The URL schema used in edits summary. This is used for EditGroups to extract the batch id from a batch of edits and for linking to the EditGroups page of the batch. The URL schema must contains the variable '${batch_id}', such as '([[:toollabs:editgroups/b/OR/${batch_id}|details]])' for Wikidata"
},
},
"required": ["url_schema"]
}
},
"required": ["version", "mediawiki", "wikibase", "entity_types"]
};

142
OpenRefine/extensions/wikidata/module/scripts/wikibase-suggest.js Normal file
View File

@ -0,0 +1,142 @@
/**
* Adaptation of the suggest widget to query the Wikibase autocomplete API directly
*/
(function() {
/**
* Options:
* mediawiki_endpoint: url of the MediaWiki API
* entity_type: type of entity to suggest (one of form, item, lexeme, property, sense, mediainfo…)
* language: language code of the language to search in
*/
$.suggest(
"suggestWikibase",
$.extend(
true,
{},
$.suggest.suggest.prototype,
{
create_item: function(data, response_data) {
var css = this.options.css;
var li = $("<li>").addClass(css.item);
var name = $("<div>")
.addClass(css.item_name)
.append(
$("<label>")
.append($.suggest.strongify(data.name || data.guid, response_data.prefix)));
data.name = name.text(); // this converts html escaped strings like "&amp;" back to "&"
if (data.description) {
var descriptionSpan = $("<span></span>").text(data.description);
name.append(descriptionSpan);
}
li.append(name);
name.prepend($("<div>").addClass(css.item_type).text(data.id));
return li;
},
request: function(val, cursor) {
var self = this,
o = this.options;
var query = val;
var entity_type = o.entity_type || 'item';
var data = {
action: 'wbsearchentities',
language: o.language,
search: query,
type: entity_type,
format: 'json',
origin: '*'
};
if (cursor) {
data['continue'] = cursor;
}
var url = o.mediawiki_endpoint + "?" + $.param(data, true);
var cached = $.suggest.cache[url];
if (cached) {
this.response(cached, cursor ? cursor : -1, true);
return;
}
clearTimeout(this.request.timeout);
var ajax_options = {
url: o.mediawiki_endpoint,
data: data,
traditional: true,
beforeSend: function(xhr) {
var calls = self.input.data("request.count.suggest") || 0;
if (!calls) {
self.trackEvent(self.name, "start_session");
}
calls += 1;
self.trackEvent(self.name, "request", "count", calls);
self.input.data("request.count.suggest", calls);
},
success: function(data) {
$.suggest.cache[url] = data;
// translate the results of the MediaWiki API to that of the reconciliation API
var translated = {
prefix: val, // keep track of prefix to match up response with input value
result: (data.search || []).map(result => { return {
id: result.id,
name: result.label,
description: result.description};})
};
self.response(translated, cursor ? cursor : -1);
},
error: function(xhr) {
self.status_error();
self.trackEvent(self.name, "request", "error", {
url: this.url,
response: xhr ? xhr.responseText : ''
});
self.input.trigger("fb-error", Array.prototype.slice.call(arguments));
},
complete: function(xhr) {
if (xhr) {
self.trackEvent(self.name, "request", "tid",
xhr.getResponseHeader("X-Metaweb-TID"));
}
},
dataType: "json",
cache: true
};
this.request.timeout = setTimeout(function() {
$.ajax(ajax_options);
}, o.xhr_delay);
}
}));
$.extend(
$.suggest.suggestWikibase,
{
defaults: $.extend(
true,
{},
$.suggest.suggest.defaults, {
scoring: "schema",
css: { pane: "fbs-pane fbs-pane-type" }
}
)
}
);
})();

23
OpenRefine/extensions/wikidata/pom.xml
View File

@ -15,8 +15,7 @@
</parent> </parent>
<properties> <properties>
<powermock.version>2.0.9</powermock.version> <wdtk.version>0.12.1-SNAPSHOT</wdtk.version>
<wdtk.version>0.12.1</wdtk.version>
</properties> </properties>
<build> <build>
@ -78,7 +77,7 @@
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.wikidata.wdtk</groupId> <groupId>org.openrefine.dependencies.wdtk</groupId>
<artifactId>wdtk-wikibaseapi</artifactId> <artifactId>wdtk-wikibaseapi</artifactId>
<version>${wdtk.version}</version> <version>${wdtk.version}</version>
</dependency> </dependency>
@ -88,12 +87,12 @@
<version>${jackson.version}</version> <version>${jackson.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.wikidata.wdtk</groupId> <groupId>org.openrefine.dependencies.wdtk</groupId>
<artifactId>wdtk-datamodel</artifactId> <artifactId>wdtk-datamodel</artifactId>
<version>${wdtk.version}</version> <version>${wdtk.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.wikidata.wdtk</groupId> <groupId>org.openrefine.dependencies.wdtk</groupId>
<artifactId>wdtk-util</artifactId> <artifactId>wdtk-util</artifactId>
<version>${wdtk.version}</version> <version>${wdtk.version}</version>
</dependency> </dependency>
@ -121,17 +120,11 @@
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.powermock</groupId> <groupId>org.mockito</groupId>
<artifactId>powermock-module-testng</artifactId> <artifactId>mockito-core</artifactId>
<version>${powermock.version}</version> <version>${mockito.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<version>${powermock.version}</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>com.squareup.okhttp3</groupId> <groupId>com.squareup.okhttp3</groupId>
<artifactId>mockwebserver</artifactId> <artifactId>mockwebserver</artifactId>

2
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/commands/ConnectionManager.java
View File

@ -165,7 +165,7 @@ public class ConnectionManager {
/** /**
* For testability. * For testability.
*/ */
static BasicApiConnection convertToBasicApiConnection(Map<String, Object> map) throws JsonProcessingException { BasicApiConnection convertToBasicApiConnection(Map<String, Object> map) throws JsonProcessingException {
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();
String json = mapper.writeValueAsString(map); String json = mapper.writeValueAsString(map);
return mapper.readValue(json, BasicApiConnection.class); return mapper.readValue(json, BasicApiConnection.class);

11
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/commands/LoginCommand.java
View File

@ -71,6 +71,8 @@ public class LoginCommand extends Command {
static final String ACCESS_SECRET = "wb-access-secret"; static final String ACCESS_SECRET = "wb-access-secret";
static final Pattern cookieKeyDisallowedCharacters = Pattern.compile("[^a-zA-Z0-9\\-!#$%&'*+.?\\^_`|~]"); static final Pattern cookieKeyDisallowedCharacters = Pattern.compile("[^a-zA-Z0-9\\-!#$%&'*+.?\\^_`|~]");
protected ConnectionManager manager = null;
@Override @Override
public void doPost(HttpServletRequest request, HttpServletResponse response) public void doPost(HttpServletRequest request, HttpServletResponse response)
@ -80,7 +82,9 @@ public class LoginCommand extends Command {
return; return;
} }
ConnectionManager manager = ConnectionManager.getInstance(); if (manager == null) {
manager = ConnectionManager.getInstance();
}
String mediawikiApiEndpoint = removeCRLF(request.getParameter(API_ENDPOINT)); String mediawikiApiEndpoint = removeCRLF(request.getParameter(API_ENDPOINT));
if (isBlank(mediawikiApiEndpoint)) { if (isBlank(mediawikiApiEndpoint)) {
@ -188,7 +192,6 @@ public class LoginCommand extends Command {
return; return;
} }
ConnectionManager manager = ConnectionManager.getInstance();
Map<String, Object> jsonResponse = new HashMap<>(); Map<String, Object> jsonResponse = new HashMap<>();
if (manager.isLoggedIn(mediawikiApiEndpoint)) { if (manager.isLoggedIn(mediawikiApiEndpoint)) {
jsonResponse.put("logged_in", manager.isLoggedIn(mediawikiApiEndpoint)); jsonResponse.put("logged_in", manager.isLoggedIn(mediawikiApiEndpoint));
@ -281,4 +284,8 @@ public class LoginCommand extends Command {
Matcher matcher = cookieKeyDisallowedCharacters.matcher(key); Matcher matcher = cookieKeyDisallowedCharacters.matcher(key);
return matcher.replaceAll("-"); return matcher.replaceAll("-");
} }
protected void setConnectionManager(ConnectionManager connectionManager) {
this.manager = connectionManager;
}
} }

2
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java
View File

@ -73,7 +73,7 @@ public class PreviewWikibaseSchemaCommand extends Command {
try { try {
schema = WikibaseSchema.reconstruct(schemaJson); schema = WikibaseSchema.reconstruct(schemaJson);
} catch (IOException e) { } catch (IOException e) {
respondError(response, "Wikibase schema could not be parsed."); respondError(response, "Wikibase schema could not be parsed. Error message: "+e.getMessage());
return; return;
} }
} else { } else {

2
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/commands/SaveWikibaseSchemaCommand.java
View File

@ -71,7 +71,7 @@ public class SaveWikibaseSchemaCommand extends Command {
// We do not use respondException here because this is an expected // We do not use respondException here because this is an expected
// exception which happens every time a user tries to save an incomplete // exception which happens every time a user tries to save an incomplete
// schema - the exception should not be logged. // schema - the exception should not be logged.
respondError(response, "Wikibase schema could not be parsed."); respondError(response, String.format("Wikibase schema could not be parsed: ", e.getMessage()));
} catch (Exception e) { } catch (Exception e) {
// This is an unexpected exception, so we log it. // This is an unexpected exception, so we log it.
respondException(response, e); respondException(response, e);

82
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java
View File

@ -25,11 +25,15 @@ package org.openrefine.wikidata.editing;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.commons.lang.NotImplementedException;
import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue; import org.openrefine.wikidata.schema.entityvalues.ReconEntityIdValue;
import org.openrefine.wikidata.schema.exceptions.NewItemNotCreatedYetException; import org.openrefine.wikidata.schema.exceptions.NewItemNotCreatedYetException;
import org.openrefine.wikidata.updates.TermedStatementEntityUpdate; import org.openrefine.wikidata.updates.TermedStatementEntityUpdate;
@ -37,10 +41,17 @@ import org.openrefine.wikidata.updates.scheduler.WikibaseAPIUpdateScheduler;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.AliasUpdate;
import org.wikidata.wdtk.datamodel.interfaces.EntityDocument; import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument; import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MediaInfoDocument;
import org.wikidata.wdtk.datamodel.interfaces.MediaInfoIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MediaInfoUpdate;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.StatementUpdate;
import org.wikidata.wdtk.datamodel.interfaces.TermUpdate;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor; import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher; import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
@ -149,33 +160,56 @@ public class EditBatchProcessor {
// New item // New item
if (update.isNew()) { if (update.isNew()) {
ReconEntityIdValue newCell = (ReconEntityIdValue) update.getItemId(); ReconEntityIdValue newCell = (ReconEntityIdValue) update.getItemId();
update = update.normalizeLabelsAndAliases(); if (newCell instanceof ItemIdValue) {
update = update.normalizeLabelsAndAliases();
ItemDocument itemDocument = Datamodel.makeItemDocument((ItemIdValue) update.getItemId(),
update.getLabels().stream().collect(Collectors.toList()),
update.getDescriptions().stream().collect(Collectors.toList()),
update.getAliases().stream().collect(Collectors.toList()), update.getAddedStatementGroups(),
Collections.emptyMap());
ItemDocument itemDocument = Datamodel.makeItemDocument((ItemIdValue) update.getItemId(), ItemDocument createdDoc = editor.createItemDocument(itemDocument, summary, tags);
update.getLabels().stream().collect(Collectors.toList()), library.setQid(newCell.getReconInternalId(), createdDoc.getEntityId().getId());
update.getDescriptions().stream().collect(Collectors.toList()), } else if (newCell instanceof MediaInfoIdValue) {
update.getAliases().stream().collect(Collectors.toList()), update.getAddedStatementGroups(), update = update.normalizeLabelsAndAliases();
Collections.emptyMap()); throw new NotImplementedException();
}
ItemDocument createdDoc = editor.createItemDocument(itemDocument, summary, tags);
library.setQid(newCell.getReconInternalId(), createdDoc.getEntityId().getId());
} else { } else {
// Existing item // Existing item
ItemDocument currentDocument = (ItemDocument) currentDocs.get(update.getItemId().getId()); EntityIdValue newCell = (EntityIdValue) update.getItemId();
List<MonolingualTextValue> labels = update.getLabels().stream().collect(Collectors.toList()); if (newCell instanceof ItemIdValue) {
labels.addAll(update.getLabelsIfNew().stream() ItemDocument currentDocument = (ItemDocument) currentDocs.get(update.getItemId().getId());
.filter(label -> !currentDocument.getLabels().containsKey(label.getLanguageCode())).collect(Collectors.toList())); List<MonolingualTextValue> labels = update.getLabels().stream().collect(Collectors.toList());
List<MonolingualTextValue> descriptions = update.getDescriptions().stream().collect(Collectors.toList()); labels.addAll(update.getLabelsIfNew().stream()
descriptions.addAll(update.getDescriptionsIfNew().stream() .filter(label -> !currentDocument.getLabels().containsKey(label.getLanguageCode())).collect(Collectors.toList()));
.filter(desc -> !currentDocument.getDescriptions().containsKey(desc.getLanguageCode())).collect(Collectors.toList())); List<MonolingualTextValue> descriptions = update.getDescriptions().stream().collect(Collectors.toList());
editor.updateTermsStatements(currentDocument, descriptions.addAll(update.getDescriptionsIfNew().stream()
labels, .filter(desc -> !currentDocument.getDescriptions().containsKey(desc.getLanguageCode())).collect(Collectors.toList()));
descriptions, Set<MonolingualTextValue> aliases = update.getAliases();
update.getAliases().stream().collect(Collectors.toList()), Map<String, List<MonolingualTextValue>> aliasesMap = aliases.stream()
new ArrayList<MonolingualTextValue>(), .collect(Collectors.groupingBy(MonolingualTextValue::getLanguageCode));
update.getAddedStatements().stream().collect(Collectors.toList()), Map<String, AliasUpdate> aliasMap = aliasesMap.entrySet().stream()
update.getDeletedStatements().stream().collect(Collectors.toList()), .collect(Collectors.toMap(Entry::getKey, e -> Datamodel.makeAliasUpdate(e.getValue(), Collections.emptyList())));
summary, tags); editor.editEntityDocument(Datamodel.makeItemUpdate((ItemIdValue) update.getItemId(),
currentDocument.getRevisionId(),
Datamodel.makeTermUpdate(labels, Collections.emptyList()),
Datamodel.makeTermUpdate(descriptions, Collections.emptyList()),
aliasMap,
Datamodel.makeStatementUpdate(update.getAddedStatements(), update.getDeletedStatements(), Collections.emptyList()),
Collections.emptyList(), Collections.emptyList()),
false, summary, tags);
} else if (newCell instanceof MediaInfoIdValue) {
MediaInfoDocument currentDocument = (MediaInfoDocument) currentDocs.get(update.getItemId().getId());
List<MonolingualTextValue> labels = update.getLabels().stream().collect(Collectors.toList());
labels.addAll(update.getLabelsIfNew().stream()
.filter(label -> !currentDocument.getLabels().containsKey(label.getLanguageCode())).collect(Collectors.toList()));
TermUpdate labelUpdate = Datamodel.makeTermUpdate(labels, Collections.emptyList());
StatementUpdate statementUpdate = Datamodel.makeStatementUpdate(update.getAddedStatements(), update.getDeletedStatements(),
Collections.emptyList());
MediaInfoUpdate updatesCollection = Datamodel.makeMediaInfoUpdate((MediaInfoIdValue) update.getItemId(),
currentDocument.getRevisionId(), labelUpdate, statementUpdate);
editor.editEntityDocument(updatesCollection, false, summary, tags);
}
} }
} catch (MediaWikiApiErrorException e) { } catch (MediaWikiApiErrorException e) {
// TODO find a way to report these errors to the user in a nice way // TODO find a way to report these errors to the user in a nice way

2
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/editing/ReconEntityRewriter.java
View File

@ -114,7 +114,7 @@ public class ReconEntityRewriter extends DatamodelConverter {
return Datamodel.makeMediaInfoIdValue(newId, recon.getRecon().identifierSpace); return Datamodel.makeMediaInfoIdValue(newId, recon.getRecon().identifierSpace);
} }
} }
return (MediaInfoIdValue) super.copy((ItemIdValue) value); return super.copy((MediaInfoIdValue) value);
} }
@Override @Override

82
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/manifests/Manifest.java
View File

@ -1,24 +1,104 @@
package org.openrefine.wikidata.manifests; package org.openrefine.wikidata.manifests;
public interface Manifest { import java.util.List;
/**
* A configuration object for a Wikibase instance.
* The deserialization of this object is versioned, via {@link org.openrefine.wikidata.manifests.ManifestParser}.
*
* @author Lu Liu, Antonin Delpeuch
*
*/
public interface Manifest {
public static final String ITEM_TYPE = "item";
public static final String PROPERTY_TYPE = "property";
public static final String MEDIAINFO_TYPE = "mediainfo";
/**
* The version of the manifest object, which determines its JSON format.
*/
String getVersion(); String getVersion();
/**
* The name of the Wikibase instance, displayed in the UI.
*/
String getName(); String getName();
/**
* The RDF serialization prefix for entities stored in this instance.
*/
String getSiteIri(); String getSiteIri();
/**
* The recommended `maxlag` value for edits on this instance.
*/
int getMaxlag(); int getMaxlag();
/**
* The property id used to link an entity to the entity represing its type (class).
* In Wikidata, this is P31 (instance of)
*/
String getInstanceOfPid(); String getInstanceOfPid();
/**
* The property id used to link a class to its superclasses.
* In Wikidata, this is P279 (subclass of)
*/
String getSubclassOfPid(); String getSubclassOfPid();
/**
* The MediaWiki API endpoint of this Wikibase instance.
*/
String getMediaWikiApiEndpoint(); String getMediaWikiApiEndpoint();
/**
* The reconciliation service for items in this Wikibase instance.
* @deprecated use {@link #getReconServiceEndpoint(String)} with "item" as argument
*/
String getReconServiceEndpoint(); String getReconServiceEndpoint();
/**
* Get the reconciliation service endpoint for a given entity type supported by this Wikibase instance.
* @param entityType
* @return null if there is no recon service for this entity type.
*/
String getReconServiceEndpoint(String entityType);
/**
* Get the MediaWiki endpoint of the source Wikibase for the entity type.
* This can be different than the MediaWiki endpoint for this Wikibase instance,
* when federation is used.
* @param entityType
* @return null if there is no recon service for this entity type.
*/
String getMediaWikiApiEndpoint(String entityType);
/**
* Gets the site IRI used for a particular entity type.
* - if the entity type is editable on this Wikibase instance, then it should be identical
* to the site IRI for this instance.
* - if the entity type is federated from another instance, then it should be the site IRI
* for that instance.
* @param entityType
* @return null if the entity type is not supported by the Wikibase instance
*/
String getEntityTypeSiteIri(String entityType);
/**
* The list of all entity types in use on this instance.
*/
List<String> getAvailableEntityTypes();
/**
* Returns an item or property id used in the WikibaseQualityConstraints extension.
* @param name our internal identifier for the entity id
* @return the entity id
*/
String getConstraintsRelatedId(String name); String getConstraintsRelatedId(String name);
/**
* Returns the template that should be inserted in edit summaries for edits to be tracked by EditGroups.
*/
String getEditGroupsUrlSchema(); String getEditGroupsUrlSchema();
} }

9
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/manifests/ManifestParser.java
View File

@ -3,6 +3,9 @@ package org.openrefine.wikidata.manifests;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -33,6 +36,12 @@ public final class ManifestParser {
// support only v1.x for now // support only v1.x for now
if ("1".equals(majorVersion)) { if ("1".equals(majorVersion)) {
return new ManifestV1(root); return new ManifestV1(root);
} else if ("2".equals(majorVersion)) {
try {
return new ManifestV2(root);
} catch (IOException e) {
throw new ManifestException("invalid manifest format: " + e.getMessage());
}
} else { } else {
throw new ManifestException("unsupported manifest version: " + version); throw new ManifestException("unsupported manifest version: " + version);
} }

34
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/manifests/ManifestV1.java
View File

@ -1,12 +1,14 @@
package org.openrefine.wikidata.manifests; package org.openrefine.wikidata.manifests;
import com.fasterxml.jackson.databind.JsonNode; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List;
import java.util.Map; import java.util.Map;
import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import com.fasterxml.jackson.databind.JsonNode;
public class ManifestV1 implements Manifest { public class ManifestV1 implements Manifest {
@ -102,4 +104,30 @@ public class ManifestV1 implements Manifest {
return editGroupsUrlSchema; return editGroupsUrlSchema;
} }
@Override
public String getReconServiceEndpoint(String entityType) {
if (ITEM_TYPE.equals(entityType)) {
return reconServiceEndpoint;
}
return null;
}
@Override
public String getEntityTypeSiteIri(String entityType) {
if (ITEM_TYPE.equals(entityType) || PROPERTY_TYPE.equals(entityType)) {
return siteIri;
}
return null;
}
@Override
public String getMediaWikiApiEndpoint(String entityType) {
return getMediaWikiApiEndpoint();
}
@Override
public List<String> getAvailableEntityTypes() {
return Arrays.asList(ITEM_TYPE, PROPERTY_TYPE);
}
} }

19
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/schema/ExpressionContext.java
View File

@ -23,6 +23,8 @@
******************************************************************************/ ******************************************************************************/
package org.openrefine.wikidata.schema; package org.openrefine.wikidata.schema;
import java.util.Map;
import org.apache.commons.lang.Validate; import org.apache.commons.lang.Validate;
import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.qa.QAWarning;
import org.openrefine.wikidata.qa.QAWarningStore; import org.openrefine.wikidata.qa.QAWarningStore;
@ -42,6 +44,7 @@ import com.google.refine.model.Row;
public class ExpressionContext { public class ExpressionContext {
private String baseIRI; private String baseIRI;
private Map<String, String> entityTypeIRIs;
private String mediaWikiApiEndpoint; private String mediaWikiApiEndpoint;
private int rowId; private int rowId;
private Row row; private Row row;
@ -53,6 +56,8 @@ public class ExpressionContext {
* *
* @param baseIRI * @param baseIRI
* the siteIRI of the schema * the siteIRI of the schema
* @param entityTypeBaseIRIS
* the siteIRI for specific entity types, falling back on the baseIRI otherwise
* @param mediaWikiApiEndpoint * @param mediaWikiApiEndpoint
* the MediaWiki API endpoint of the Wikibase * the MediaWiki API endpoint of the Wikibase
* @param rowId * @param rowId
@ -65,9 +70,17 @@ public class ExpressionContext {
* where to store the issues encountered when evaluating (can be set * where to store the issues encountered when evaluating (can be set
* to null if these issues should be ignored) * to null if these issues should be ignored)
*/ */
public ExpressionContext(String baseIRI, String mediaWikiApiEndpoint, int rowId, Row row, ColumnModel columnModel, QAWarningStore warningStore) { public ExpressionContext(
String baseIRI,
Map<String, String> entityTypeBaseIRIs,
String mediaWikiApiEndpoint,
int rowId,
Row row,
ColumnModel columnModel,
QAWarningStore warningStore) {
Validate.notNull(baseIRI); Validate.notNull(baseIRI);
this.baseIRI = baseIRI; this.baseIRI = baseIRI;
this.entityTypeIRIs = entityTypeBaseIRIs;
this.mediaWikiApiEndpoint = mediaWikiApiEndpoint; this.mediaWikiApiEndpoint = mediaWikiApiEndpoint;
this.rowId = rowId; this.rowId = rowId;
Validate.notNull(row); Validate.notNull(row);
@ -80,6 +93,10 @@ public class ExpressionContext {
public String getBaseIRI() { public String getBaseIRI() {
return baseIRI; return baseIRI;
} }
public String getBaseIRIForEntityType(String entityType) {
return entityTypeIRIs.getOrDefault(entityType, baseIRI);
}
public String getMediaWikiApiEndpoint() { public String getMediaWikiApiEndpoint() {
return mediaWikiApiEndpoint; return mediaWikiApiEndpoint;

28
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/schema/WbEntityVariable.java
View File

@ -66,23 +66,33 @@ public class WbEntityVariable extends WbVariableExpr<EntityIdValue> {
throws SkipSchemaExpressionException { throws SkipSchemaExpressionException {
if (cell.recon != null if (cell.recon != null
&& (Judgment.Matched.equals(cell.recon.judgment) || Judgment.New.equals(cell.recon.judgment))) { && (Judgment.Matched.equals(cell.recon.judgment) || Judgment.New.equals(cell.recon.judgment))) {
if (cell.recon.identifierSpace == null || !cell.recon.identifierSpace.equals(ctxt.getBaseIRI())) {
QAWarning warning = new QAWarning("invalid-identifier-space", null, QAWarning.Severity.INFO, 1);
warning.setProperty("example_cell", cell.value.toString());
ctxt.addWarning(warning);
throw new SkipSchemaExpressionException();
}
if (Judgment.New.equals(cell.recon.judgment)) { if (Judgment.New.equals(cell.recon.judgment)) {
return new ReconItemIdValue(cell.recon, cell.value.toString()); return new ReconItemIdValue(cell.recon, cell.value.toString());
} }
EntityIdValue entityIdValue = EntityIdValueImpl.fromId(cell.recon.match.id, cell.recon.identifierSpace); EntityIdValue entityIdValue = EntityIdValueImpl.fromId(cell.recon.match.id, cell.recon.identifierSpace);
EntityIdValue reconEntityIdValue = null;
String entityType = null;
if (entityIdValue instanceof ItemIdValue) { if (entityIdValue instanceof ItemIdValue) {
return new ReconItemIdValue(cell.recon, cell.value.toString()); reconEntityIdValue = new ReconItemIdValue(cell.recon, cell.value.toString());
entityType = "item";
} else if (entityIdValue instanceof MediaInfoIdValue) { } else if (entityIdValue instanceof MediaInfoIdValue) {
return new ReconMediaInfoIdValue(cell.recon, cell.value.toString()); reconEntityIdValue = new ReconMediaInfoIdValue(cell.recon, cell.value.toString());
entityType = "mediainfo";
} else if (entityIdValue instanceof PropertyIdValue) { } else if (entityIdValue instanceof PropertyIdValue) {
return new ReconPropertyIdValue(cell.recon, cell.value.toString()); reconEntityIdValue = new ReconPropertyIdValue(cell.recon, cell.value.toString());
entityType = "property";
} }
if (reconEntityIdValue == null) {
throw new SkipSchemaExpressionException();
}
if (cell.recon.identifierSpace == null || !cell.recon.identifierSpace.equals(ctxt.getBaseIRIForEntityType(entityType))) {
QAWarning warning = new QAWarning("invalid-identifier-space", null, QAWarning.Severity.INFO, 1);
warning.setProperty("example_cell", cell.value.toString());
warning.setProperty("expected_site_iri", ctxt.getBaseIRIForEntityType(entityType));
ctxt.addWarning(warning);
throw new SkipSchemaExpressionException();
}
return reconEntityIdValue;
} }
throw new SkipSchemaExpressionException(); throw new SkipSchemaExpressionException();
} }

1
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/schema/WbExpression.java
View File

@ -47,6 +47,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
@Type(value = WbDateVariable.class, name = "wbdatevariable"), @Type(value = WbDateVariable.class, name = "wbdatevariable"),
@Type(value = WbMonolingualExpr.class, name = "wbmonolingualexpr"), @Type(value = WbMonolingualExpr.class, name = "wbmonolingualexpr"),
@Type(value = WbPropConstant.class, name = "wbpropconstant"), @Type(value = WbPropConstant.class, name = "wbpropconstant"),
@Type(value = WbEntityIdValueConstant.class, name = "wbentityidvalueconstant"),
@Type(value = WbLanguageConstant.class, name = "wblanguageconstant"), @Type(value = WbLanguageConstant.class, name = "wblanguageconstant"),
@Type(value = WbLanguageVariable.class, name = "wblanguagevariable"), @Type(value = WbLanguageVariable.class, name = "wblanguagevariable"),
@Type(value = WbQuantityExpr.class, name = "wbquantityexpr"), }) @Type(value = WbQuantityExpr.class, name = "wbquantityexpr"), })

4
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java
View File

@ -29,7 +29,7 @@ import java.util.List;
import org.jsoup.helper.Validate; import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.updates.TermedStatementEntityUpdate; import org.openrefine.wikidata.updates.TermedStatementEntityUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.openrefine.wikidata.updates.TermedStatementEntityUpdateBuilder;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.Statement;
@ -74,7 +74,7 @@ public class WbItemDocumentExpr implements WbExpression<TermedStatementEntityUpd
public TermedStatementEntityUpdate evaluate(ExpressionContext ctxt) public TermedStatementEntityUpdate evaluate(ExpressionContext ctxt)
throws SkipSchemaExpressionException { throws SkipSchemaExpressionException {
EntityIdValue subjectId = getSubject().evaluate(ctxt); EntityIdValue subjectId = getSubject().evaluate(ctxt);
ItemUpdateBuilder update = new ItemUpdateBuilder(subjectId); TermedStatementEntityUpdateBuilder update = new TermedStatementEntityUpdateBuilder(subjectId);
for (WbStatementGroupExpr expr : getStatementGroups()) { for (WbStatementGroupExpr expr : getStatementGroups()) {
try { try {
for (Statement s : expr.evaluate(ctxt, subjectId).getStatements()) { for (Statement s : expr.evaluate(ctxt, subjectId).getStatements()) {

6
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/schema/WbNameDescExpr.java
View File

@ -25,7 +25,7 @@ package org.openrefine.wikidata.schema;
import org.jsoup.helper.Validate; import org.jsoup.helper.Validate;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.openrefine.wikidata.updates.TermedStatementEntityUpdateBuilder;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
@ -35,7 +35,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/** /**
* An expression that represent a term (label, description or alias). The * An expression that represent a term (label, description or alias). The
* structure is slightly different from other expressions because we need to * structure is slightly different from other expressions because we need to
* call different methods on {@link ItemUpdateBuilder}. * call different methods on {@link TermedStatementEntityUpdateBuilder}.
* *
* @author Antonin Delpeuch * @author Antonin Delpeuch
* *
@ -67,7 +67,7 @@ public class WbNameDescExpr {
* @param ctxt * @param ctxt
* the evaluation context for the expression * the evaluation context for the expression
*/ */
public void contributeTo(ItemUpdateBuilder item, ExpressionContext ctxt) { public void contributeTo(TermedStatementEntityUpdateBuilder item, ExpressionContext ctxt) {
try { try {
MonolingualTextValue val = getValue().evaluate(ctxt); MonolingualTextValue val = getValue().evaluate(ctxt);
switch (getType()) { switch (getType()) {

19
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java
View File

@ -27,6 +27,7 @@ import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map;
import org.openrefine.wikidata.qa.QAWarningStore; import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
@ -63,15 +64,19 @@ public class WikibaseSchema implements OverlayModel {
@JsonProperty("siteIri") @JsonProperty("siteIri")
protected String siteIri; protected String siteIri;
@JsonProperty("entityTypeSiteIRI")
protected Map<String, String> entityTypeSiteIri;
@JsonProperty("mediaWikiApiEndpoint") @JsonProperty("mediaWikiApiEndpoint")
protected String mediaWikiApiEndpoint; protected String mediaWikiApiEndpoint;
/** /**
* Constructor. * Constructor.
* @todo remove this, it does not create a valid schema.
*/ */
public WikibaseSchema() { public WikibaseSchema() {
entityTypeSiteIri = Collections.emptyMap();
} }
/** /**
@ -80,9 +85,11 @@ public class WikibaseSchema implements OverlayModel {
@JsonCreator @JsonCreator
public WikibaseSchema(@JsonProperty("itemDocuments") List<WbItemDocumentExpr> exprs, public WikibaseSchema(@JsonProperty("itemDocuments") List<WbItemDocumentExpr> exprs,
@JsonProperty("siteIri") String siteIri, @JsonProperty("siteIri") String siteIri,
@JsonProperty("entityTypeSiteIRI") Map<String, String> entityTypeSiteIri,
@JsonProperty("mediaWikiApiEndpoint") String mediaWikiApiEndpoint) { @JsonProperty("mediaWikiApiEndpoint") String mediaWikiApiEndpoint) {
this.itemDocumentExprs = exprs; this.itemDocumentExprs = exprs;
this.siteIri = siteIri; this.siteIri = siteIri;
this.entityTypeSiteIri = entityTypeSiteIri != null ? entityTypeSiteIri : Collections.emptyMap();
this.mediaWikiApiEndpoint = mediaWikiApiEndpoint != null ? mediaWikiApiEndpoint : ApiConnection.URL_WIKIDATA_API; this.mediaWikiApiEndpoint = mediaWikiApiEndpoint != null ? mediaWikiApiEndpoint : ApiConnection.URL_WIKIDATA_API;
} }
@ -93,6 +100,14 @@ public class WikibaseSchema implements OverlayModel {
public String getSiteIri() { public String getSiteIri() {
return siteIri; return siteIri;
} }
/**
* @return the site IRI of the Wikibase instance referenced by this schema
*/
@JsonProperty("entityTypeSiteIRI")
public Map<String, String> getEntityTypeSiteIri() {
return entityTypeSiteIri;
}
/** /**
* @return the list of document expressions for this schema * @return the list of document expressions for this schema
@ -177,7 +192,7 @@ public class WikibaseSchema implements OverlayModel {
@Override @Override
public boolean visit(Project project, int rowIndex, Row row) { public boolean visit(Project project, int rowIndex, Row row) {
ExpressionContext ctxt = new ExpressionContext(siteIri, mediaWikiApiEndpoint, rowIndex, row, project.columnModel, warningStore); ExpressionContext ctxt = new ExpressionContext(siteIri, entityTypeSiteIri, mediaWikiApiEndpoint, rowIndex, row, project.columnModel, warningStore);
result.addAll(evaluateItemDocuments(ctxt)); result.addAll(evaluateItemDocuments(ctxt));
return false; return false;
} }

5
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconItemIdValue.java
View File

@ -33,6 +33,11 @@ public class ReconItemIdValue extends ReconEntityIdValue implements ItemIdValue
super(recon, cellValue); super(recon, cellValue);
} }
@Override
public boolean isPlaceholder() {
return isNew();
}
@Override @Override
public String getEntityType() { public String getEntityType() {
return ET_ITEM; return ET_ITEM;

5
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconMediaInfoIdValue.java
View File

@ -34,6 +34,11 @@ public class ReconMediaInfoIdValue extends ReconEntityIdValue implements MediaIn
super(recon, cellValue); super(recon, cellValue);
} }
@Override
public boolean isPlaceholder() {
return isNew();
}
@Override @Override
public String getEntityType() { public String getEntityType() {
return EntityIdValue.ET_MEDIA_INFO; return EntityIdValue.ET_MEDIA_INFO;

5
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/ReconPropertyIdValue.java
View File

@ -33,6 +33,11 @@ public class ReconPropertyIdValue extends ReconEntityIdValue implements Property
super(recon, cellValue); super(recon, cellValue);
} }
@Override
public boolean isPlaceholder() {
return isNew();
}
@Override @Override
public String getEntityType() { public String getEntityType() {
return ET_PROPERTY; return ET_PROPERTY;

32
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedEntityIdValue.java
View File

@ -27,6 +27,8 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.wikidata.wdtk.datamodel.helpers.Hash; import org.wikidata.wdtk.datamodel.helpers.Hash;
import org.wikidata.wdtk.datamodel.implementation.EntityIdValueImpl;
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor; import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
@ -50,6 +52,26 @@ public abstract class SuggestedEntityIdValue implements PrefetchedEntityIdValue
_siteIRI = siteIRI; _siteIRI = siteIRI;
_label = label; _label = label;
} }
public static SuggestedEntityIdValue build(String id, String siteIRI, String label) {
String entityType = EntityIdValueImpl.guessEntityTypeFromId(id);
if (DatatypeIdValue.DT_ITEM.equals(entityType)) {
return new SuggestedItemIdValue(id, siteIRI, label);
} else if (DatatypeIdValue.DT_PROPERTY.equals(entityType)) {
return new SuggestedPropertyIdValue(id, siteIRI, label);
} else if (DatatypeIdValue.DT_MEDIA_INFO.equals(entityType)) {
return new SuggestedMediaInfoIdValue(id, siteIRI, label);
} else if (DatatypeIdValue.DT_LEXEME.equals(entityType)) {
return new SuggestedLexemeIdValue(id, siteIRI, label);
} else if (DatatypeIdValue.DT_FORM.equals(entityType)) {
return new SuggestedFormIdValue(id, siteIRI, label);
} else if (DatatypeIdValue.DT_SENSE.equals(entityType)) {
return new SuggestedSenseIdValue(id, siteIRI, label);
} else {
throw new IllegalArgumentException(
String.format("Unsupported datatype for suggested entity values: %s", entityType));
}
}
@Override @Override
@JsonProperty("id") @JsonProperty("id")
@ -80,6 +102,11 @@ public abstract class SuggestedEntityIdValue implements PrefetchedEntityIdValue
public String getIri() { public String getIri() {
return getSiteIri() + getId(); return getSiteIri() + getId();
} }
@Override
public boolean isPlaceholder() {
return false;
}
@Override @Override
public <T> T accept(ValueVisitor<T> valueVisitor) { public <T> T accept(ValueVisitor<T> valueVisitor) {
@ -99,5 +126,10 @@ public abstract class SuggestedEntityIdValue implements PrefetchedEntityIdValue
public int hashCode() { public int hashCode() {
return Hash.hashCode(this); return Hash.hashCode(this);
} }
@Override
public String toString() {
return getIri();
}
} }

233
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdateBuilder.java
View File

@ -1,233 +0,0 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.updates;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.jsoup.helper.Validate;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
/**
* Constructs a {@link ItemUpdate} incrementally.
*
* @author Antonin Delpeuch
*
*/
public class ItemUpdateBuilder {
private EntityIdValue qid;
private List<Statement> addedStatements;
private Set<Statement> deletedStatements;
private Set<MonolingualTextValue> labels;
private Set<MonolingualTextValue> labelsIfNew;
private Set<MonolingualTextValue> descriptions;
private Set<MonolingualTextValue> descriptionsIfNew;
private Set<MonolingualTextValue> aliases;
private boolean built;
/**
* Constructor.
*
* @param qid
* the subject of the document. It can be a reconciled item value for
* new items.
*/
public ItemUpdateBuilder(EntityIdValue qid) {
Validate.notNull(qid);
this.qid = qid;
this.addedStatements = new ArrayList<>();
this.deletedStatements = new HashSet<Statement>();
this.labels = new HashSet<MonolingualTextValue>();
this.labelsIfNew = new HashSet<MonolingualTextValue>();
this.descriptions = new HashSet<MonolingualTextValue>();
this.descriptionsIfNew = new HashSet<MonolingualTextValue>();
this.aliases = new HashSet<MonolingualTextValue>();
this.built = false;
}
/**
* Mark a statement for insertion. If it matches an existing statement, it will
* update the statement instead.
*
* @param statement
* the statement to add or update
*/
public ItemUpdateBuilder addStatement(Statement statement) {
Validate.isTrue(!built, "ItemUpdate has already been built");
addedStatements.add(statement);
return this;
}
/**
* Mark a statement for deletion. If no such statement exists, nothing will be
* deleted.
*
* @param statement
* the statement to delete
*/
public ItemUpdateBuilder deleteStatement(Statement statement) {
Validate.isTrue(!built, "ItemUpdate has already been built");
deletedStatements.add(statement);
return this;
}
/**
* Add a list of statement, as in {@link addStatement}.
*
* @param statements
* the statements to add
*/
public ItemUpdateBuilder addStatements(Set<Statement> statements) {
Validate.isTrue(!built, "ItemUpdate has already been built");
addedStatements.addAll(statements);
return this;
}
/**
* Delete a list of statements, as in {@link deleteStatement}.
*
* @param statements
* the statements to delete
*/
public ItemUpdateBuilder deleteStatements(Set<Statement> statements) {
Validate.isTrue(!built, "ItemUpdate has already been built");
deletedStatements.addAll(statements);
return this;
}
/**
* Adds a label to the item.
*
* @param label
* the label to add
* @param override
* whether the label should be added even if there is already a label in that language
*/
public ItemUpdateBuilder addLabel(MonolingualTextValue label, boolean override) {
Validate.isTrue(!built, "ItemUpdate has already been built");
if (override) {
labels.add(label);
} else {
labelsIfNew.add(label);
}
return this;
}
/**
* Adds a list of labels to the item.
*
* @param labels
* the labels to add
* @param override
* whether the label should be added even if there is already a label in that language
*/
public ItemUpdateBuilder addLabels(Set<MonolingualTextValue> labels, boolean override) {
Validate.isTrue(!built, "ItemUpdate has already been built");
if (override) {
this.labels.addAll(labels);
} else {
labelsIfNew.addAll(labels);
}
return this;
}
/**
* Adds a description to the item.
*
* @param description
* the description to add
* @param override
* whether the description should be added even if there is already a description in that language
*/
public ItemUpdateBuilder addDescription(MonolingualTextValue description, boolean override) {
Validate.isTrue(!built, "ItemUpdate has already been built");
if (override) {
descriptions.add(description);
} else {
descriptionsIfNew.add(description);
}
return this;
}
/**
* Adds a list of descriptions to the item.
*
* @param descriptions
* the descriptions to add
* @param override
* whether the description should be added even if there is already a description in that language
*/
public ItemUpdateBuilder addDescriptions(Set<MonolingualTextValue> descriptions, boolean override) {
Validate.isTrue(!built, "ItemUpdate has already been built");
if (override) {
this.descriptions.addAll(descriptions);
} else {
descriptionsIfNew.addAll(descriptions);
}
return this;
}
/**
* Adds an alias to the item. It will be added to any existing aliases in that
* language.
*
* @param alias
* the alias to add
*/
public ItemUpdateBuilder addAlias(MonolingualTextValue alias) {
Validate.isTrue(!built, "ItemUpdate has already been built");
aliases.add(alias);
return this;
}
/**
* Adds a list of aliases to the item. They will be added to any existing
* aliases in each language.
*
* @param aliases
* the aliases to add
*/
public ItemUpdateBuilder addAliases(Set<MonolingualTextValue> aliases) {
Validate.isTrue(!built, "ItemUpdate has already been built");
this.aliases.addAll(aliases);
return this;
}
/**
* Constructs the {@link ItemUpdate}.
*
* @return
*/
public TermedStatementEntityUpdate build() {
built = true;
return new TermedStatementEntityUpdate(qid, addedStatements, deletedStatements, labels, labelsIfNew, descriptions, descriptionsIfNew, aliases);
}
}

14
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/QuickStatementsUpdateScheduler.java
View File

@ -33,7 +33,7 @@ import java.util.Set;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue; import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.updates.TermedStatementEntityUpdate; import org.openrefine.wikidata.updates.TermedStatementEntityUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.openrefine.wikidata.updates.TermedStatementEntityUpdateBuilder;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.Statement;
@ -64,14 +64,14 @@ public class QuickStatementsUpdateScheduler implements UpdateScheduler {
*/ */
protected void splitUpdate(TermedStatementEntityUpdate update) protected void splitUpdate(TermedStatementEntityUpdate update)
throws ImpossibleSchedulingException { throws ImpossibleSchedulingException {
ItemUpdateBuilder remainingUpdateBuilder = new ItemUpdateBuilder(update.getItemId()) TermedStatementEntityUpdateBuilder remainingUpdateBuilder = new TermedStatementEntityUpdateBuilder(update.getItemId())
.addLabels(update.getLabels(), true) .addLabels(update.getLabels(), true)
.addLabels(update.getLabelsIfNew(), false) .addLabels(update.getLabelsIfNew(), false)
.addDescriptions(update.getDescriptions(), true) .addDescriptions(update.getDescriptions(), true)
.addDescriptions(update.getDescriptionsIfNew(), false) .addDescriptions(update.getDescriptionsIfNew(), false)
.addAliases(update.getAliases()) .addAliases(update.getAliases())
.deleteStatements(update.getDeletedStatements()); .deleteStatements(update.getDeletedStatements());
Map<ItemIdValue, ItemUpdateBuilder> referencingUpdates = new HashMap<>(); Map<ItemIdValue, TermedStatementEntityUpdateBuilder> referencingUpdates = new HashMap<>();
for (Statement statement : update.getAddedStatements()) { for (Statement statement : update.getAddedStatements()) {
Set<ReconItemIdValue> pointers = extractor.extractPointers(statement); Set<ReconItemIdValue> pointers = extractor.extractPointers(statement);
@ -79,9 +79,9 @@ public class QuickStatementsUpdateScheduler implements UpdateScheduler {
remainingUpdateBuilder.addStatement(statement); remainingUpdateBuilder.addStatement(statement);
} else if (pointers.size() == 1 && !update.isNew()) { } else if (pointers.size() == 1 && !update.isNew()) {
ItemIdValue pointer = pointers.stream().findFirst().get(); ItemIdValue pointer = pointers.stream().findFirst().get();
ItemUpdateBuilder referencingBuilder = referencingUpdates.get(pointer); TermedStatementEntityUpdateBuilder referencingBuilder = referencingUpdates.get(pointer);
if (referencingBuilder == null) { if (referencingBuilder == null) {
referencingBuilder = new ItemUpdateBuilder(update.getItemId()); referencingBuilder = new TermedStatementEntityUpdateBuilder(update.getItemId());
} }
referencingBuilder.addStatement(statement); referencingBuilder.addStatement(statement);
referencingUpdates.put(pointer, referencingBuilder); referencingUpdates.put(pointer, referencingBuilder);
@ -98,7 +98,7 @@ public class QuickStatementsUpdateScheduler implements UpdateScheduler {
pointerFreeUpdates.add(pointerFree); pointerFreeUpdates.add(pointerFree);
} }
// Add the other updates to the map // Add the other updates to the map
for (Entry<ItemIdValue, ItemUpdateBuilder> entry : referencingUpdates.entrySet()) { for (Entry<ItemIdValue, TermedStatementEntityUpdateBuilder> entry : referencingUpdates.entrySet()) {
TermedStatementEntityUpdate pointerUpdate = entry.getValue().build(); TermedStatementEntityUpdate pointerUpdate = entry.getValue().build();
UpdateSequence pointerUpdatesForKey = pointerUpdates.get(entry.getKey()); UpdateSequence pointerUpdatesForKey = pointerUpdates.get(entry.getKey());
if (pointerUpdatesForKey == null) { if (pointerUpdatesForKey == null) {
@ -135,7 +135,7 @@ public class QuickStatementsUpdateScheduler implements UpdateScheduler {
// (this is just for the sake of correctness, it would be bad to do that // (this is just for the sake of correctness, it would be bad to do that
// as the items would remain blank in this batch). // as the items would remain blank in this batch).
for (ItemIdValue missingId : mentionedNewEntities) { for (ItemIdValue missingId : mentionedNewEntities) {
fullSchedule.add(new ItemUpdateBuilder(missingId).build()); fullSchedule.add(new TermedStatementEntityUpdateBuilder(missingId).build());
fullSchedule.addAll(pointerUpdates.get(missingId).getUpdates()); fullSchedule.addAll(pointerUpdates.get(missingId).getUpdates());
} }
return fullSchedule; return fullSchedule;

6
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/UpdateSequence.java
View File

@ -48,7 +48,7 @@ public class UpdateSequence {
/** /**
* An index to keep track of where each item is touched in the sequence * An index to keep track of where each item is touched in the sequence
*/ */
private Map<ItemIdValue, Integer> index = new HashMap<>(); private Map<EntityIdValue, Integer> index = new HashMap<>();
/** /**
* Adds a new update to the list, merging it with any existing one with the same * Adds a new update to the list, merging it with any existing one with the same
@ -63,7 +63,7 @@ public class UpdateSequence {
TermedStatementEntityUpdate oldUpdate = updates.get(i); TermedStatementEntityUpdate oldUpdate = updates.get(i);
updates.set(i, oldUpdate.merge(update)); updates.set(i, oldUpdate.merge(update));
} else { } else {
index.put((ItemIdValue) subject, updates.size()); index.put(subject, updates.size());
updates.add(update); updates.add(update);
} }
} }
@ -78,7 +78,7 @@ public class UpdateSequence {
/** /**
* @return the set of touched subjects * @return the set of touched subjects
*/ */
public Set<ItemIdValue> getSubjects() { public Set<EntityIdValue> getSubjects() {
return index.keySet(); return index.keySet();
} }
} }

8
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/WikibaseAPIUpdateScheduler.java
View File

@ -31,7 +31,7 @@ import java.util.stream.Collectors;
import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue; import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue;
import org.openrefine.wikidata.updates.TermedStatementEntityUpdate; import org.openrefine.wikidata.updates.TermedStatementEntityUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.openrefine.wikidata.updates.TermedStatementEntityUpdateBuilder;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.Statement;
@ -84,7 +84,7 @@ public class WikibaseAPIUpdateScheduler implements UpdateScheduler {
Set<ItemIdValue> unseenPointers = new HashSet<>(allPointers); Set<ItemIdValue> unseenPointers = new HashSet<>(allPointers);
unseenPointers.removeAll(pointerFreeUpdates.getSubjects()); unseenPointers.removeAll(pointerFreeUpdates.getSubjects());
result.addAll(unseenPointers.stream().map(e -> new ItemUpdateBuilder(e).build()).collect(Collectors.toList())); result.addAll(unseenPointers.stream().map(e -> new TermedStatementEntityUpdateBuilder(e).build()).collect(Collectors.toList()));
// Part 2: add all the pointer full updates // Part 2: add all the pointer full updates
result.addAll(pointerFullUpdates.getUpdates()); result.addAll(pointerFullUpdates.getUpdates());
@ -98,14 +98,14 @@ public class WikibaseAPIUpdateScheduler implements UpdateScheduler {
* @param update * @param update
*/ */
protected void splitUpdate(TermedStatementEntityUpdate update) { protected void splitUpdate(TermedStatementEntityUpdate update) {
ItemUpdateBuilder pointerFreeBuilder = new ItemUpdateBuilder(update.getItemId()) TermedStatementEntityUpdateBuilder pointerFreeBuilder = new TermedStatementEntityUpdateBuilder(update.getItemId())
.addLabels(update.getLabels(), true) .addLabels(update.getLabels(), true)
.addLabels(update.getLabelsIfNew(), false) .addLabels(update.getLabelsIfNew(), false)
.addDescriptions(update.getDescriptions(), true) .addDescriptions(update.getDescriptions(), true)
.addDescriptions(update.getDescriptionsIfNew(), false) .addDescriptions(update.getDescriptionsIfNew(), false)
.addAliases(update.getAliases()) .addAliases(update.getAliases())
.deleteStatements(update.getDeletedStatements()); .deleteStatements(update.getDeletedStatements());
ItemUpdateBuilder pointerFullBuilder = new ItemUpdateBuilder(update.getItemId()); TermedStatementEntityUpdateBuilder pointerFullBuilder = new TermedStatementEntityUpdateBuilder(update.getItemId());
for (Statement statement : update.getAddedStatements()) { for (Statement statement : update.getAddedStatements()) {
Set<ReconItemIdValue> pointers = extractor.extractPointers(statement); Set<ReconItemIdValue> pointers = extractor.extractPointers(statement);

19
OpenRefine/extensions/wikidata/src/org/openrefine/wikidata/utils/EntityCache.java
View File

@ -85,6 +85,12 @@ public class EntityCache {
return cache.apply(id.getId()); return cache.apply(id.getId());
} }
/**
* Get an entity cache for a given Wikibase instance.
* @param siteIri
* @param mediaWikiApiEndpoint
* @return
*/
public static EntityCache getEntityCache(String siteIri, String mediaWikiApiEndpoint) { public static EntityCache getEntityCache(String siteIri, String mediaWikiApiEndpoint) {
EntityCache entityCache = entityCacheMap.get(siteIri); EntityCache entityCache = entityCacheMap.get(siteIri);
if (entityCache == null) { if (entityCache == null) {
@ -93,6 +99,15 @@ public class EntityCache {
} }
return entityCache; return entityCache;
} }
/**
* Provided for testability.
* @param siteIri
* @param cache
*/
public static void setEntityCache(String siteIri, EntityCache cache) {
entityCacheMap.put(siteIri, cache);
}
public List<EntityDocument> getMultipleDocuments(List<EntityIdValue> entityIds) throws ExecutionException { public List<EntityDocument> getMultipleDocuments(List<EntityIdValue> entityIds) throws ExecutionException {
List<String> ids = entityIds.stream().map(entityId -> entityId.getId()).collect(Collectors.toList()); List<String> ids = entityIds.stream().map(entityId -> entityId.getId()).collect(Collectors.toList());
@ -102,4 +117,8 @@ public class EntityCache {
public static EntityDocument getEntityDocument(String entityPrefix, String mediaWikiApiEndpoint, EntityIdValue id) { public static EntityDocument getEntityDocument(String entityPrefix, String mediaWikiApiEndpoint, EntityIdValue id) {
return getEntityCache(entityPrefix, mediaWikiApiEndpoint).get(id); return getEntityCache(entityPrefix, mediaWikiApiEndpoint).get(id);
} }
public static void removeEntityCache(String siteIri) {
entityCacheMap.remove(siteIri);
}
} }

46
OpenRefine/extensions/wikidata/tests/data/manifest/commons-manifest-v2.0.json Normal file
View File

@ -0,0 +1,46 @@
{
"version": "2.0",
"mediawiki": {
"name": "Wikimedia Commons",
"root": "https://commons.wikimedia.org/wiki/",
"main_page": "https://commons.wikimedia.org/wiki/Main_Page",
"api": "https://commons.wikimedia.org/w/api.php"
},
"wikibase": {
"site_iri": "https://commons.wikimedia.org/entity/",
"maxlag": 5,
"properties": {
"instance_of": "P31",
"subclass_of": "P279"
},
"constraints": {
"property_constraint_pid": "P2302",
"exception_to_constraint_pid": "P2303",
"constraint_status_pid": "P2316",
"mandatory_constraint_qid": "Q21502408",
"suggestion_constraint_qid": "Q62026391",
"distinct_values_constraint_qid": "Q21502410"
}
},
"oauth": {
"registration_page": "https://commons.wikimedia.org/wiki/Special:OAuthConsumerRegistration/propose"
},
"entity_types": {
"item": {
"site_iri": "http://www.wikidata.org/entity/",
"reconciliation_endpoint": "https://wikidata.reconci.link/${lang}/api",
"mediawiki_api": "https://www.wikidata.org/w/api.php"
},
"property": {
"site_iri": "http://www.wikidata.org/entity/",
"mediawiki_api": "https://www.wikidata.org/w/api.php"
},
"mediainfo": {
"site_iri": "https://commons.wikimedia.org/entity/",
"reconciliation_endpoint": "https://commonsreconcile.toolforge.org/${lang}/api"
}
},
"editgroups": {
"url_schema": "([[:toollabs:editgroups-commons/b/OR/${batch_id}|details]])"
}
}

3
OpenRefine/extensions/wikidata/tests/data/operations/save-schema.json
View File

@ -30,6 +30,7 @@
} }
], ],
"siteIri": "http://www.wikidata.org/entity/", "siteIri": "http://www.wikidata.org/entity/",
"mediaWikiApiEndpoint":"https://www.wikidata.org/w/api.php" "mediaWikiApiEndpoint":"https://www.wikidata.org/w/api.php",
"entityTypeSiteIRI": {}
} }
} }

2
OpenRefine/extensions/wikidata/tests/data/schema/history_of_medicine_normalized.json
View File

@ -1 +1 @@
{"itemDocuments":[{"subject":{"type":"wbitemvariable","columnName":"name"},"statementGroups":[{"property":{"datatype":"wikibase-item","pid":"P39","label":"position held","type":"wbpropconstant"},"statements":[{"references":[{"snaks":[{"prop":{"datatype":"wikibase-item","pid":"P248","label":"stated in","type":"wbpropconstant"},"value":{"label":"The History of The Royal Society of Medicine","type":"wbitemconstant","qid":"Q42036099"}},{"prop":{"datatype":"string","pid":"P304","label":"page","type":"wbpropconstant"},"value":{"type":"wbstringconstant","value":"330-333"}}]}],"qualifiers":[{"prop":{"datatype":"wikibase-item","pid":"P642","label":"of","type":"wbpropconstant"},"value":{"label":"History of Medicine Society","type":"wbitemconstant","qid":"Q37461404"}},{"prop":{"datatype":"time","pid":"P580","label":"start date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"start"}},{"prop":{"datatype":"time","pid":"P582","label":"end date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"end"}}],"value":{"label":"president","type":"wbitemconstant","qid":"Q30461"}}]}],"nameDescs":[]},{"subject":{"label":"History of Medicine Society","type":"wbitemconstant","qid":"Q37461404"},"statementGroups":[{"property":{"datatype":"wikibase-item","pid":"P488","label":"chairperson","type":"wbpropconstant"},"statements":[{"references":[{"snaks":[{"prop":{"datatype":"wikibase-item","pid":"P248","label":"stated in","type":"wbpropconstant"},"value":{"label":"The History of The Royal Society of Medicine","type":"wbitemconstant","qid":"Q42036099"}},{"prop":{"datatype":"string","pid":"P304","label":"page","type":"wbpropconstant"},"value":{"type":"wbstringconstant","value":"330-333"}}]}],"qualifiers":[{"prop":{"datatype":"time","pid":"P580","label":"start date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"start"}},{"prop":{"datatype":"time","pid":"P582","label":"end date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"end"}}],"value":{"type":"wbitemvariable","columnName":"name"}}]}],"nameDescs":[]}],"siteIri":"http://www.wikidata.org/entity/","mediaWikiApiEndpoint":"https://www.wikidata.org/w/api.php"} {"itemDocuments":[{"subject":{"type":"wbitemvariable","columnName":"name"},"statementGroups":[{"property":{"datatype":"wikibase-item","pid":"P39","label":"position held","type":"wbpropconstant"},"statements":[{"references":[{"snaks":[{"prop":{"datatype":"wikibase-item","pid":"P248","label":"stated in","type":"wbpropconstant"},"value":{"label":"The History of The Royal Society of Medicine","type":"wbitemconstant","qid":"Q42036099"}},{"prop":{"datatype":"string","pid":"P304","label":"page","type":"wbpropconstant"},"value":{"type":"wbstringconstant","value":"330-333"}}]}],"qualifiers":[{"prop":{"datatype":"wikibase-item","pid":"P642","label":"of","type":"wbpropconstant"},"value":{"label":"History of Medicine Society","type":"wbitemconstant","qid":"Q37461404"}},{"prop":{"datatype":"time","pid":"P580","label":"start date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"start"}},{"prop":{"datatype":"time","pid":"P582","label":"end date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"end"}}],"value":{"label":"president","type":"wbitemconstant","qid":"Q30461"}}]}],"nameDescs":[]},{"subject":{"label":"History of Medicine Society","type":"wbitemconstant","qid":"Q37461404"},"statementGroups":[{"property":{"datatype":"wikibase-item","pid":"P488","label":"chairperson","type":"wbpropconstant"},"statements":[{"references":[{"snaks":[{"prop":{"datatype":"wikibase-item","pid":"P248","label":"stated in","type":"wbpropconstant"},"value":{"label":"The History of The Royal Society of Medicine","type":"wbitemconstant","qid":"Q42036099"}},{"prop":{"datatype":"string","pid":"P304","label":"page","type":"wbpropconstant"},"value":{"type":"wbstringconstant","value":"330-333"}}]}],"qualifiers":[{"prop":{"datatype":"time","pid":"P580","label":"start date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"start"}},{"prop":{"datatype":"time","pid":"P582","label":"end date","type":"wbpropconstant"},"value":{"type":"wbdatevariable","columnName":"end"}}],"value":{"type":"wbitemvariable","columnName":"name"}}]}],"nameDescs":[]}],"siteIri":"http://www.wikidata.org/entity/","mediaWikiApiEndpoint":"https://www.wikidata.org/w/api.php","entityTypeSiteIRI":{}}

2
OpenRefine/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/CommandTest.java
View File

@ -21,6 +21,7 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE. * SOFTWARE.
******************************************************************************/ ******************************************************************************/
package org.openrefine.wikidata.commands; package org.openrefine.wikidata.commands;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
@ -49,7 +50,6 @@ public abstract class CommandTest extends WikidataRefineTest {
protected StringWriter writer = null; protected StringWriter writer = null;
protected Command command = null; protected Command command = null;
@BeforeMethod(alwaysRun = true) @BeforeMethod(alwaysRun = true)
public void setUpProject() { public void setUpProject() {

Some files were not shown because too many files have changed in this diff Show More