s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #2323 from mrcsparker/sqlite-support

Browse Source 
Added SQLite support
This commit is contained in:
Antonin Delpeuch 2020-02-25 01:47:37 +00:00 committed by GitHub
commit 76eded0d9d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 774 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
extensions/database/module/scripts/index/database-import-controller.js
View File

@ -2,28 +2,28 @@
* Copyright (c) 2017, Tony Opara
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@ -48,7 +48,7 @@ $.i18n().load(dictionary, lang);
Refine.DatabaseImportController = function(createProjectUI) {
this._createProjectUI = createProjectUI;
this._parsingPanel = createProjectUI.addCustomPanel();
createProjectUI.addSourceSelectionUI({
@ -64,7 +64,7 @@ Refine.DatabaseImportController.prototype.startImportingDocument = function(quer
var dismiss = DialogSystem.showBusy($.i18n('database-import/preparing'));
//alert(queryInfo.query);
var self = this;
Refine.postCSRF(
"command/core/create-importing-job",
null,
@ -77,17 +77,17 @@ Refine.DatabaseImportController.prototype.startImportingDocument = function(quer
"csrf_token": token
}),
queryInfo,
function(data2) {
dismiss();
if (data2.status == 'ok') {
self._queryInfo = queryInfo;
self._jobID = data.jobID;
self._options = data2.options;
self._showParsingPanel();
} else {
alert(data2.message);
}
@ -102,9 +102,9 @@ Refine.DatabaseImportController.prototype.startImportingDocument = function(quer
Refine.DatabaseImportController.prototype.getOptions = function() {
var options = {
};
var parseIntDefault = function(s, def) {
try {
var n = parseInt(s);
@ -117,7 +117,7 @@ Refine.DatabaseImportController.prototype.getOptions = function() {
return def;
};
if (this._parsingPanelElmts.skipCheckbox[0].checked) {
options.skipDataLines = parseIntDefault(this._parsingPanelElmts.skipInput[0].value, 0);
} else {
@ -136,10 +136,10 @@ Refine.DatabaseImportController.prototype.getOptions = function() {
Refine.DatabaseImportController.prototype._showParsingPanel = function() {
var self = this;
this._parsingPanel.unbind().empty().html(
DOM.loadHTML("database",'scripts/index/database-parsing-panel.html'));
this._parsingPanelElmts = DOM.bind(this._parsingPanel);
this._parsingPanelElmts.startOverButton.html($.i18n('database-parsing/start-over'));
@ -155,11 +155,11 @@ Refine.DatabaseImportController.prototype._showParsingPanel = function() {
this._parsingPanelElmts.database_limit.html($.i18n('database-parsing/limit'));
this._parsingPanelElmts.database_store_row.html($.i18n('database-parsing/store-row'));
this._parsingPanelElmts.database_store_cell.html($.i18n('database-parsing/store-cell'));
if (this._parsingPanelResizer) {
$(window).unbind('resize', this._parsingPanelResizer);
}
this._parsingPanelResizer = function() {
var elmts = self._parsingPanelElmts;
var width = self._parsingPanel.width();
@ -184,27 +184,27 @@ Refine.DatabaseImportController.prototype._showParsingPanel = function() {
.css("width", (width - DOM.getHPaddings(elmts.controlPanel)) + "px")
.css("height", (controlPanelHeight - DOM.getVPaddings(elmts.controlPanel)) + "px");
};
$(window).resize(this._parsingPanelResizer);
this._parsingPanelResizer();
this._parsingPanelElmts.startOverButton.click(function() {
// explicitly cancel the import job
Refine.CreateProjectUI.cancelImportingJob(self._jobID);
delete self._jobID;
delete self._options;
self._createProjectUI.showSourceSelectionPanel();
});
this._parsingPanelElmts.createProjectButton.click(function() { self._createProject(); });
this._parsingPanelElmts.previewButton.click(function() { self._updatePreview(); });
//alert("datetime::" + $.now());
//this._parsingPanelElmts.projectNameInput[0].value = this._queryInfo.connectionName + "_" + this._queryInfo.databaseUser + "_" + $.now();
this._parsingPanelElmts.projectNameInput[0].value = this._queryInfo.databaseServer + "_" + this._queryInfo.initialDatabase + "_" + $.now();
if (this._options.limit > 0) {
this._parsingPanelElmts.limitCheckbox.prop("checked", true);
this._parsingPanelElmts.limitInput[0].value = this._options.limit.toString();
@ -225,7 +225,7 @@ Refine.DatabaseImportController.prototype._showParsingPanel = function() {
};
this._parsingPanel.find("input").bind("change", onChange);
this._parsingPanel.find("select").bind("change", onChange);
this._createProjectUI.showCustomPanel(this._parsingPanel);
this._updatePreview();
};
@ -259,9 +259,9 @@ Refine.DatabaseImportController.prototype._updatePreview = function() {
"subCommand": "parse-preview",
"csrf_token": token
}),
this._queryInfo,
self._queryInfo,
function(result) {
if (result.status == "ok") {
self._getPreviewData(function(projectData) {
@ -271,18 +271,18 @@ Refine.DatabaseImportController.prototype._updatePreview = function() {
new Refine.PreviewTable(projectData, self._parsingPanelElmts.dataPanel.unbind().empty());
});
} else {
alert('Errors:\n' + (result.message) ? result.message : Refine.CreateProjectUI.composeErrorMessage(job));
self._parsingPanelElmts.progressPanel.hide();
Refine.CreateProjectUI.cancelImportingJob(self._jobID);
delete self._jobID;
delete self._options;
self._createProjectUI.showSourceSelectionPanel();
}
},
"json"
@ -303,7 +303,7 @@ Refine.DatabaseImportController.prototype._getPreviewData = function(callback, n
result[n] = data[n];
}
}
$.post(
"command/core/get-rows?" + $.param({
"importingJobID" : self._jobID,
@ -329,11 +329,11 @@ Refine.DatabaseImportController.prototype._createProject = function() {
this._parsingPanelElmts.projectNameInput.focus();
return;
}
var self = this;
var options = this.getOptions();
options.projectName = projectName;
this._queryInfo.options = JSON.stringify(options);
Refine.wrapCSRF(function(token) {
$.post(
@ -343,7 +343,7 @@ Refine.DatabaseImportController.prototype._createProject = function() {
"subCommand": "create-project",
"csrf_token": token
}),
this._queryInfo,
self._queryInfo,
function(o) {
if (o.status == 'error') {
alert(o.message);
@ -385,4 +385,4 @@ Refine.DatabaseImportController.prototype._createProject = function() {
"json"
);
});
};
};

1
extensions/database/module/scripts/index/database-import-form.html
View File

@ -38,6 +38,7 @@
<option value="postgresql">PostgreSQL</option>
<option value="mysql">MySQL</option>
<option value="mariadb">MariaDB</option>
<option value="sqlite">SQLite</option>
</select>
</div>

10
extensions/database/module/scripts/index/database-source-ui.js
View File

@ -100,7 +100,11 @@ Refine.DatabaseSourceUI.prototype.attachUI = function(body) {
$( "#databasePort" ).val("3306");
}else if(type === "mariadb"){
$( "#databaseUser" ).val("root");
$( "#databasePort" ).val("3306");
$( "#databasePort" ).val("3306");
}else if(type === "sqlite"){
$( "#databaseUser" ).val("na");
$( "#databasePort" ).val("0");
$( "#databaseHost" ).val("na");
}else{
$( "#databaseUser" ).val("root");
$( "#databasePort" ).val("3306");
@ -267,7 +271,7 @@ Refine.DatabaseSourceUI.prototype._executeQuery = function(jdbcQueryInfo) {
//remove start line
var dismiss = DialogSystem.showBusy($.i18n('database-import/checking'));
Refine.postCSRF(
"command/database/test-query",
jdbcQueryInfo,
@ -440,7 +444,7 @@ Refine.DatabaseSourceUI.prototype._validateNewConnectionForm = function() {
var databaseHostTestResult = alphaNumRE.test(databaseHostInput);
var databasePortTestResult = numRE.test(databasePortInput);
var databaseUserTestResult = alphaNumRE.test(databaseUserInput);
if(alphaNumConnNameTestResult == false){
window.alert($.i18n('database-source/alert-conn-name-invalid-character'));
return false;

5
extensions/database/pom.xml
View File

@ -150,6 +150,11 @@
<artifactId>mariadb-java-client</artifactId>
<version>2.2.6</version>
</dependency>
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<version>3.30.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>

2
extensions/database/src/com/google/refine/extension/database/DatabaseService.java
View File

@ -33,6 +33,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.google.refine.extension.database.sqlite.SQLiteDatabaseService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -57,6 +58,7 @@ public abstract class DatabaseService {
DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance());
DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance());
DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance());
} catch (Exception e) {
logger.error("Exception occurred while trying to prepare databases!", e);

140
extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManager.java Normal file
View File

@ -0,0 +1,140 @@
/*
* Copyright (c) 2020, Chris Parker
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.extension.database.sqlite;
import com.google.refine.extension.database.DatabaseConfiguration;
import com.google.refine.extension.database.DatabaseServiceException;
import com.google.refine.extension.database.SQLType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
public class SQLiteConnectionManager {
private static final Logger logger = LoggerFactory.getLogger("SQLiteConnectionManager");
private static SQLiteConnectionManager instance;
private final SQLType type;
private Connection connection;
private SQLiteConnectionManager() {
type = SQLType.forName(SQLiteDatabaseService.DB_NAME);
}
/**
* Create a new instance of this connection manager.
*
* @return an instance of the manager
*/
public static SQLiteConnectionManager getInstance() {
if (instance == null) {
if (logger.isDebugEnabled()) {
logger.debug("::Creating new SQLite ConnectionManager ::");
}
instance = new SQLiteConnectionManager();
}
return instance;
}
public static String getDatabaseUrl(DatabaseConfiguration dbConfig) {
return "jdbc:" + dbConfig.getDatabaseType().toLowerCase() + ":" + dbConfig.getDatabaseName();
}
/**
* Get the SQL Database type.
*
* @return the type
*/
public SQLType getType() {
return this.type;
}
/**
* testConnection
*
* @param dbConfig
* @return boolean
*/
public boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException {
try {
boolean connResult = false;
Connection conn = getConnection(dbConfig);
if (conn != null) {
connResult = true;
conn.close();
}
return connResult;
} catch (SQLException e) {
logger.error("Test connection Failed!", e);
throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
}
}
/**
* Get a connection form the connection pool.
*
* @return connection from the pool
*/
public Connection getConnection(DatabaseConfiguration databaseConfiguration) throws DatabaseServiceException {
try {
if (connection != null) {
connection.close();
}
Class.forName(type.getClassPath());
String dbURL = getDatabaseUrl(databaseConfiguration);
connection = DriverManager.getConnection(dbURL);
logger.debug("*** Acquired New connection for ::{} **** ", dbURL);
return connection;
} catch (ClassNotFoundException e) {
logger.error("Jdbc Driver not found", e);
throw new DatabaseServiceException(e.getMessage());
} catch (SQLException e) {
logger.error("SQLException::Couldn't get a Connection!", e);
throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
}
}
public void shutdown() {
if (connection != null) {
try {
connection.close();
} catch (SQLException e) {
logger.warn("Non-Managed connection could not be closed. Whoops!", e);
}
}
}
}

229
extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseService.java Normal file
View File

@ -0,0 +1,229 @@
/*
* Copyright (c) 2020, Chris Parker
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.extension.database.sqlite;
import com.google.refine.extension.database.SQLType;
import com.google.refine.extension.database.*;
import com.google.refine.extension.database.model.DatabaseColumn;
import com.google.refine.extension.database.model.DatabaseInfo;
import com.google.refine.extension.database.model.DatabaseRow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
public class SQLiteDatabaseService extends DatabaseService {
public static final String DB_NAME = "sqlite";
public static final String DB_DRIVER = "org.sqlite.JDBC";
private static final Logger logger = LoggerFactory.getLogger("SQLiteDatabaseService");
private static SQLiteDatabaseService instance;
public static SQLiteDatabaseService getInstance() {
if (instance == null) {
SQLType.registerSQLDriver(DB_NAME, DB_DRIVER, false);
instance = new SQLiteDatabaseService();
if (logger.isDebugEnabled()) {
logger.debug("SQLiteDatabaseService Instance: {}", instance);
}
}
return instance;
}
@Override
protected String getDatabaseUrl(DatabaseConfiguration dbConfig) {
return SQLiteConnectionManager.getDatabaseUrl(dbConfig);
}
@Override
public Connection getConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException {
return SQLiteConnectionManager.getInstance().getConnection(dbConfig);
}
@Override
public boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException {
return SQLiteConnectionManager.getInstance().testConnection(dbConfig);
}
@Override
public DatabaseInfo connect(DatabaseConfiguration dbConfig) throws DatabaseServiceException {
return getMetadata(dbConfig);
}
/**
* @param dbConfig
* @return
* @throws DatabaseServiceException
*/
private DatabaseInfo getMetadata(DatabaseConfiguration connectionInfo) throws DatabaseServiceException {
try {
Connection connection = SQLiteConnectionManager.getInstance().getConnection(connectionInfo);
if (connection != null) {
java.sql.DatabaseMetaData metadata = connection.getMetaData();
int dbMajorVersion = metadata.getDatabaseMajorVersion();
int dbMinorVersion = metadata.getDatabaseMinorVersion();
String dbProductVersion = metadata.getDatabaseProductVersion();
String dbProductName = metadata.getDatabaseProductName();
DatabaseInfo dbInfo = new DatabaseInfo();
dbInfo.setDatabaseMajorVersion(dbMajorVersion);
dbInfo.setDatabaseMinorVersion(dbMinorVersion);
dbInfo.setDatabaseProductVersion(dbProductVersion);
dbInfo.setDatabaseProductName(dbProductName);
return dbInfo;
}
} catch (SQLException e) {
logger.error("SQLException::", e);
throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
}
return null;
}
@Override
public DatabaseInfo executeQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException {
try {
Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig);
Statement statement = connection.createStatement();
ResultSet queryResult = statement.executeQuery(query);
ResultSetMetaData metadata = queryResult.getMetaData();
int columnCount = metadata.getColumnCount();
ArrayList<DatabaseColumn> columns = new ArrayList<>(columnCount);
for (int i = 1; i <= columnCount; i++) {
DatabaseColumn dc = new DatabaseColumn(metadata.getColumnName(i), metadata.getColumnLabel(i),
DatabaseUtils.getDbColumnType(metadata.getColumnType(i)),
metadata.getColumnDisplaySize(i));
columns.add(dc);
}
int index = 0;
List<DatabaseRow> rows = new ArrayList<>();
while (queryResult.next()) {
DatabaseRow row = new DatabaseRow();
row.setIndex(index);
List<String> values = new ArrayList<>(columnCount);
for (int i = 1; i <= columnCount; i++) {
values.add(queryResult.getString(i));
}
row.setValues(values);
rows.add(row);
index++;
}
DatabaseInfo dbInfo = new DatabaseInfo();
dbInfo.setColumns(columns);
dbInfo.setRows(rows);
return dbInfo;
} catch (SQLException e) {
logger.error("SQLException::", e);
throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
} finally {
SQLiteConnectionManager.getInstance().shutdown();
}
}
@Override
public DatabaseInfo testQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException {
Statement statement = null;
ResultSet queryResult = null;
try {
Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig);
statement = connection.createStatement();
queryResult = statement.executeQuery(query);
return new DatabaseInfo();
} catch (SQLException e) {
logger.error("SQLException::", e);
throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
} finally {
try {
if (queryResult != null) {
queryResult.close();
}
if (statement != null) {
statement.close();
}
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
SQLiteConnectionManager.getInstance().shutdown();
}
}
@Override
public List<DatabaseColumn> getColumns(DatabaseConfiguration dbConfig, String query)
throws DatabaseServiceException {
try {
Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig);
Statement statement = connection.createStatement();
ResultSet queryResult = statement.executeQuery(query);
ResultSetMetaData metadata = queryResult.getMetaData();
int columnCount = metadata.getColumnCount();
ArrayList<DatabaseColumn> columns = new ArrayList<>(columnCount);
for (int i = 1; i <= columnCount; i++) {
DatabaseColumn dc = new DatabaseColumn(metadata.getColumnName(i), metadata.getColumnLabel(i),
DatabaseUtils.getDbColumnType(metadata.getColumnType(i)),
metadata.getColumnDisplaySize(i));
columns.add(dc);
}
return columns;
} catch (SQLException e) {
logger.error("SQLException::", e);
throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
}
}
@Override
public List<DatabaseRow> getRows(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException {
try {
Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig);
Statement statement = connection.createStatement();
statement.setFetchSize(10);
ResultSet queryResult = statement.executeQuery(query);
ResultSetMetaData metadata = queryResult.getMetaData();
int columnCount = metadata.getColumnCount();
int index = 0;
List<DatabaseRow> rows = new ArrayList<>();
while (queryResult.next()) {
DatabaseRow row = new DatabaseRow();
row.setIndex(index);
List<String> values = new ArrayList<>(columnCount);
for (int i = 1; i <= columnCount; i++) {
values.add(queryResult.getString(i));
}
row.setValues(values);
rows.add(row);
index++;
}
return rows;
} catch (SQLException e) {
logger.error("SQLException::", e);
throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
}
}
}

10
extensions/database/tests/conf/appveyor_tests.xml
View File

@ -24,6 +24,14 @@
<parameter name = "mariaDbUser" value="root"/>
<parameter name = "mariaDbPassword" value=""/>
<parameter name = "mariaDbTestTable" value="test_table"/>
<parameter name = "sqliteDbName" value="extension_test_db.sqlite"/>
<parameter name = "sqliteDbHost" value=""/>
<parameter name = "sqliteDbPort" value=""/>
<parameter name = "sqliteDbUser" value=""/>
<parameter name = "sqliteDbPassword" value=""/>
<parameter name = "sqliteDbTestTable" value="test_table"/>
<test name="DatabaseExtensionUnitTest">
<groups>
<run>
@ -45,6 +53,8 @@
<class name="com.google.refine.extension.database.mysql.MySQLDatabaseServiceTest"/>
<class name="com.google.refine.extension.database.pgsql.PgSQLConnectionManagerTest"/>
<class name="com.google.refine.extension.database.pgsql.PgSQLDatabaseServiceTest" />
<class name="com.google.refine.extension.database.sqlite.SQLiteConnectionManagerTest"/>
<class name="com.google.refine.extension.database.sqlite.SQLiteDatabaseServiceTest" />
</classes>
</test>
</suite>

9
extensions/database/tests/conf/tests.xml
View File

@ -28,12 +28,21 @@
<parameter name = "mariaDbUser" value="root"/>
<parameter name = "mariaDbPassword" value=""/>
<parameter name = "mariaDbTestTable" value="test_table"/>
<parameter name = "sqliteDbName" value="extension_test_db.sqlite"/>
<parameter name = "sqliteDbHost" value=""/>
<parameter name = "sqliteDbPort" value=""/>
<parameter name = "sqliteDbUser" value=""/>
<parameter name = "sqliteDbPassword" value=""/>
<parameter name = "sqliteDbTestTable" value="test_table"/>
<test name="DatabaseExtensionUnitTest">
<groups>
<run>
<exclude name="requiresMySQL"/>
<exclude name="requiresPgSQL"/>
<exclude name="requiresMariaDB" />
<exclude name="requiresSQLite" />
</run>
</groups> <!--
<classes>

15
extensions/database/tests/conf/travis-sqlite.sql Normal file
View File

@ -0,0 +1,15 @@
CREATE TABLE IF NOT EXISTS test_table (
id integer NOT NULL,
ue_id char(8) NOT NULL,
start_time timestamp NOT NULL,
end_date date DEFAULT NULL,
bytes_upload integer NOT NULL,
bytes_download integer NOT NULL,
mcc char(3) DEFAULT NULL,
mnc char(3) NOT NULL,
lac varchar(11) DEFAULT NULL,
imei char(16) NOT NULL
);
INSERT INTO test_table(id, ue_id, start_time, end_date, bytes_upload, bytes_download, mcc, mnc, lac, imei)
VALUES (1, '11100022', now(), now(), 1024, 2048, 321, 543, 12209823498, 1344498988877487);

10
extensions/database/tests/conf/travis_tests.xml
View File

@ -20,6 +20,14 @@
<parameter name = "mariaDbUser" value="root"/>
<parameter name = "mariaDbPassword" value=""/>
<parameter name = "mariaDbTestTable" value="test_table"/>
<parameter name = "sqliteDbName" value="extension_test_db.sqlite"/>
<parameter name = "sqliteDbHost" value=""/>
<parameter name = "sqliteDbPort" value=""/>
<parameter name = "sqliteDbUser" value=""/>
<parameter name = "sqliteDbPassword" value=""/>
<parameter name = "sqliteDbTestTable" value="test_table"/>
<test name="DatabaseExtensionUnitTest">
<!-- Temporarily exclude MySQL and MariaDB from Travis tests until MySQL is working on Travis again. -->
<!-- See https://github.com/OpenRefine/OpenRefine/issues/2043 -->
@ -42,6 +50,8 @@
<class name="com.google.refine.extension.database.mysql.MySQLDatabaseServiceTest"/>
<class name="com.google.refine.extension.database.pgsql.PgSQLConnectionManagerTest"/>
<class name="com.google.refine.extension.database.pgsql.PgSQLDatabaseServiceTest" />
<class name="com.google.refine.extension.database.sqlite.SQLiteConnectionManagerTest"/>
<class name="com.google.refine.extension.database.sqlite.SQLiteDatabaseServiceTest" />
</classes>
</test>
</suite>

13
extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTestUtils.java
View File

@ -66,10 +66,15 @@ public class DBExtensionTestUtils {
DatabaseMetaData dbm = conn.getMetaData();
// check if "employee" table is there
ResultSet tables = dbm.getTables(null, null, tableName, null);
boolean dropTable = false;
if (tables.next()) {
stmt.executeUpdate("DROP TABLE " + tableName);
dropTable = true;
//System.out.println("Drop Table Result::" + dropResult);
}
tables.close();
if (dropTable) {
stmt.executeUpdate("DROP TABLE " + tableName);
}
String createSQL = " CREATE TABLE " + tableName + " ( "
+ " ID INT NOT NULL, "
@ -123,7 +128,13 @@ public class DBExtensionTestUtils {
DatabaseMetaData dbm = conn.getMetaData();
// check if "employee" table is there
ResultSet tables = dbm.getTables(null, null, DEFAULT_TEST_TABLE, null);
boolean dropTable = false;
if (tables.next()) {
dropTable = true;
//System.out.println("Drop Table Result::" + dropResult);
}
tables.close();
if (dropTable) {
stmt.executeUpdate("DROP TABLE " + DEFAULT_TEST_TABLE);
//System.out.println("Drop Table Result::" + dropResult);
}

3
extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTests.java
View File

@ -60,6 +60,9 @@ public class DBExtensionTests {
protected final String DEFAULT_MARIADB_USER = "root";
protected final String DEFAULT_MARIADB_PASSWORD = "secret";
protected final String DEFAULT_MARIADB_NAME = "testdb";
protected final String SQLITE_DB_NAME = "sqlite";
protected final String DEFAULT_SQLITE_DB_NAME = "extension_test_db.sqlite";
protected final String DEFAULT_TEST_TABLE = "test_data";

9
extensions/database/tests/src/com/google/refine/extension/database/DatabaseServiceTest.java
View File

@ -4,6 +4,7 @@ package com.google.refine.extension.database;
import java.sql.Connection;
import java.util.List;
import com.google.refine.extension.database.sqlite.SQLiteDatabaseService;
import org.testng.Assert;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Optional;
@ -85,6 +86,14 @@ public class DatabaseServiceTest extends DBExtensionTests{
Assert.assertEquals(dbService.getClass(), MariaDBDatabaseService.class);
}
@Test(groups = {"requiresSQLite"})
public void testGetSQLiteDBService() {
DatabaseService dbService = DatabaseService.get(SQLiteDatabaseService.DB_NAME);
Assert.assertNotNull(dbService);
Assert.assertEquals(dbService.getClass(), SQLiteDatabaseService.class);
}
@Test(groups = {"requiresMySQL"})
public void testGetConnection() throws DatabaseServiceException {
DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType());

13
extensions/database/tests/src/com/google/refine/extension/database/DatabaseTestConfig.java
View File

@ -16,11 +16,13 @@ public class DatabaseTestConfig extends DBExtensionTests {
private DatabaseConfiguration mysqlDbConfig;
private DatabaseConfiguration pgsqlDbConfig;
private DatabaseConfiguration mariadbDbConfig;
private DatabaseConfiguration sqliteDbConfig;
@BeforeSuite
@Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable",
"pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable",
"mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbyDbUser", "mariadbDbPassword", "mariadbTestTable"})
"mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbyDbUser", "mariadbDbPassword", "mariadbTestTable",
"sqliteDbName", "sqliteTestTable"})
public void beforeSuite(
@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost,
@Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser,
@ -32,7 +34,9 @@ public class DatabaseTestConfig extends DBExtensionTests {
@Optional(DEFAULT_MARIADB_NAME) String mariadbDbName, @Optional(DEFAULT_MARIADB_HOST) String mariadbDbHost,
@Optional(DEFAULT_MARIADB_PORT) String mariadbDbPort, @Optional(DEFAULT_MARIADB_USER) String mariadbyDbUser,
@Optional(DEFAULT_MARIADB_PASSWORD) String mariadbDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariadbTestTable)
@Optional(DEFAULT_MARIADB_PASSWORD) String mariadbDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariadbTestTable,
@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable)
throws DatabaseServiceException, SQLException {
//System.out.println("@BeforeSuite\n");
@ -62,10 +66,14 @@ public class DatabaseTestConfig extends DBExtensionTests {
mariadbDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME);
mariadbDbConfig.setDatabaseUser(mariadbyDbUser);
mariadbDbConfig.setUseSSL(false);
sqliteDbConfig = new DatabaseConfiguration();
sqliteDbConfig.setDatabaseName(sqliteDbName);
DBExtensionTestUtils.initTestData(mysqlDbConfig);
DBExtensionTestUtils.initTestData(pgsqlDbConfig);
DBExtensionTestUtils.initTestData(mariadbDbConfig);
DBExtensionTestUtils.initTestData(sqliteDbConfig);
}
@AfterSuite
@ -75,6 +83,7 @@ public class DatabaseTestConfig extends DBExtensionTests {
DBExtensionTestUtils.cleanUpTestData(mysqlDbConfig);
DBExtensionTestUtils.cleanUpTestData(pgsqlDbConfig);
DBExtensionTestUtils.cleanUpTestData(mariadbDbConfig);
DBExtensionTestUtils.cleanUpTestData(sqliteDbConfig);
}
}

99
extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManagerTest.java Normal file
View File

@ -0,0 +1,99 @@
/*
* Copyright (c) 2020, Chris Parker
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.extension.database.sqlite;
import com.google.refine.extension.database.DBExtensionTests;
import com.google.refine.extension.database.DatabaseConfiguration;
import com.google.refine.extension.database.DatabaseService;
import com.google.refine.extension.database.DatabaseServiceException;
import org.mockito.MockitoAnnotations;
import org.testng.Assert;
import org.testng.annotations.*;
import java.io.File;
import java.sql.Connection;
import java.sql.SQLException;
@Test(groups = { "requiresSQLite" })
public class SQLiteConnectionManagerTest extends DBExtensionTests {
private DatabaseConfiguration testDbConfig;
@BeforeTest
@Parameters({ "sqliteDbName", "sqliteTestTable" })
public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName,
@Optional(DEFAULT_TEST_TABLE) String sqliteTestTable)
throws DatabaseServiceException, SQLException {
MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration();
testDbConfig.setDatabaseName(sqliteDbName);
testDbConfig.setDatabaseType(SQLiteDatabaseService.DB_NAME);
DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance());
}
@AfterTest
@Parameters({ "sqliteDbName" })
public void afterTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName) {
File f = new File(sqliteDbName);
if (f.exists()) {
f.delete();
}
}
@Test
public void testTestConnection() throws DatabaseServiceException {
boolean isConnected = SQLiteConnectionManager.getInstance().testConnection(testDbConfig);
Assert.assertTrue(isConnected);
}
@Test
public void testGetConnection() throws DatabaseServiceException {
Connection conn = SQLiteConnectionManager.getInstance().getConnection(testDbConfig);
Assert.assertNotNull(conn);
}
@Test
public void testShutdown() throws DatabaseServiceException, SQLException {
Connection conn = SQLiteConnectionManager.getInstance().getConnection(testDbConfig);
Assert.assertNotNull(conn);
SQLiteConnectionManager.getInstance().shutdown();
if (conn != null) {
Assert.assertTrue(conn.isClosed());
}
}
}

160
extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseServiceTest.java Normal file
View File

@ -0,0 +1,160 @@
/*
* Copyright (c) 2020, Chris Parker
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.extension.database.sqlite;
import com.google.refine.extension.database.*;
import com.google.refine.extension.database.model.DatabaseColumn;
import com.google.refine.extension.database.model.DatabaseInfo;
import com.google.refine.extension.database.model.DatabaseRow;
import org.mockito.MockitoAnnotations;
import org.testng.Assert;
import org.testng.annotations.*;
import java.io.File;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;
@Test(groups = { "requiresSQLite" })
public class SQLiteDatabaseServiceTest extends DBExtensionTests {
private DatabaseConfiguration testDbConfig;
private String testTable;
@BeforeTest
@Parameters({ "sqliteDbName", "sqliteTestTable" })
public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName,
@Optional(DEFAULT_TEST_TABLE) String sqliteTestTable)
throws DatabaseServiceException, SQLException {
MockitoAnnotations.initMocks(this);
testDbConfig = new DatabaseConfiguration();
testDbConfig.setDatabaseName(sqliteDbName);
testDbConfig.setDatabaseType(SQLiteDatabaseService.DB_NAME);
testTable = sqliteTestTable;
DBExtensionTestUtils.initTestData(testDbConfig, sqliteTestTable);
DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance());
}
@AfterTest
@Parameters({ "sqliteDbName" })
public void afterTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName) {
File f = new File(sqliteDbName);
if (f.exists()) {
f.delete();
}
}
@Test
public void testGetDatabaseUrl() {
SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME);
String dbUrl = sqLiteDatabaseService.getDatabaseUrl(testDbConfig);
Assert.assertNotNull(dbUrl);
Assert.assertEquals(dbUrl, "jdbc:sqlite:extension_test_db.sqlite");
}
@Test
public void testGetConnection() throws DatabaseServiceException {
SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME);
Connection conn = sqLiteDatabaseService.getConnection(testDbConfig);
Assert.assertNotNull(conn);
}
@Test
public void testTestConnection() throws DatabaseServiceException {
SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME);
boolean result = sqLiteDatabaseService.testConnection(testDbConfig);
Assert.assertTrue(result);
}
@Test
public void testConnect() throws DatabaseServiceException {
SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME);
DatabaseInfo databaseInfo = sqLiteDatabaseService.connect(testDbConfig);
Assert.assertNotNull(databaseInfo);
}
@Test
public void testExecuteQuery() throws DatabaseServiceException {
SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME);
DatabaseInfo databaseInfo = sqLiteDatabaseService.testQuery(testDbConfig, "SELECT * FROM " + testTable);
Assert.assertNotNull(databaseInfo);
}
@Test
public void testBuildLimitQuery() {
SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME);
String limitQuery = sqliteSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable);
Assert.assertNotNull(limitQuery);
Assert.assertEquals(limitQuery,
"SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";");
}
@Test
public void testGetRows() throws DatabaseServiceException {
SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME);
List<DatabaseRow> dbRows = sqliteSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable);
Assert.assertNotNull(dbRows);
}
@Test
public void testGetInstance() {
SQLiteDatabaseService instance = SQLiteDatabaseService.getInstance();
Assert.assertNotNull(instance);
}
@Test
public void testGetColumns() throws DatabaseServiceException {
List<DatabaseColumn> dbColumns;
SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService
.get(SQLiteDatabaseService.DB_NAME);
dbColumns = sqliteSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable);
Assert.assertNotNull(dbColumns);
}
}