diff --git a/extensions/database/module/scripts/index/database-import-controller.js b/extensions/database/module/scripts/index/database-import-controller.js index 30eb34938..c7aae2c93 100644 --- a/extensions/database/module/scripts/index/database-import-controller.js +++ b/extensions/database/module/scripts/index/database-import-controller.js @@ -2,28 +2,28 @@ * Copyright (c) 2017, Tony Opara * All rights reserved. * - * Redistribution and use in source and binary forms, with or without + * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this + * - Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ @@ -48,7 +48,7 @@ $.i18n().load(dictionary, lang); Refine.DatabaseImportController = function(createProjectUI) { this._createProjectUI = createProjectUI; - + this._parsingPanel = createProjectUI.addCustomPanel(); createProjectUI.addSourceSelectionUI({ @@ -64,7 +64,7 @@ Refine.DatabaseImportController.prototype.startImportingDocument = function(quer var dismiss = DialogSystem.showBusy($.i18n('database-import/preparing')); //alert(queryInfo.query); var self = this; - + Refine.postCSRF( "command/core/create-importing-job", null, @@ -77,17 +77,17 @@ Refine.DatabaseImportController.prototype.startImportingDocument = function(quer "csrf_token": token }), queryInfo, - + function(data2) { dismiss(); - + if (data2.status == 'ok') { self._queryInfo = queryInfo; self._jobID = data.jobID; self._options = data2.options; - + self._showParsingPanel(); - + } else { alert(data2.message); } @@ -102,9 +102,9 @@ Refine.DatabaseImportController.prototype.startImportingDocument = function(quer Refine.DatabaseImportController.prototype.getOptions = function() { var options = { - + }; - + var parseIntDefault = function(s, def) { try { var n = parseInt(s); @@ -117,7 +117,7 @@ Refine.DatabaseImportController.prototype.getOptions = function() { return def; }; - + if (this._parsingPanelElmts.skipCheckbox[0].checked) { options.skipDataLines = parseIntDefault(this._parsingPanelElmts.skipInput[0].value, 0); } else { @@ -136,10 +136,10 @@ Refine.DatabaseImportController.prototype.getOptions = function() { Refine.DatabaseImportController.prototype._showParsingPanel = function() { var self = this; - + this._parsingPanel.unbind().empty().html( DOM.loadHTML("database",'scripts/index/database-parsing-panel.html')); - + this._parsingPanelElmts = DOM.bind(this._parsingPanel); this._parsingPanelElmts.startOverButton.html($.i18n('database-parsing/start-over')); @@ -155,11 +155,11 @@ Refine.DatabaseImportController.prototype._showParsingPanel = function() { this._parsingPanelElmts.database_limit.html($.i18n('database-parsing/limit')); this._parsingPanelElmts.database_store_row.html($.i18n('database-parsing/store-row')); this._parsingPanelElmts.database_store_cell.html($.i18n('database-parsing/store-cell')); - + if (this._parsingPanelResizer) { $(window).unbind('resize', this._parsingPanelResizer); } - + this._parsingPanelResizer = function() { var elmts = self._parsingPanelElmts; var width = self._parsingPanel.width(); @@ -184,27 +184,27 @@ Refine.DatabaseImportController.prototype._showParsingPanel = function() { .css("width", (width - DOM.getHPaddings(elmts.controlPanel)) + "px") .css("height", (controlPanelHeight - DOM.getVPaddings(elmts.controlPanel)) + "px"); }; - + $(window).resize(this._parsingPanelResizer); this._parsingPanelResizer(); - + this._parsingPanelElmts.startOverButton.click(function() { // explicitly cancel the import job Refine.CreateProjectUI.cancelImportingJob(self._jobID); - + delete self._jobID; delete self._options; - + self._createProjectUI.showSourceSelectionPanel(); }); - + this._parsingPanelElmts.createProjectButton.click(function() { self._createProject(); }); this._parsingPanelElmts.previewButton.click(function() { self._updatePreview(); }); //alert("datetime::" + $.now()); //this._parsingPanelElmts.projectNameInput[0].value = this._queryInfo.connectionName + "_" + this._queryInfo.databaseUser + "_" + $.now(); this._parsingPanelElmts.projectNameInput[0].value = this._queryInfo.databaseServer + "_" + this._queryInfo.initialDatabase + "_" + $.now(); - + if (this._options.limit > 0) { this._parsingPanelElmts.limitCheckbox.prop("checked", true); this._parsingPanelElmts.limitInput[0].value = this._options.limit.toString(); @@ -225,7 +225,7 @@ Refine.DatabaseImportController.prototype._showParsingPanel = function() { }; this._parsingPanel.find("input").bind("change", onChange); this._parsingPanel.find("select").bind("change", onChange); - + this._createProjectUI.showCustomPanel(this._parsingPanel); this._updatePreview(); }; @@ -259,9 +259,9 @@ Refine.DatabaseImportController.prototype._updatePreview = function() { "subCommand": "parse-preview", "csrf_token": token }), - - this._queryInfo, - + + self._queryInfo, + function(result) { if (result.status == "ok") { self._getPreviewData(function(projectData) { @@ -271,18 +271,18 @@ Refine.DatabaseImportController.prototype._updatePreview = function() { new Refine.PreviewTable(projectData, self._parsingPanelElmts.dataPanel.unbind().empty()); }); } else { - + alert('Errors:\n' + (result.message) ? result.message : Refine.CreateProjectUI.composeErrorMessage(job)); self._parsingPanelElmts.progressPanel.hide(); - + Refine.CreateProjectUI.cancelImportingJob(self._jobID); - + delete self._jobID; delete self._options; - + self._createProjectUI.showSourceSelectionPanel(); - - + + } }, "json" @@ -303,7 +303,7 @@ Refine.DatabaseImportController.prototype._getPreviewData = function(callback, n result[n] = data[n]; } } - + $.post( "command/core/get-rows?" + $.param({ "importingJobID" : self._jobID, @@ -329,11 +329,11 @@ Refine.DatabaseImportController.prototype._createProject = function() { this._parsingPanelElmts.projectNameInput.focus(); return; } - + var self = this; var options = this.getOptions(); options.projectName = projectName; - + this._queryInfo.options = JSON.stringify(options); Refine.wrapCSRF(function(token) { $.post( @@ -343,7 +343,7 @@ Refine.DatabaseImportController.prototype._createProject = function() { "subCommand": "create-project", "csrf_token": token }), - this._queryInfo, + self._queryInfo, function(o) { if (o.status == 'error') { alert(o.message); @@ -385,4 +385,4 @@ Refine.DatabaseImportController.prototype._createProject = function() { "json" ); }); -}; +}; \ No newline at end of file diff --git a/extensions/database/module/scripts/index/database-import-form.html b/extensions/database/module/scripts/index/database-import-form.html index de1f96f08..cff60ed20 100644 --- a/extensions/database/module/scripts/index/database-import-form.html +++ b/extensions/database/module/scripts/index/database-import-form.html @@ -38,6 +38,7 @@ + diff --git a/extensions/database/module/scripts/index/database-source-ui.js b/extensions/database/module/scripts/index/database-source-ui.js index 14c1d0acd..84abc1c14 100644 --- a/extensions/database/module/scripts/index/database-source-ui.js +++ b/extensions/database/module/scripts/index/database-source-ui.js @@ -100,7 +100,11 @@ Refine.DatabaseSourceUI.prototype.attachUI = function(body) { $( "#databasePort" ).val("3306"); }else if(type === "mariadb"){ $( "#databaseUser" ).val("root"); - $( "#databasePort" ).val("3306"); + $( "#databasePort" ).val("3306"); + }else if(type === "sqlite"){ + $( "#databaseUser" ).val("na"); + $( "#databasePort" ).val("0"); + $( "#databaseHost" ).val("na"); }else{ $( "#databaseUser" ).val("root"); $( "#databasePort" ).val("3306"); @@ -267,7 +271,7 @@ Refine.DatabaseSourceUI.prototype._executeQuery = function(jdbcQueryInfo) { //remove start line var dismiss = DialogSystem.showBusy($.i18n('database-import/checking')); - + Refine.postCSRF( "command/database/test-query", jdbcQueryInfo, @@ -440,7 +444,7 @@ Refine.DatabaseSourceUI.prototype._validateNewConnectionForm = function() { var databaseHostTestResult = alphaNumRE.test(databaseHostInput); var databasePortTestResult = numRE.test(databasePortInput); var databaseUserTestResult = alphaNumRE.test(databaseUserInput); - + if(alphaNumConnNameTestResult == false){ window.alert($.i18n('database-source/alert-conn-name-invalid-character')); return false; diff --git a/extensions/database/pom.xml b/extensions/database/pom.xml index 49e861d9c..709dbfab3 100644 --- a/extensions/database/pom.xml +++ b/extensions/database/pom.xml @@ -150,6 +150,11 @@ mariadb-java-client 2.2.6 + + org.xerial + sqlite-jdbc + 3.30.1 + com.fasterxml.jackson.core jackson-core diff --git a/extensions/database/src/com/google/refine/extension/database/DatabaseService.java b/extensions/database/src/com/google/refine/extension/database/DatabaseService.java index c68785d46..f25d5b114 100644 --- a/extensions/database/src/com/google/refine/extension/database/DatabaseService.java +++ b/extensions/database/src/com/google/refine/extension/database/DatabaseService.java @@ -33,6 +33,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.google.refine.extension.database.sqlite.SQLiteDatabaseService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,6 +58,7 @@ public abstract class DatabaseService { DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance()); DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); + DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance()); } catch (Exception e) { logger.error("Exception occurred while trying to prepare databases!", e); diff --git a/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManager.java b/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManager.java new file mode 100644 index 000000000..5b9b4cfcd --- /dev/null +++ b/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManager.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2020, Chris Parker + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.extension.database.sqlite; + +import com.google.refine.extension.database.DatabaseConfiguration; +import com.google.refine.extension.database.DatabaseServiceException; +import com.google.refine.extension.database.SQLType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +public class SQLiteConnectionManager { + + private static final Logger logger = LoggerFactory.getLogger("SQLiteConnectionManager"); + private static SQLiteConnectionManager instance; + private final SQLType type; + private Connection connection; + + private SQLiteConnectionManager() { + type = SQLType.forName(SQLiteDatabaseService.DB_NAME); + } + + /** + * Create a new instance of this connection manager. + * + * @return an instance of the manager + */ + public static SQLiteConnectionManager getInstance() { + if (instance == null) { + if (logger.isDebugEnabled()) { + logger.debug("::Creating new SQLite ConnectionManager ::"); + } + instance = new SQLiteConnectionManager(); + } + return instance; + } + + public static String getDatabaseUrl(DatabaseConfiguration dbConfig) { + return "jdbc:" + dbConfig.getDatabaseType().toLowerCase() + ":" + dbConfig.getDatabaseName(); + } + + /** + * Get the SQL Database type. + * + * @return the type + */ + public SQLType getType() { + return this.type; + } + + /** + * testConnection + * + * @param dbConfig + * @return boolean + */ + public boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException { + try { + boolean connResult = false; + + Connection conn = getConnection(dbConfig); + if (conn != null) { + connResult = true; + conn.close(); + } + + return connResult; + } catch (SQLException e) { + logger.error("Test connection Failed!", e); + throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); + } + } + + /** + * Get a connection form the connection pool. + * + * @return connection from the pool + */ + public Connection getConnection(DatabaseConfiguration databaseConfiguration) throws DatabaseServiceException { + try { + if (connection != null) { + connection.close(); + } + + Class.forName(type.getClassPath()); + String dbURL = getDatabaseUrl(databaseConfiguration); + connection = DriverManager.getConnection(dbURL); + + logger.debug("*** Acquired New connection for ::{} **** ", dbURL); + + return connection; + } catch (ClassNotFoundException e) { + logger.error("Jdbc Driver not found", e); + throw new DatabaseServiceException(e.getMessage()); + } catch (SQLException e) { + logger.error("SQLException::Couldn't get a Connection!", e); + throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); + } + } + + public void shutdown() { + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + logger.warn("Non-Managed connection could not be closed. Whoops!", e); + } + } + } +} diff --git a/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseService.java b/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseService.java new file mode 100644 index 000000000..5971a5e3f --- /dev/null +++ b/extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseService.java @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2020, Chris Parker + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.extension.database.sqlite; + +import com.google.refine.extension.database.SQLType; +import com.google.refine.extension.database.*; +import com.google.refine.extension.database.model.DatabaseColumn; +import com.google.refine.extension.database.model.DatabaseInfo; +import com.google.refine.extension.database.model.DatabaseRow; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; + +public class SQLiteDatabaseService extends DatabaseService { + + public static final String DB_NAME = "sqlite"; + public static final String DB_DRIVER = "org.sqlite.JDBC"; + private static final Logger logger = LoggerFactory.getLogger("SQLiteDatabaseService"); + private static SQLiteDatabaseService instance; + + public static SQLiteDatabaseService getInstance() { + if (instance == null) { + SQLType.registerSQLDriver(DB_NAME, DB_DRIVER, false); + instance = new SQLiteDatabaseService(); + if (logger.isDebugEnabled()) { + logger.debug("SQLiteDatabaseService Instance: {}", instance); + } + } + return instance; + } + + @Override + protected String getDatabaseUrl(DatabaseConfiguration dbConfig) { + return SQLiteConnectionManager.getDatabaseUrl(dbConfig); + } + + @Override + public Connection getConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException { + return SQLiteConnectionManager.getInstance().getConnection(dbConfig); + } + + @Override + public boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException { + return SQLiteConnectionManager.getInstance().testConnection(dbConfig); + } + + @Override + public DatabaseInfo connect(DatabaseConfiguration dbConfig) throws DatabaseServiceException { + return getMetadata(dbConfig); + } + + /** + * @param dbConfig + * @return + * @throws DatabaseServiceException + */ + private DatabaseInfo getMetadata(DatabaseConfiguration connectionInfo) throws DatabaseServiceException { + try { + Connection connection = SQLiteConnectionManager.getInstance().getConnection(connectionInfo); + if (connection != null) { + java.sql.DatabaseMetaData metadata = connection.getMetaData(); + int dbMajorVersion = metadata.getDatabaseMajorVersion(); + int dbMinorVersion = metadata.getDatabaseMinorVersion(); + String dbProductVersion = metadata.getDatabaseProductVersion(); + String dbProductName = metadata.getDatabaseProductName(); + DatabaseInfo dbInfo = new DatabaseInfo(); + dbInfo.setDatabaseMajorVersion(dbMajorVersion); + dbInfo.setDatabaseMinorVersion(dbMinorVersion); + dbInfo.setDatabaseProductVersion(dbProductVersion); + dbInfo.setDatabaseProductName(dbProductName); + return dbInfo; + } + } catch (SQLException e) { + logger.error("SQLException::", e); + throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); + } + return null; + } + + @Override + public DatabaseInfo executeQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException { + try { + Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig); + Statement statement = connection.createStatement(); + ResultSet queryResult = statement.executeQuery(query); + ResultSetMetaData metadata = queryResult.getMetaData(); + int columnCount = metadata.getColumnCount(); + ArrayList columns = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + DatabaseColumn dc = new DatabaseColumn(metadata.getColumnName(i), metadata.getColumnLabel(i), + DatabaseUtils.getDbColumnType(metadata.getColumnType(i)), + metadata.getColumnDisplaySize(i)); + columns.add(dc); + } + int index = 0; + List rows = new ArrayList<>(); + while (queryResult.next()) { + DatabaseRow row = new DatabaseRow(); + row.setIndex(index); + List values = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + values.add(queryResult.getString(i)); + } + row.setValues(values); + rows.add(row); + index++; + } + DatabaseInfo dbInfo = new DatabaseInfo(); + dbInfo.setColumns(columns); + dbInfo.setRows(rows); + return dbInfo; + } catch (SQLException e) { + logger.error("SQLException::", e); + throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); + } finally { + SQLiteConnectionManager.getInstance().shutdown(); + } + } + + @Override + public DatabaseInfo testQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException { + Statement statement = null; + ResultSet queryResult = null; + try { + Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig); + statement = connection.createStatement(); + queryResult = statement.executeQuery(query); + return new DatabaseInfo(); + } catch (SQLException e) { + logger.error("SQLException::", e); + throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); + } finally { + try { + if (queryResult != null) { + queryResult.close(); + } + if (statement != null) { + statement.close(); + } + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + SQLiteConnectionManager.getInstance().shutdown(); + } + } + + @Override + public List getColumns(DatabaseConfiguration dbConfig, String query) + throws DatabaseServiceException { + try { + Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig); + Statement statement = connection.createStatement(); + ResultSet queryResult = statement.executeQuery(query); + ResultSetMetaData metadata = queryResult.getMetaData(); + int columnCount = metadata.getColumnCount(); + ArrayList columns = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + DatabaseColumn dc = new DatabaseColumn(metadata.getColumnName(i), metadata.getColumnLabel(i), + DatabaseUtils.getDbColumnType(metadata.getColumnType(i)), + metadata.getColumnDisplaySize(i)); + columns.add(dc); + } + return columns; + } catch (SQLException e) { + logger.error("SQLException::", e); + throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); + } + } + + @Override + public List getRows(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException { + try { + Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig); + Statement statement = connection.createStatement(); + statement.setFetchSize(10); + ResultSet queryResult = statement.executeQuery(query); + ResultSetMetaData metadata = queryResult.getMetaData(); + int columnCount = metadata.getColumnCount(); + int index = 0; + List rows = new ArrayList<>(); + while (queryResult.next()) { + DatabaseRow row = new DatabaseRow(); + row.setIndex(index); + List values = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + values.add(queryResult.getString(i)); + } + row.setValues(values); + rows.add(row); + index++; + } + return rows; + } catch (SQLException e) { + logger.error("SQLException::", e); + throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); + } + } +} diff --git a/extensions/database/tests/conf/appveyor_tests.xml b/extensions/database/tests/conf/appveyor_tests.xml index 1d906be12..c913efbed 100644 --- a/extensions/database/tests/conf/appveyor_tests.xml +++ b/extensions/database/tests/conf/appveyor_tests.xml @@ -24,6 +24,14 @@ + + + + + + + + @@ -45,6 +53,8 @@ + + diff --git a/extensions/database/tests/conf/tests.xml b/extensions/database/tests/conf/tests.xml index 27a73a24a..726ccf615 100644 --- a/extensions/database/tests/conf/tests.xml +++ b/extensions/database/tests/conf/tests.xml @@ -28,12 +28,21 @@ + + + + + + + + + @@ -42,6 +50,8 @@ + + diff --git a/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTestUtils.java b/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTestUtils.java index a0d50fa06..c098458b6 100644 --- a/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTestUtils.java +++ b/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTestUtils.java @@ -66,10 +66,15 @@ public class DBExtensionTestUtils { DatabaseMetaData dbm = conn.getMetaData(); // check if "employee" table is there ResultSet tables = dbm.getTables(null, null, tableName, null); + boolean dropTable = false; if (tables.next()) { - stmt.executeUpdate("DROP TABLE " + tableName); + dropTable = true; //System.out.println("Drop Table Result::" + dropResult); } + tables.close(); + if (dropTable) { + stmt.executeUpdate("DROP TABLE " + tableName); + } String createSQL = " CREATE TABLE " + tableName + " ( " + " ID INT NOT NULL, " @@ -123,7 +128,13 @@ public class DBExtensionTestUtils { DatabaseMetaData dbm = conn.getMetaData(); // check if "employee" table is there ResultSet tables = dbm.getTables(null, null, DEFAULT_TEST_TABLE, null); + boolean dropTable = false; if (tables.next()) { + dropTable = true; + //System.out.println("Drop Table Result::" + dropResult); + } + tables.close(); + if (dropTable) { stmt.executeUpdate("DROP TABLE " + DEFAULT_TEST_TABLE); //System.out.println("Drop Table Result::" + dropResult); } diff --git a/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTests.java b/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTests.java index 7d31085ef..167611558 100644 --- a/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTests.java +++ b/extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTests.java @@ -60,6 +60,9 @@ public class DBExtensionTests { protected final String DEFAULT_MARIADB_USER = "root"; protected final String DEFAULT_MARIADB_PASSWORD = "secret"; protected final String DEFAULT_MARIADB_NAME = "testdb"; + + protected final String SQLITE_DB_NAME = "sqlite"; + protected final String DEFAULT_SQLITE_DB_NAME = "extension_test_db.sqlite"; protected final String DEFAULT_TEST_TABLE = "test_data"; diff --git a/extensions/database/tests/src/com/google/refine/extension/database/DatabaseServiceTest.java b/extensions/database/tests/src/com/google/refine/extension/database/DatabaseServiceTest.java index 10dabca46..b85a7cfd0 100644 --- a/extensions/database/tests/src/com/google/refine/extension/database/DatabaseServiceTest.java +++ b/extensions/database/tests/src/com/google/refine/extension/database/DatabaseServiceTest.java @@ -4,6 +4,7 @@ package com.google.refine.extension.database; import java.sql.Connection; import java.util.List; +import com.google.refine.extension.database.sqlite.SQLiteDatabaseService; import org.testng.Assert; import org.testng.annotations.BeforeTest; import org.testng.annotations.Optional; @@ -85,6 +86,14 @@ public class DatabaseServiceTest extends DBExtensionTests{ Assert.assertEquals(dbService.getClass(), MariaDBDatabaseService.class); } + @Test(groups = {"requiresSQLite"}) + public void testGetSQLiteDBService() { + + DatabaseService dbService = DatabaseService.get(SQLiteDatabaseService.DB_NAME); + Assert.assertNotNull(dbService); + Assert.assertEquals(dbService.getClass(), SQLiteDatabaseService.class); + } + @Test(groups = {"requiresMySQL"}) public void testGetConnection() throws DatabaseServiceException { DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); diff --git a/extensions/database/tests/src/com/google/refine/extension/database/DatabaseTestConfig.java b/extensions/database/tests/src/com/google/refine/extension/database/DatabaseTestConfig.java index ffebbe8e6..5f1c3d225 100644 --- a/extensions/database/tests/src/com/google/refine/extension/database/DatabaseTestConfig.java +++ b/extensions/database/tests/src/com/google/refine/extension/database/DatabaseTestConfig.java @@ -16,11 +16,13 @@ public class DatabaseTestConfig extends DBExtensionTests { private DatabaseConfiguration mysqlDbConfig; private DatabaseConfiguration pgsqlDbConfig; private DatabaseConfiguration mariadbDbConfig; + private DatabaseConfiguration sqliteDbConfig; @BeforeSuite @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable", "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable", - "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbyDbUser", "mariadbDbPassword", "mariadbTestTable"}) + "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbyDbUser", "mariadbDbPassword", "mariadbTestTable", + "sqliteDbName", "sqliteTestTable"}) public void beforeSuite( @Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, @@ -32,7 +34,9 @@ public class DatabaseTestConfig extends DBExtensionTests { @Optional(DEFAULT_MARIADB_NAME) String mariadbDbName, @Optional(DEFAULT_MARIADB_HOST) String mariadbDbHost, @Optional(DEFAULT_MARIADB_PORT) String mariadbDbPort, @Optional(DEFAULT_MARIADB_USER) String mariadbyDbUser, - @Optional(DEFAULT_MARIADB_PASSWORD) String mariadbDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariadbTestTable) + @Optional(DEFAULT_MARIADB_PASSWORD) String mariadbDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariadbTestTable, + + @Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable) throws DatabaseServiceException, SQLException { //System.out.println("@BeforeSuite\n"); @@ -62,10 +66,14 @@ public class DatabaseTestConfig extends DBExtensionTests { mariadbDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME); mariadbDbConfig.setDatabaseUser(mariadbyDbUser); mariadbDbConfig.setUseSSL(false); + + sqliteDbConfig = new DatabaseConfiguration(); + sqliteDbConfig.setDatabaseName(sqliteDbName); DBExtensionTestUtils.initTestData(mysqlDbConfig); DBExtensionTestUtils.initTestData(pgsqlDbConfig); DBExtensionTestUtils.initTestData(mariadbDbConfig); + DBExtensionTestUtils.initTestData(sqliteDbConfig); } @AfterSuite @@ -75,6 +83,7 @@ public class DatabaseTestConfig extends DBExtensionTests { DBExtensionTestUtils.cleanUpTestData(mysqlDbConfig); DBExtensionTestUtils.cleanUpTestData(pgsqlDbConfig); DBExtensionTestUtils.cleanUpTestData(mariadbDbConfig); + DBExtensionTestUtils.cleanUpTestData(sqliteDbConfig); } } diff --git a/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManagerTest.java b/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManagerTest.java new file mode 100644 index 000000000..693500197 --- /dev/null +++ b/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManagerTest.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2020, Chris Parker + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.extension.database.sqlite; + +import com.google.refine.extension.database.DBExtensionTests; +import com.google.refine.extension.database.DatabaseConfiguration; +import com.google.refine.extension.database.DatabaseService; +import com.google.refine.extension.database.DatabaseServiceException; +import org.mockito.MockitoAnnotations; +import org.testng.Assert; +import org.testng.annotations.*; + +import java.io.File; +import java.sql.Connection; +import java.sql.SQLException; + +@Test(groups = { "requiresSQLite" }) +public class SQLiteConnectionManagerTest extends DBExtensionTests { + + private DatabaseConfiguration testDbConfig; + + @BeforeTest + @Parameters({ "sqliteDbName", "sqliteTestTable" }) + public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, + @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable) + throws DatabaseServiceException, SQLException { + + MockitoAnnotations.initMocks(this); + + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseName(sqliteDbName); + testDbConfig.setDatabaseType(SQLiteDatabaseService.DB_NAME); + + DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance()); + } + + @AfterTest + @Parameters({ "sqliteDbName" }) + public void afterTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName) { + File f = new File(sqliteDbName); + if (f.exists()) { + f.delete(); + } + } + + @Test + public void testTestConnection() throws DatabaseServiceException { + boolean isConnected = SQLiteConnectionManager.getInstance().testConnection(testDbConfig); + Assert.assertTrue(isConnected); + } + + @Test + public void testGetConnection() throws DatabaseServiceException { + + Connection conn = SQLiteConnectionManager.getInstance().getConnection(testDbConfig); + Assert.assertNotNull(conn); + } + + @Test + public void testShutdown() throws DatabaseServiceException, SQLException { + + Connection conn = SQLiteConnectionManager.getInstance().getConnection(testDbConfig); + Assert.assertNotNull(conn); + + SQLiteConnectionManager.getInstance().shutdown(); + + if (conn != null) { + Assert.assertTrue(conn.isClosed()); + } + + } +} diff --git a/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseServiceTest.java b/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseServiceTest.java new file mode 100644 index 000000000..a313f25d8 --- /dev/null +++ b/extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseServiceTest.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2020, Chris Parker + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.extension.database.sqlite; + +import com.google.refine.extension.database.*; +import com.google.refine.extension.database.model.DatabaseColumn; +import com.google.refine.extension.database.model.DatabaseInfo; +import com.google.refine.extension.database.model.DatabaseRow; +import org.mockito.MockitoAnnotations; +import org.testng.Assert; +import org.testng.annotations.*; + +import java.io.File; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; + +@Test(groups = { "requiresSQLite" }) +public class SQLiteDatabaseServiceTest extends DBExtensionTests { + + private DatabaseConfiguration testDbConfig; + private String testTable; + + @BeforeTest + @Parameters({ "sqliteDbName", "sqliteTestTable" }) + public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, + @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable) + throws DatabaseServiceException, SQLException { + + MockitoAnnotations.initMocks(this); + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseName(sqliteDbName); + testDbConfig.setDatabaseType(SQLiteDatabaseService.DB_NAME); + + testTable = sqliteTestTable; + DBExtensionTestUtils.initTestData(testDbConfig, sqliteTestTable); + + DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance()); + } + + @AfterTest + @Parameters({ "sqliteDbName" }) + public void afterTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName) { + File f = new File(sqliteDbName); + if (f.exists()) { + f.delete(); + } + } + + @Test + public void testGetDatabaseUrl() { + SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService + .get(SQLiteDatabaseService.DB_NAME); + String dbUrl = sqLiteDatabaseService.getDatabaseUrl(testDbConfig); + + Assert.assertNotNull(dbUrl); + Assert.assertEquals(dbUrl, "jdbc:sqlite:extension_test_db.sqlite"); + } + + @Test + public void testGetConnection() throws DatabaseServiceException { + + SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService + .get(SQLiteDatabaseService.DB_NAME); + Connection conn = sqLiteDatabaseService.getConnection(testDbConfig); + + Assert.assertNotNull(conn); + } + + @Test + public void testTestConnection() throws DatabaseServiceException { + SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService + .get(SQLiteDatabaseService.DB_NAME); + + boolean result = sqLiteDatabaseService.testConnection(testDbConfig); + Assert.assertTrue(result); + } + + @Test + public void testConnect() throws DatabaseServiceException { + + SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService + .get(SQLiteDatabaseService.DB_NAME); + DatabaseInfo databaseInfo = sqLiteDatabaseService.connect(testDbConfig); + Assert.assertNotNull(databaseInfo); + } + + @Test + public void testExecuteQuery() throws DatabaseServiceException { + SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService + .get(SQLiteDatabaseService.DB_NAME); + DatabaseInfo databaseInfo = sqLiteDatabaseService.testQuery(testDbConfig, "SELECT * FROM " + testTable); + + Assert.assertNotNull(databaseInfo); + } + + @Test + public void testBuildLimitQuery() { + SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService + .get(SQLiteDatabaseService.DB_NAME); + String limitQuery = sqliteSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable); + Assert.assertNotNull(limitQuery); + Assert.assertEquals(limitQuery, + "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";"); + } + + @Test + public void testGetRows() throws DatabaseServiceException { + SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService + .get(SQLiteDatabaseService.DB_NAME); + List dbRows = sqliteSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable); + + Assert.assertNotNull(dbRows); + } + + @Test + public void testGetInstance() { + SQLiteDatabaseService instance = SQLiteDatabaseService.getInstance(); + Assert.assertNotNull(instance); + } + + @Test + public void testGetColumns() throws DatabaseServiceException { + List dbColumns; + + SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService + .get(SQLiteDatabaseService.DB_NAME); + + dbColumns = sqliteSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable); + + Assert.assertNotNull(dbColumns); + } +}