Merge pull request #2323 from mrcsparker/sqlite-support

Added SQLite support

extensions/database/module/scripts/index/database-import-controller.js

@@ -260,7 +260,7 @@ Refine.DatabaseImportController.prototype._updatePreview = function() {
             "csrf_token": token
         }),
 
-        this._queryInfo,
+        self._queryInfo,
 
         function(result) {
             if (result.status == "ok") {
@@ -343,7 +343,7 @@ Refine.DatabaseImportController.prototype._createProject = function() {
             "subCommand": "create-project",
             "csrf_token": token
         }),
-        this._queryInfo,
+        self._queryInfo,
         function(o) {
             if (o.status == 'error') {
             alert(o.message);

extensions/database/module/scripts/index/database-import-form.html

@@ -38,6 +38,7 @@
                                             	  <option value="postgresql">PostgreSQL</option>  
                                                <option value="mysql">MySQL</option>
                                                <option value="mariadb">MariaDB</option>
+                                                <option value="sqlite">SQLite</option>
                                             </select>
                                        </div>
                                

extensions/database/module/scripts/index/database-source-ui.js

@@ -101,6 +101,10 @@ Refine.DatabaseSourceUI.prototype.attachUI = function(body) {
       }else if(type === "mariadb"){
           $( "#databaseUser" ).val("root");
           $( "#databasePort" ).val("3306");
+      }else if(type === "sqlite"){
+          $( "#databaseUser" ).val("na");
+          $( "#databasePort" ).val("0");
+          $( "#databaseHost" ).val("na");
       }else{
           $( "#databaseUser" ).val("root");
           $( "#databasePort" ).val("3306");

extensions/database/pom.xml

@@ -150,6 +150,11 @@
       <artifactId>mariadb-java-client</artifactId>
       <version>2.2.6</version>
     </dependency>
+    <dependency>
+      <groupId>org.xerial</groupId>
+      <artifactId>sqlite-jdbc</artifactId>
+      <version>3.30.1</version>
+    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>

extensions/database/src/com/google/refine/extension/database/DatabaseService.java

@@ -33,6 +33,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import com.google.refine.extension.database.sqlite.SQLiteDatabaseService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -57,6 +58,7 @@ public abstract class DatabaseService {
                 DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance());
                 DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance());
                 DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance());
+                DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance());
 
             } catch (Exception e) {
                 logger.error("Exception occurred while trying to prepare databases!", e);

extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManager.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2020, Chris Parker
+ *        All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * Neither the name of Google nor the names of its contributors may be used to
+ * endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.google.refine.extension.database.sqlite;
+
+import com.google.refine.extension.database.DatabaseConfiguration;
+import com.google.refine.extension.database.DatabaseServiceException;
+import com.google.refine.extension.database.SQLType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+public class SQLiteConnectionManager {
+
+    private static final Logger logger = LoggerFactory.getLogger("SQLiteConnectionManager");
+    private static SQLiteConnectionManager instance;
+    private final SQLType type;
+    private Connection connection;
+
+    private SQLiteConnectionManager() {
+        type = SQLType.forName(SQLiteDatabaseService.DB_NAME);
+    }
+
+    /**
+     * Create a new instance of this connection manager.
+     *
+     * @return an instance of the manager
+     */
+    public static SQLiteConnectionManager getInstance() {
+        if (instance == null) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("::Creating new SQLite ConnectionManager ::");
+            }
+            instance = new SQLiteConnectionManager();
+        }
+        return instance;
+    }
+
+    public static String getDatabaseUrl(DatabaseConfiguration dbConfig) {
+        return "jdbc:" + dbConfig.getDatabaseType().toLowerCase() + ":" + dbConfig.getDatabaseName();
+    }
+
+    /**
+     * Get the SQL Database type.
+     *
+     * @return the type
+     */
+    public SQLType getType() {
+        return this.type;
+    }
+
+    /**
+     * testConnection
+     *
+     * @param dbConfig
+     * @return boolean
+     */
+    public boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException {
+        try {
+            boolean connResult = false;
+
+            Connection conn = getConnection(dbConfig);
+            if (conn != null) {
+                connResult = true;
+                conn.close();
+            }
+
+            return connResult;
+        } catch (SQLException e) {
+            logger.error("Test connection Failed!", e);
+            throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
+        }
+    }
+
+    /**
+     * Get a connection form the connection pool.
+     *
+     * @return connection from the pool
+     */
+    public Connection getConnection(DatabaseConfiguration databaseConfiguration) throws DatabaseServiceException {
+        try {
+            if (connection != null) {
+                connection.close();
+            }
+
+            Class.forName(type.getClassPath());
+            String dbURL = getDatabaseUrl(databaseConfiguration);
+            connection = DriverManager.getConnection(dbURL);
+
+            logger.debug("*** Acquired New  connection for ::{} **** ", dbURL);
+
+            return connection;
+        } catch (ClassNotFoundException e) {
+            logger.error("Jdbc Driver not found", e);
+            throw new DatabaseServiceException(e.getMessage());
+        } catch (SQLException e) {
+            logger.error("SQLException::Couldn't get a Connection!", e);
+            throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
+        }
+    }
+
+    public void shutdown() {
+        if (connection != null) {
+            try {
+                connection.close();
+            } catch (SQLException e) {
+                logger.warn("Non-Managed connection could not be closed. Whoops!", e);
+            }
+        }
+    }
+}

extensions/database/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseService.java

@@ -0,0 +1,229 @@
+/*
+ * Copyright (c) 2020, Chris Parker
+ *        All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * Neither the name of Google nor the names of its contributors may be used to
+ * endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.google.refine.extension.database.sqlite;
+
+import com.google.refine.extension.database.SQLType;
+import com.google.refine.extension.database.*;
+import com.google.refine.extension.database.model.DatabaseColumn;
+import com.google.refine.extension.database.model.DatabaseInfo;
+import com.google.refine.extension.database.model.DatabaseRow;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.List;
+
+public class SQLiteDatabaseService extends DatabaseService {
+
+    public static final String DB_NAME = "sqlite";
+    public static final String DB_DRIVER = "org.sqlite.JDBC";
+    private static final Logger logger = LoggerFactory.getLogger("SQLiteDatabaseService");
+    private static SQLiteDatabaseService instance;
+
+    public static SQLiteDatabaseService getInstance() {
+        if (instance == null) {
+            SQLType.registerSQLDriver(DB_NAME, DB_DRIVER, false);
+            instance = new SQLiteDatabaseService();
+            if (logger.isDebugEnabled()) {
+                logger.debug("SQLiteDatabaseService Instance: {}", instance);
+            }
+        }
+        return instance;
+    }
+
+    @Override
+    protected String getDatabaseUrl(DatabaseConfiguration dbConfig) {
+        return SQLiteConnectionManager.getDatabaseUrl(dbConfig);
+    }
+
+    @Override
+    public Connection getConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException {
+        return SQLiteConnectionManager.getInstance().getConnection(dbConfig);
+    }
+
+    @Override
+    public boolean testConnection(DatabaseConfiguration dbConfig) throws DatabaseServiceException {
+        return SQLiteConnectionManager.getInstance().testConnection(dbConfig);
+    }
+
+    @Override
+    public DatabaseInfo connect(DatabaseConfiguration dbConfig) throws DatabaseServiceException {
+        return getMetadata(dbConfig);
+    }
+
+    /**
+     * @param dbConfig
+     * @return
+     * @throws DatabaseServiceException
+     */
+    private DatabaseInfo getMetadata(DatabaseConfiguration connectionInfo) throws DatabaseServiceException {
+        try {
+            Connection connection = SQLiteConnectionManager.getInstance().getConnection(connectionInfo);
+            if (connection != null) {
+                java.sql.DatabaseMetaData metadata = connection.getMetaData();
+                int dbMajorVersion = metadata.getDatabaseMajorVersion();
+                int dbMinorVersion = metadata.getDatabaseMinorVersion();
+                String dbProductVersion = metadata.getDatabaseProductVersion();
+                String dbProductName = metadata.getDatabaseProductName();
+                DatabaseInfo dbInfo = new DatabaseInfo();
+                dbInfo.setDatabaseMajorVersion(dbMajorVersion);
+                dbInfo.setDatabaseMinorVersion(dbMinorVersion);
+                dbInfo.setDatabaseProductVersion(dbProductVersion);
+                dbInfo.setDatabaseProductName(dbProductName);
+                return dbInfo;
+            }
+        } catch (SQLException e) {
+            logger.error("SQLException::", e);
+            throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
+        }
+        return null;
+    }
+
+    @Override
+    public DatabaseInfo executeQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException {
+        try {
+            Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig);
+            Statement statement = connection.createStatement();
+            ResultSet queryResult = statement.executeQuery(query);
+            ResultSetMetaData metadata = queryResult.getMetaData();
+            int columnCount = metadata.getColumnCount();
+            ArrayList<DatabaseColumn> columns = new ArrayList<>(columnCount);
+            for (int i = 1; i <= columnCount; i++) {
+                DatabaseColumn dc = new DatabaseColumn(metadata.getColumnName(i), metadata.getColumnLabel(i),
+                                DatabaseUtils.getDbColumnType(metadata.getColumnType(i)),
+                                metadata.getColumnDisplaySize(i));
+                columns.add(dc);
+            }
+            int index = 0;
+            List<DatabaseRow> rows = new ArrayList<>();
+            while (queryResult.next()) {
+                DatabaseRow row = new DatabaseRow();
+                row.setIndex(index);
+                List<String> values = new ArrayList<>(columnCount);
+                for (int i = 1; i <= columnCount; i++) {
+                    values.add(queryResult.getString(i));
+                }
+                row.setValues(values);
+                rows.add(row);
+                index++;
+            }
+            DatabaseInfo dbInfo = new DatabaseInfo();
+            dbInfo.setColumns(columns);
+            dbInfo.setRows(rows);
+            return dbInfo;
+        } catch (SQLException e) {
+            logger.error("SQLException::", e);
+            throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
+        } finally {
+            SQLiteConnectionManager.getInstance().shutdown();
+        }
+    }
+
+    @Override
+    public DatabaseInfo testQuery(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException {
+        Statement statement = null;
+        ResultSet queryResult = null;
+        try {
+            Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig);
+            statement = connection.createStatement();
+            queryResult = statement.executeQuery(query);
+            return new DatabaseInfo();
+        } catch (SQLException e) {
+            logger.error("SQLException::", e);
+            throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
+        } finally {
+            try {
+                if (queryResult != null) {
+                    queryResult.close();
+                }
+                if (statement != null) {
+                    statement.close();
+                }
+            } catch (SQLException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+            }
+            SQLiteConnectionManager.getInstance().shutdown();
+        }
+    }
+
+    @Override
+    public List<DatabaseColumn> getColumns(DatabaseConfiguration dbConfig, String query)
+                    throws DatabaseServiceException {
+        try {
+            Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig);
+            Statement statement = connection.createStatement();
+            ResultSet queryResult = statement.executeQuery(query);
+            ResultSetMetaData metadata = queryResult.getMetaData();
+            int columnCount = metadata.getColumnCount();
+            ArrayList<DatabaseColumn> columns = new ArrayList<>(columnCount);
+            for (int i = 1; i <= columnCount; i++) {
+                DatabaseColumn dc = new DatabaseColumn(metadata.getColumnName(i), metadata.getColumnLabel(i),
+                                DatabaseUtils.getDbColumnType(metadata.getColumnType(i)),
+                                metadata.getColumnDisplaySize(i));
+                columns.add(dc);
+            }
+            return columns;
+        } catch (SQLException e) {
+            logger.error("SQLException::", e);
+            throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
+        }
+    }
+
+    @Override
+    public List<DatabaseRow> getRows(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException {
+        try {
+            Connection connection = SQLiteConnectionManager.getInstance().getConnection(dbConfig);
+            Statement statement = connection.createStatement();
+            statement.setFetchSize(10);
+            ResultSet queryResult = statement.executeQuery(query);
+            ResultSetMetaData metadata = queryResult.getMetaData();
+            int columnCount = metadata.getColumnCount();
+            int index = 0;
+            List<DatabaseRow> rows = new ArrayList<>();
+            while (queryResult.next()) {
+                DatabaseRow row = new DatabaseRow();
+                row.setIndex(index);
+                List<String> values = new ArrayList<>(columnCount);
+                for (int i = 1; i <= columnCount; i++) {
+                    values.add(queryResult.getString(i));
+                }
+                row.setValues(values);
+                rows.add(row);
+                index++;
+            }
+            return rows;
+        } catch (SQLException e) {
+            logger.error("SQLException::", e);
+            throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage());
+        }
+    }
+}

extensions/database/tests/conf/appveyor_tests.xml

@@ -24,6 +24,14 @@
     <parameter name = "mariaDbUser" value="root"/>
     <parameter name = "mariaDbPassword" value=""/>
     <parameter name = "mariaDbTestTable" value="test_table"/>
+
+    <parameter name = "sqliteDbName" value="extension_test_db.sqlite"/>
+    <parameter name = "sqliteDbHost" value=""/>
+    <parameter name = "sqliteDbPort" value=""/>
+    <parameter name = "sqliteDbUser" value=""/>
+    <parameter name = "sqliteDbPassword" value=""/>
+    <parameter name = "sqliteDbTestTable" value="test_table"/>
+
     <test name="DatabaseExtensionUnitTest">
         <groups>
             <run>
@@ -45,6 +53,8 @@
             <class name="com.google.refine.extension.database.mysql.MySQLDatabaseServiceTest"/>
             <class name="com.google.refine.extension.database.pgsql.PgSQLConnectionManagerTest"/>
             <class name="com.google.refine.extension.database.pgsql.PgSQLDatabaseServiceTest" />
+            <class name="com.google.refine.extension.database.sqlite.SQLiteConnectionManagerTest"/>
+            <class name="com.google.refine.extension.database.sqlite.SQLiteDatabaseServiceTest" />
         </classes>
     </test>
 </suite>

extensions/database/tests/conf/tests.xml

@@ -28,12 +28,21 @@
     <parameter name = "mariaDbUser" value="root"/>
     <parameter name = "mariaDbPassword" value=""/>
     <parameter name = "mariaDbTestTable" value="test_table"/>
+
+    <parameter name = "sqliteDbName" value="extension_test_db.sqlite"/>
+    <parameter name = "sqliteDbHost" value=""/>
+    <parameter name = "sqliteDbPort" value=""/>
+    <parameter name = "sqliteDbUser" value=""/>
+    <parameter name = "sqliteDbPassword" value=""/>
+    <parameter name = "sqliteDbTestTable" value="test_table"/>
+
     <test name="DatabaseExtensionUnitTest">
         <groups>
             <run>
                 <exclude name="requiresMySQL"/>
                 <exclude name="requiresPgSQL"/>
                 <exclude name="requiresMariaDB" />
+                <exclude name="requiresSQLite" />
             </run>
         </groups> <!--
         <classes>

extensions/database/tests/conf/travis-sqlite.sql

@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS test_table (
+  id integer NOT NULL,
+  ue_id char(8) NOT NULL,
+  start_time timestamp NOT NULL,
+  end_date date DEFAULT NULL,
+  bytes_upload integer NOT NULL,
+  bytes_download integer NOT NULL,
+  mcc char(3) DEFAULT NULL,
+  mnc char(3) NOT NULL,
+  lac varchar(11) DEFAULT NULL,
+  imei char(16) NOT NULL
+);
+
+INSERT INTO test_table(id, ue_id, start_time, end_date, bytes_upload, bytes_download, mcc, mnc, lac, imei)
+     VALUES (1, '11100022', now(), now(), 1024, 2048, 321, 543, 12209823498,  1344498988877487);

extensions/database/tests/conf/travis_tests.xml

@@ -20,6 +20,14 @@
 	<parameter name = "mariaDbUser" value="root"/>
     <parameter name = "mariaDbPassword" value=""/>
     <parameter name = "mariaDbTestTable" value="test_table"/>
+
+	<parameter name = "sqliteDbName" value="extension_test_db.sqlite"/>
+	<parameter name = "sqliteDbHost" value=""/>
+	<parameter name = "sqliteDbPort" value=""/>
+	<parameter name = "sqliteDbUser" value=""/>
+	<parameter name = "sqliteDbPassword" value=""/>
+	<parameter name = "sqliteDbTestTable" value="test_table"/>
+
 	<test name="DatabaseExtensionUnitTest">
         <!-- Temporarily exclude MySQL and MariaDB from Travis tests until MySQL is working on Travis again. --> 
         <!-- See https://github.com/OpenRefine/OpenRefine/issues/2043 -->
@@ -42,6 +50,8 @@
 			<class name="com.google.refine.extension.database.mysql.MySQLDatabaseServiceTest"/>
 		    <class name="com.google.refine.extension.database.pgsql.PgSQLConnectionManagerTest"/>
 			<class name="com.google.refine.extension.database.pgsql.PgSQLDatabaseServiceTest" />
+			<class name="com.google.refine.extension.database.sqlite.SQLiteConnectionManagerTest"/>
+			<class name="com.google.refine.extension.database.sqlite.SQLiteDatabaseServiceTest" />
 		</classes>
 	</test>
 </suite>

extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTestUtils.java

@@ -66,10 +66,15 @@ public class DBExtensionTestUtils {
              DatabaseMetaData dbm = conn.getMetaData();
              // check if "employee" table is there
              ResultSet tables = dbm.getTables(null, null, tableName, null);
+             boolean dropTable = false;
              if (tables.next()) {
-                 stmt.executeUpdate("DROP TABLE " + tableName);
+                 dropTable = true;
                  //System.out.println("Drop Table Result::" + dropResult);
              }
+             tables.close();
+             if (dropTable) {
+                 stmt.executeUpdate("DROP TABLE " + tableName);
+             }
           
             String createSQL = " CREATE TABLE " + tableName + " ( "
                     + " ID   INT  NOT NULL, "
@@ -123,7 +128,13 @@ public class DBExtensionTestUtils {
              DatabaseMetaData dbm = conn.getMetaData();
              // check if "employee" table is there
              ResultSet tables = dbm.getTables(null, null, DEFAULT_TEST_TABLE, null);
+             boolean dropTable = false;
              if (tables.next()) {
+                dropTable = true;
+                //System.out.println("Drop Table Result::" + dropResult);
+             }
+             tables.close();
+             if (dropTable) {
                  stmt.executeUpdate("DROP TABLE " + DEFAULT_TEST_TABLE);
                  //System.out.println("Drop Table Result::" + dropResult);
              }

extensions/database/tests/src/com/google/refine/extension/database/DBExtensionTests.java

@@ -61,6 +61,9 @@ public class DBExtensionTests {
     protected final String DEFAULT_MARIADB_PASSWORD = "secret";
     protected final String DEFAULT_MARIADB_NAME = "testdb";
 
+    protected final String SQLITE_DB_NAME = "sqlite";
+    protected final String DEFAULT_SQLITE_DB_NAME = "extension_test_db.sqlite";
+    
     protected final String DEFAULT_TEST_TABLE = "test_data";
     
     protected Properties properties;

extensions/database/tests/src/com/google/refine/extension/database/DatabaseServiceTest.java

@@ -4,6 +4,7 @@ package com.google.refine.extension.database;
 import java.sql.Connection;
 import java.util.List;
 
+import com.google.refine.extension.database.sqlite.SQLiteDatabaseService;
 import org.testng.Assert;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Optional;
@@ -85,6 +86,14 @@ public class DatabaseServiceTest extends DBExtensionTests{
         Assert.assertEquals(dbService.getClass(), MariaDBDatabaseService.class);
     }
 
+    @Test(groups = {"requiresSQLite"})
+    public void testGetSQLiteDBService() {
+
+        DatabaseService dbService = DatabaseService.get(SQLiteDatabaseService.DB_NAME);
+        Assert.assertNotNull(dbService);
+        Assert.assertEquals(dbService.getClass(), SQLiteDatabaseService.class);
+    }
+
     @Test(groups = {"requiresMySQL"})
     public void testGetConnection() throws DatabaseServiceException {
        DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType());

extensions/database/tests/src/com/google/refine/extension/database/DatabaseTestConfig.java

@@ -16,11 +16,13 @@ public class DatabaseTestConfig extends DBExtensionTests {
     private DatabaseConfiguration mysqlDbConfig;
     private DatabaseConfiguration pgsqlDbConfig;
     private DatabaseConfiguration mariadbDbConfig;
+    private DatabaseConfiguration sqliteDbConfig;
 
     @BeforeSuite
     @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable",
                   "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable",
-                  "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbyDbUser", "mariadbDbPassword", "mariadbTestTable"})
+                  "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbyDbUser", "mariadbDbPassword", "mariadbTestTable",
+                  "sqliteDbName", "sqliteTestTable"})
     public void beforeSuite(
             @Optional(DEFAULT_MYSQL_DB_NAME)   String mySqlDbName,     @Optional(DEFAULT_MYSQL_HOST)  String mySqlDbHost, 
             @Optional(DEFAULT_MYSQL_PORT)      String mySqlDbPort,     @Optional(DEFAULT_MYSQL_USER)  String mySqlDbUser,
@@ -32,7 +34,9 @@ public class DatabaseTestConfig extends DBExtensionTests {
             
             @Optional(DEFAULT_MARIADB_NAME)      String mariadbDbName,     @Optional(DEFAULT_MARIADB_HOST)  String mariadbDbHost, 
             @Optional(DEFAULT_MARIADB_PORT)      String mariadbDbPort,     @Optional(DEFAULT_MARIADB_USER)  String mariadbyDbUser,
-            @Optional(DEFAULT_MARIADB_PASSWORD)  String mariadbDbPassword, @Optional(DEFAULT_TEST_TABLE)    String mariadbTestTable)
+            @Optional(DEFAULT_MARIADB_PASSWORD)  String mariadbDbPassword, @Optional(DEFAULT_TEST_TABLE)    String mariadbTestTable,
+
+            @Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable)
                     throws DatabaseServiceException, SQLException {
         
         //System.out.println("@BeforeSuite\n");
@@ -63,9 +67,13 @@ public class DatabaseTestConfig extends DBExtensionTests {
         mariadbDbConfig.setDatabaseUser(mariadbyDbUser);
         mariadbDbConfig.setUseSSL(false);
 
+        sqliteDbConfig = new DatabaseConfiguration();
+        sqliteDbConfig.setDatabaseName(sqliteDbName);
+    
         DBExtensionTestUtils.initTestData(mysqlDbConfig);
         DBExtensionTestUtils.initTestData(pgsqlDbConfig);
         DBExtensionTestUtils.initTestData(mariadbDbConfig);
+        DBExtensionTestUtils.initTestData(sqliteDbConfig);
     }
   
     @AfterSuite
@@ -75,6 +83,7 @@ public class DatabaseTestConfig extends DBExtensionTests {
         DBExtensionTestUtils.cleanUpTestData(mysqlDbConfig);
         DBExtensionTestUtils.cleanUpTestData(pgsqlDbConfig);
         DBExtensionTestUtils.cleanUpTestData(mariadbDbConfig);
+        DBExtensionTestUtils.cleanUpTestData(sqliteDbConfig);
     }
 
 }

extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteConnectionManagerTest.java

@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2020, Chris Parker
+ *        All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * Neither the name of Google nor the names of its contributors may be used to
+ * endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.google.refine.extension.database.sqlite;
+
+import com.google.refine.extension.database.DBExtensionTests;
+import com.google.refine.extension.database.DatabaseConfiguration;
+import com.google.refine.extension.database.DatabaseService;
+import com.google.refine.extension.database.DatabaseServiceException;
+import org.mockito.MockitoAnnotations;
+import org.testng.Assert;
+import org.testng.annotations.*;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+@Test(groups = { "requiresSQLite" })
+public class SQLiteConnectionManagerTest extends DBExtensionTests {
+
+    private DatabaseConfiguration testDbConfig;
+
+    @BeforeTest
+    @Parameters({ "sqliteDbName", "sqliteTestTable" })
+    public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName,
+                    @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable)
+                    throws DatabaseServiceException, SQLException {
+
+        MockitoAnnotations.initMocks(this);
+
+        testDbConfig = new DatabaseConfiguration();
+        testDbConfig.setDatabaseName(sqliteDbName);
+        testDbConfig.setDatabaseType(SQLiteDatabaseService.DB_NAME);
+
+        DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance());
+    }
+
+    @AfterTest
+    @Parameters({ "sqliteDbName" })
+    public void afterTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName) {
+        File f = new File(sqliteDbName);
+        if (f.exists()) {
+            f.delete();
+        }
+    }
+
+    @Test
+    public void testTestConnection() throws DatabaseServiceException {
+        boolean isConnected = SQLiteConnectionManager.getInstance().testConnection(testDbConfig);
+        Assert.assertTrue(isConnected);
+    }
+
+    @Test
+    public void testGetConnection() throws DatabaseServiceException {
+
+        Connection conn = SQLiteConnectionManager.getInstance().getConnection(testDbConfig);
+        Assert.assertNotNull(conn);
+    }
+
+    @Test
+    public void testShutdown() throws DatabaseServiceException, SQLException {
+
+        Connection conn = SQLiteConnectionManager.getInstance().getConnection(testDbConfig);
+        Assert.assertNotNull(conn);
+
+        SQLiteConnectionManager.getInstance().shutdown();
+
+        if (conn != null) {
+            Assert.assertTrue(conn.isClosed());
+        }
+
+    }
+}

extensions/database/tests/src/com/google/refine/extension/database/sqlite/SQLiteDatabaseServiceTest.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2020, Chris Parker
+ *        All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * - Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * Neither the name of Google nor the names of its contributors may be used to
+ * endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.google.refine.extension.database.sqlite;
+
+import com.google.refine.extension.database.*;
+import com.google.refine.extension.database.model.DatabaseColumn;
+import com.google.refine.extension.database.model.DatabaseInfo;
+import com.google.refine.extension.database.model.DatabaseRow;
+import org.mockito.MockitoAnnotations;
+import org.testng.Assert;
+import org.testng.annotations.*;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.List;
+
+@Test(groups = { "requiresSQLite" })
+public class SQLiteDatabaseServiceTest extends DBExtensionTests {
+
+    private DatabaseConfiguration testDbConfig;
+    private String testTable;
+
+    @BeforeTest
+    @Parameters({ "sqliteDbName", "sqliteTestTable" })
+    public void beforeTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName,
+                    @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable)
+                    throws DatabaseServiceException, SQLException {
+
+        MockitoAnnotations.initMocks(this);
+        testDbConfig = new DatabaseConfiguration();
+        testDbConfig.setDatabaseName(sqliteDbName);
+        testDbConfig.setDatabaseType(SQLiteDatabaseService.DB_NAME);
+
+        testTable = sqliteTestTable;
+        DBExtensionTestUtils.initTestData(testDbConfig, sqliteTestTable);
+
+        DatabaseService.DBType.registerDatabase(SQLiteDatabaseService.DB_NAME, SQLiteDatabaseService.getInstance());
+    }
+
+    @AfterTest
+    @Parameters({ "sqliteDbName" })
+    public void afterTest(@Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName) {
+        File f = new File(sqliteDbName);
+        if (f.exists()) {
+            f.delete();
+        }
+    }
+
+    @Test
+    public void testGetDatabaseUrl() {
+        SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
+                        .get(SQLiteDatabaseService.DB_NAME);
+        String dbUrl = sqLiteDatabaseService.getDatabaseUrl(testDbConfig);
+
+        Assert.assertNotNull(dbUrl);
+        Assert.assertEquals(dbUrl, "jdbc:sqlite:extension_test_db.sqlite");
+    }
+
+    @Test
+    public void testGetConnection() throws DatabaseServiceException {
+
+        SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
+                        .get(SQLiteDatabaseService.DB_NAME);
+        Connection conn = sqLiteDatabaseService.getConnection(testDbConfig);
+
+        Assert.assertNotNull(conn);
+    }
+
+    @Test
+    public void testTestConnection() throws DatabaseServiceException {
+        SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
+                        .get(SQLiteDatabaseService.DB_NAME);
+
+        boolean result = sqLiteDatabaseService.testConnection(testDbConfig);
+        Assert.assertTrue(result);
+    }
+
+    @Test
+    public void testConnect() throws DatabaseServiceException {
+
+        SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
+                        .get(SQLiteDatabaseService.DB_NAME);
+        DatabaseInfo databaseInfo = sqLiteDatabaseService.connect(testDbConfig);
+        Assert.assertNotNull(databaseInfo);
+    }
+
+    @Test
+    public void testExecuteQuery() throws DatabaseServiceException {
+        SQLiteDatabaseService sqLiteDatabaseService = (SQLiteDatabaseService) DatabaseService
+                        .get(SQLiteDatabaseService.DB_NAME);
+        DatabaseInfo databaseInfo = sqLiteDatabaseService.testQuery(testDbConfig, "SELECT * FROM " + testTable);
+
+        Assert.assertNotNull(databaseInfo);
+    }
+
+    @Test
+    public void testBuildLimitQuery() {
+        SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService
+                        .get(SQLiteDatabaseService.DB_NAME);
+        String limitQuery = sqliteSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable);
+        Assert.assertNotNull(limitQuery);
+        Assert.assertEquals(limitQuery,
+                        "SELECT * FROM (SELECT * FROM " + testTable + ") data LIMIT " + 100 + " OFFSET " + 0 + ";");
+    }
+
+    @Test
+    public void testGetRows() throws DatabaseServiceException {
+        SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService
+                        .get(SQLiteDatabaseService.DB_NAME);
+        List<DatabaseRow> dbRows = sqliteSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable);
+
+        Assert.assertNotNull(dbRows);
+    }
+
+    @Test
+    public void testGetInstance() {
+        SQLiteDatabaseService instance = SQLiteDatabaseService.getInstance();
+        Assert.assertNotNull(instance);
+    }
+
+    @Test
+    public void testGetColumns() throws DatabaseServiceException {
+        List<DatabaseColumn> dbColumns;
+
+        SQLiteDatabaseService sqliteSqlService = (SQLiteDatabaseService) DatabaseService
+                        .get(SQLiteDatabaseService.DB_NAME);
+
+        dbColumns = sqliteSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable);
+
+        Assert.assertNotNull(dbColumns);
+    }
+}