diff --git a/.settings/org.eclipse.wst.jsdt.core.prefs b/.settings/org.eclipse.wst.jsdt.core.prefs index 0a9384361..7b594a52b 100644 --- a/.settings/org.eclipse.wst.jsdt.core.prefs +++ b/.settings/org.eclipse.wst.jsdt.core.prefs @@ -186,7 +186,6 @@ org.eclipse.wst.jsdt.core.formatter.insert_space_after_comma_in_superinterfaces= org.eclipse.wst.jsdt.core.formatter.insert_space_after_comma_in_type_arguments=insert org.eclipse.wst.jsdt.core.formatter.insert_space_after_comma_in_type_parameters=insert org.eclipse.wst.jsdt.core.formatter.insert_space_after_ellipsis=insert -org.eclipse.wst.jsdt.core.formatter.insert_space_after_function_keyword=do not insert org.eclipse.wst.jsdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert org.eclipse.wst.jsdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert org.eclipse.wst.jsdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert diff --git a/extensions/build.xml b/extensions/build.xml index 0cd013394..1b7fa19ad 100644 --- a/extensions/build.xml +++ b/extensions/build.xml @@ -13,7 +13,6 @@ - @@ -22,6 +21,5 @@ - diff --git a/extensions/database/.classpath b/extensions/database/.classpath index 4831eb720..8a8601a1d 100644 --- a/extensions/database/.classpath +++ b/extensions/database/.classpath @@ -11,12 +11,13 @@ - + + diff --git a/extensions/database/.eclipse-pmd b/extensions/database/.eclipse-pmd new file mode 100644 index 000000000..4a705f215 --- /dev/null +++ b/extensions/database/.eclipse-pmd @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/extensions/database/module/MOD-INF/lib/mockito-all-1.10.19.jar b/extensions/database/module/MOD-INF/lib/mockito-all-1.10.19.jar deleted file mode 100644 index c831489cd..000000000 Binary files a/extensions/database/module/MOD-INF/lib/mockito-all-1.10.19.jar and /dev/null differ diff --git a/extensions/database/module/MOD-INF/lib/powermock-api-mockito-1.7.3.jar b/extensions/database/module/MOD-INF/lib/powermock-api-mockito-1.7.3.jar deleted file mode 100644 index c8e1bba0e..000000000 Binary files a/extensions/database/module/MOD-INF/lib/powermock-api-mockito-1.7.3.jar and /dev/null differ diff --git a/extensions/database/module/scripts/index/database-import-controller.js b/extensions/database/module/scripts/index/database-import-controller.js index ded053dbf..e76dfe845 100644 --- a/extensions/database/module/scripts/index/database-import-controller.js +++ b/extensions/database/module/scripts/index/database-import-controller.js @@ -265,10 +265,17 @@ Refine.DatabaseImportController.prototype._updatePreview = function() { new Refine.PreviewTable(projectData, self._parsingPanelElmts.dataPanel.unbind().empty()); }); } else { - alert(result.message); - //self._parsingPanelElmts.progressPanel.hide(); - alert('Errors:\n' + - (result.message) ? result.message : Refine.CreateProjectUI.composeErrorMessage(job)); + + alert('Errors:\n' + (result.message) ? result.message : Refine.CreateProjectUI.composeErrorMessage(job)); + self._parsingPanelElmts.progressPanel.hide(); + + Refine.CreateProjectUI.cancelImportingJob(self._jobID); + + delete self._jobID; + delete self._options; + + self._createProjectUI.showSourceSelectionPanel(); + } }, diff --git a/extensions/database/src/com/google/refine/extension/database/DBQueryResultImportReader.java b/extensions/database/src/com/google/refine/extension/database/DBQueryResultImportReader.java index 0b2c9c3fe..2deb98bfa 100644 --- a/extensions/database/src/com/google/refine/extension/database/DBQueryResultImportReader.java +++ b/extensions/database/src/com/google/refine/extension/database/DBQueryResultImportReader.java @@ -118,12 +118,16 @@ public class DBQueryResultImportReader implements TableDataReader { setProgress(job, querySource, progress++); } if(processedRows % 10000 == 0) { + if(logger.isDebugEnabled()) { logger.debug("[[ {} rows processed... ]]",processedRows); + } } } return result; } else { - logger.debug("[[processedRows:{} ]]", processedRows); + if(logger.isDebugEnabled()) { + logger.debug("[[processedRows:{} ]]", processedRows); + } return null; } @@ -171,16 +175,17 @@ public class DBQueryResultImportReader implements TableDataReader { rowOfCells.add(Long.parseLong(text)); continue; } catch (NumberFormatException e) {} - - try { - double d = Double.parseDouble(text); - if (!Double.isInfinite(d) && !Double.isNaN(d)) { - rowOfCells.add(d); - continue; - } - } catch (NumberFormatException e) {} - - } + + }else if(col.getType() == DatabaseColumnType.DOUBLE || col.getType() == DatabaseColumnType.FLOAT ) { + try { + double d = Double.parseDouble(text); + if (!Double.isInfinite(d) && !Double.isNaN(d)) { + rowOfCells.add(d); + continue; + } + } catch (NumberFormatException e) {} + + } rowOfCells.add(text); } diff --git a/extensions/database/src/com/google/refine/extension/database/DBQueryResultPreviewReader.java b/extensions/database/src/com/google/refine/extension/database/DBQueryResultPreviewReader.java index 6910362bb..2033591fd 100644 --- a/extensions/database/src/com/google/refine/extension/database/DBQueryResultPreviewReader.java +++ b/extensions/database/src/com/google/refine/extension/database/DBQueryResultPreviewReader.java @@ -107,23 +107,21 @@ public class DBQueryResultPreviewReader implements TableDataReader { //logger.info("Exit::getNextRowOfCells :rowsOfCellsNotNull::rowsOfCells size:" + rowsOfCells.size() + ":batchRowStart:" + batchRowStart + " ::nextRow:" + nextRow); return rowsOfCells.get(nextRow++ - batchRowStart); } else { - logger.debug("nextRow:{}, batchRowStart:{}", nextRow, batchRowStart); -// -// rowsOfCells = getRowsOfCells(batchRowStart); -// if(rowsOfCells != null) { -// return rowsOfCells.get(nextRow++ - batchRowStart); -// } + if(logger.isDebugEnabled()) { + logger.debug("nextRow:{}, batchRowStart:{}", nextRow, batchRowStart); + } + return null; } }catch(DatabaseServiceException e) { - logger.error("DatabaseServiceException::{}", e); - throw new IOException(e); + logger.error("DatabaseServiceException::preview:{}", e.getMessage()); + IOException ioEx = new IOException(e.getMessage(), e); + throw ioEx; } - - + } /** @@ -139,7 +137,9 @@ public class DBQueryResultPreviewReader implements TableDataReader { List> rowsOfCells = new ArrayList>(batchSize); String query = databaseService.buildLimitQuery(batchSize, startRow, dbQueryInfo.getQuery()); - logger.debug("batchSize::" + batchSize + " startRow::" + startRow + " query::" + query ); + if(logger.isDebugEnabled()) { + logger.debug("batchSize::" + batchSize + " startRow::" + startRow + " query::" + query ); + } List dbRows = databaseService.getRows(dbQueryInfo.getDbConfig(), query); diff --git a/extensions/database/src/com/google/refine/extension/database/DatabaseColumnType.java b/extensions/database/src/com/google/refine/extension/database/DatabaseColumnType.java index a6c0cddda..79a0c8f3a 100644 --- a/extensions/database/src/com/google/refine/extension/database/DatabaseColumnType.java +++ b/extensions/database/src/com/google/refine/extension/database/DatabaseColumnType.java @@ -34,6 +34,10 @@ public enum DatabaseColumnType { STRING, NUMBER, DATETIME, - LOCATION + LOCATION, + BOOLEAN, + DATE, + DOUBLE, + FLOAT } diff --git a/extensions/database/src/com/google/refine/extension/database/DatabaseImportController.java b/extensions/database/src/com/google/refine/extension/database/DatabaseImportController.java index 4d67d4652..7f9fdff1c 100644 --- a/extensions/database/src/com/google/refine/extension/database/DatabaseImportController.java +++ b/extensions/database/src/com/google/refine/extension/database/DatabaseImportController.java @@ -52,7 +52,6 @@ import com.google.refine.commands.HttpUtilities; import com.google.refine.extension.database.model.DatabaseColumn; import com.google.refine.extension.database.model.DatabaseQueryInfo; import com.google.refine.importers.TabularImportingParserBase; -import com.google.refine.importing.DefaultImportingController; import com.google.refine.importing.ImportingController; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingManager; @@ -82,23 +81,27 @@ public class DatabaseImportController implements ImportingController { @Override public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - logger.info("DatabaseImportController::doPost::databaseName::{}", request.getParameter("databaseName")); - + logger.info("doPost Query String::{}", request.getQueryString()); + response.setCharacterEncoding("UTF-8"); Properties parameters = ParsingUtilities.parseUrlParameters(request); String subCommand = parameters.getProperty("subCommand"); - logger.debug("DatabaseImportController::doPost::subCommand::{}", subCommand); + if(logger.isDebugEnabled()){ + logger.info("doPost::subCommand::{}", subCommand); + } if ("initialize-parser-ui".equals(subCommand)) { doInitializeParserUI(request, response, parameters); } else if ("parse-preview".equals(subCommand)) { try { + doParsePreview(request, response, parameters); + } catch (DatabaseServiceException e) { - logger.error("DatabaseImportController::doPost::DatabaseServiceException::{}", e); - HttpUtilities.respond(response, "error", e.getMessage()); + logger.error("doPost::DatabaseServiceException::{}", e); + HttpUtilities.respond(response, "error", getDbServiceException(e)); } } else if ("create-project".equals(subCommand)) { doCreateProject(request, response, parameters); @@ -108,6 +111,19 @@ public class DatabaseImportController implements ImportingController { } + private String getDbServiceException(Exception ex) { + String message = ""; + if(ex instanceof DatabaseServiceException) { + DatabaseServiceException dbEx = (DatabaseServiceException) ex; + if(dbEx.isSqlException()) { + message = message + dbEx.getSqlCode() + " " + dbEx.getSqlState(); + } + } + message = message + ex.getMessage(); + + return message; + } + /** * * @param request @@ -118,7 +134,10 @@ public class DatabaseImportController implements ImportingController { */ private void doInitializeParserUI(HttpServletRequest request, HttpServletResponse response, Properties parameters) throws ServletException, IOException { - logger.info("::doInitializeParserUI::"); + if(logger.isDebugEnabled()) { + logger.debug("::doInitializeParserUI::"); + } + JSONObject result = new JSONObject(); JSONObject options = new JSONObject(); @@ -128,8 +147,10 @@ public class DatabaseImportController implements ImportingController { JSONUtilities.safePut(options, "skipDataLines", 0); JSONUtilities.safePut(options, "storeBlankRows", true); JSONUtilities.safePut(options, "storeBlankCellsAsNulls", true); - - logger.debug("doInitializeParserUI:::{}", result.toString()); + if(logger.isDebugEnabled()) { + logger.debug("doInitializeParserUI:::{}", result.toString()); + } + HttpUtilities.respond(response, result.toString()); } @@ -147,8 +168,11 @@ public class DatabaseImportController implements ImportingController { private void doParsePreview( HttpServletRequest request, HttpServletResponse response, Properties parameters) throws ServletException, IOException, DatabaseServiceException { - logger.info("DatabaseImportController::doParsePreview::JobID::{}", parameters.getProperty("jobID")); - + if(logger.isDebugEnabled()) { + logger.debug("JobID::{}", parameters.getProperty("jobID")); + } + logger.info("JobID::{}", parameters.getProperty("jobID")); + long jobID = Long.parseLong(parameters.getProperty("jobID")); ImportingJob job = ImportingManager.getJob(jobID); if (job == null) { @@ -179,6 +203,8 @@ public class DatabaseImportController implements ImportingController { optionObj, exceptions ); +// String exStr = getExceptionString(exceptions); +// logger.info("exceptions::" + exStr); Writer w = response.getWriter(); JSONWriter writer = new JSONWriter(w); @@ -186,13 +212,16 @@ public class DatabaseImportController implements ImportingController { writer.object(); if (exceptions.size() == 0) { job.project.update(); // update all internal models, indexes, caches, etc. - writer.key("status"); writer.value("ok"); + writer.key("status"); + writer.value("ok"); } else { - writer.key("status"); writer.value("error"); - writer.key("errors"); - writer.array(); - DefaultImportingController.writeErrors(writer, exceptions); - writer.endArray(); + writer.key("status"); + writer.value("error"); + writer.key("message"); + writer.value(getExceptionString(exceptions)); +// writer.array(); +// writeErrors(writer, exceptions); +// writer.endArray(); } writer.endObject(); } catch (JSONException e) { @@ -210,6 +239,17 @@ public class DatabaseImportController implements ImportingController { } } + + + private String getExceptionString(List exceptions) { + String ex = ""; + for(Exception e: exceptions) { + ex = ex + e.getLocalizedMessage() + "\n"; + } + // TODO Auto-generated method stub + return ex; + } + /** * * @param dbQueryInfo @@ -266,7 +306,9 @@ public class DatabaseImportController implements ImportingController { */ private void doCreateProject(HttpServletRequest request, HttpServletResponse response, Properties parameters) throws ServletException, IOException{ - logger.info("DatabaseImportController::doCreateProject:::{}", parameters.getProperty("jobID")); + if(logger.isDebugEnabled()) { + logger.debug("DatabaseImportController::doCreateProject:::{}", parameters.getProperty("jobID")); + } long jobID = Long.parseLong(parameters.getProperty("jobID")); final ImportingJob job = ImportingManager.getJob(jobID); @@ -309,7 +351,7 @@ public class DatabaseImportController implements ImportingController { exceptions ); } catch (DatabaseServiceException e) { - logger.info("DatabaseImportController::doCreateProject:::run{}", e); + logger.error("DatabaseImportController::doCreateProject:::run{}", e); // throw new RuntimeException("DatabaseServiceException::", e); } @@ -380,8 +422,9 @@ public class DatabaseImportController implements ImportingController { ); long endTime = System.currentTimeMillis() ; - - logger.debug("Execution Time: {}", endTime - startTime); + if(logger.isDebugEnabled()) { + logger.debug("Execution Time: {}", endTime - startTime); + } setProgress(job, querySource, 100); @@ -420,8 +463,9 @@ public class DatabaseImportController implements ImportingController { if (jdbcConfig.getDatabaseHost() == null || jdbcConfig.getDatabaseName() == null || jdbcConfig.getDatabasePassword() == null || jdbcConfig.getDatabaseType() == null || jdbcConfig.getDatabaseUser() == null || query == null) { - - logger.debug("Missing Database Configuration::{}", jdbcConfig); + if(logger.isDebugEnabled()) { + logger.debug("Missing Database Configuration::{}", jdbcConfig); + } return null; } diff --git a/extensions/database/src/com/google/refine/extension/database/DatabaseModuleImpl.java b/extensions/database/src/com/google/refine/extension/database/DatabaseModuleImpl.java index e1e2b7f73..45bb6aff7 100644 --- a/extensions/database/src/com/google/refine/extension/database/DatabaseModuleImpl.java +++ b/extensions/database/src/com/google/refine/extension/database/DatabaseModuleImpl.java @@ -52,6 +52,10 @@ public class DatabaseModuleImpl extends ButterflyModuleImpl implements Jsonizabl public static DatabaseModuleImpl instance; public static Properties extensionProperties; + + private static String DEFAULT_CREATE_PROJ_BATCH_SIZE = "100"; + private static String DEFAULT_PREVIEW_BATCH_SIZE = "100"; + @Override @@ -66,27 +70,35 @@ public class DatabaseModuleImpl extends ButterflyModuleImpl implements Jsonizabl // Set the singleton. instance = this; - logger.info("*** Database Extension Module init complete ***"); + logger.info("*** Database Extension Module Initialization Completed!!***"); } public static String getImportCreateBatchSize() { - - return extensionProperties.getProperty("create.batchSize", "100"); + if(extensionProperties == null) { + return DEFAULT_CREATE_PROJ_BATCH_SIZE; + } + return extensionProperties.getProperty("create.batchSize", DEFAULT_CREATE_PROJ_BATCH_SIZE); } public static String getImportPreviewBatchSize() { - - return extensionProperties.getProperty("preview.batchSize", "100"); + if(extensionProperties == null) { + return DEFAULT_PREVIEW_BATCH_SIZE; + } + return extensionProperties.getProperty("preview.batchSize", DEFAULT_PREVIEW_BATCH_SIZE); } private void readModuleProperty() { // The module path File f = getPath(); - logger.debug("Module getPath(): {}", f.getPath()); + if(logger.isDebugEnabled()) { + logger.debug("Module getPath(): {}", f.getPath()); + } // Load our custom properties. File modFile = new File(f,"MOD-INF"); - logger.debug("Module File: {}", modFile.getPath()); + if(logger.isDebugEnabled()) { + logger.debug("Module File: {}", modFile.getPath()); + } if (modFile.exists()) { @@ -100,7 +112,9 @@ public class DatabaseModuleImpl extends ButterflyModuleImpl implements Jsonizabl Properties ps = new Properties(); try { if (propFile.exists()) { - logger.debug("Loading Extension properties ({})", propFile); + if(logger.isDebugEnabled()) { + logger.debug("Loading Extension properties ({})", propFile); + } BufferedInputStream stream = null; try { ps = new Properties(); diff --git a/extensions/database/src/com/google/refine/extension/database/DatabaseUtils.java b/extensions/database/src/com/google/refine/extension/database/DatabaseUtils.java index f33a06cfe..4b96d1a21 100644 --- a/extensions/database/src/com/google/refine/extension/database/DatabaseUtils.java +++ b/extensions/database/src/com/google/refine/extension/database/DatabaseUtils.java @@ -49,12 +49,21 @@ public class DatabaseUtils { private static final Logger logger = LoggerFactory.getLogger("DatabaseUtils"); - private final static String DATABASE_EXTENSION_DIR = "dbextension"; - private final static String SETTINGS_FILE_NAME = ".saved-db-connections.json"; + public final static String DATABASE_EXTENSION_DIR = "dbextension"; + public final static String SETTINGS_FILE_NAME = ".saved-db-connections.json"; public final static String SAVED_CONNECTION_KEY = "savedConnections"; private static SimpleTextEncryptor textEncryptor = new SimpleTextEncryptor("Aa1Gb@tY7_Y"); + + public static int getSavedConnectionsSize() { + List scList = getSavedConnections(); + if(scList == null || scList.isEmpty()) { + return 0; + } + + return scList.size(); + } /** * GET saved connections * @return @@ -152,10 +161,6 @@ public class DatabaseUtils { public static void addToSavedConnections(DatabaseConfiguration dbConfig){ try { - -// String encPassword = encrypt(dbConfig.getDatabasePassword()); -// dbConfig.setDatabasePassword(encPassword); - ObjectMapper mapper = new ObjectMapper(); String savedConnectionFile = getExtensionFilePath(); SavedConnectionContainer savedConnectionContainer = mapper.readValue(new File(savedConnectionFile), SavedConnectionContainer.class); @@ -165,29 +170,67 @@ public class DatabaseUtils { } catch (JsonGenerationException e1) { logger.error("JsonGenerationException: {}", e1); - e1.printStackTrace(); + // e1.printStackTrace(); } catch (JsonMappingException e1) { logger.error("JsonMappingException: {}", e1); - e1.printStackTrace(); + // e1.printStackTrace(); } catch (IOException e1) { logger.error("IOException: {}", e1); - e1.printStackTrace(); + // e1.printStackTrace(); } } + + public static void deleteAllSavedConnections() { + logger.info("delete All Saved Connections called..."); + + try { + + List savedConnections = getSavedConnections(); + if(logger.isDebugEnabled()) { + logger.debug("Size before delete SavedConnections :: {}", savedConnections.size()); + } + + ArrayList newSavedConns = new ArrayList(); + + ObjectMapper mapper = new ObjectMapper(); + String savedConnectionFile = getExtensionFilePath(); + SavedConnectionContainer savedConnectionContainer = mapper.readValue(new File(savedConnectionFile), SavedConnectionContainer.class); + savedConnectionContainer.setSavedConnections(newSavedConns); + + if(logger.isDebugEnabled()) { + logger.debug("Size after delete SavedConnections :: {}", savedConnectionContainer.getSavedConnections().size()); + } + mapper.writerWithDefaultPrettyPrinter().writeValue(new File(savedConnectionFile), savedConnectionContainer); + + } catch (JsonGenerationException e1) { + logger.error("JsonGenerationException: {}", e1); + // e1.printStackTrace(); + } catch (JsonMappingException e1) { + logger.error("JsonMappingException: {}", e1); + // e1.printStackTrace(); + } catch (IOException e1) { + logger.error("IOException: {}", e1); + // e1.printStackTrace(); + } + + } + /** * DELETE saved connections * @param connectionName */ public static void deleteSavedConnections(String connectionName) { - logger.info("deleteSavedConnections called with: {}", connectionName); + logger.info("deleteSavedConnections called with: {}", connectionName); try { - List savedConnections = getSavedConnections(); - logger.debug("Size before delete SavedConnections :: {}", savedConnections.size()); - + List savedConnections = getSavedConnections();; + if(logger.isDebugEnabled()) { + logger.debug("Size before delete SavedConnections :: {}", savedConnections.size()); + } + ArrayList newSavedConns = new ArrayList(); for(DatabaseConfiguration dc: savedConnections) { if(!dc.getConnectionName().equalsIgnoreCase(connectionName.trim())) { @@ -201,18 +244,20 @@ public class DatabaseUtils { SavedConnectionContainer savedConnectionContainer = mapper.readValue(new File(savedConnectionFile), SavedConnectionContainer.class); savedConnectionContainer.setSavedConnections(newSavedConns); - logger.debug("Size after delete SavedConnections :: {}", savedConnectionContainer.getSavedConnections().size()); + if(logger.isDebugEnabled()) { + logger.debug("Size after delete SavedConnections :: {}", savedConnectionContainer.getSavedConnections().size()); + } mapper.writerWithDefaultPrettyPrinter().writeValue(new File(savedConnectionFile), savedConnectionContainer); } catch (JsonGenerationException e1) { logger.error("JsonGenerationException: {}", e1); - e1.printStackTrace(); + // e1.printStackTrace(); } catch (JsonMappingException e1) { logger.error("JsonMappingException: {}", e1); - e1.printStackTrace(); + // e1.printStackTrace(); } catch (IOException e1) { logger.error("IOException: {}", e1); - e1.printStackTrace(); + // e1.printStackTrace(); } } @@ -221,9 +266,11 @@ public class DatabaseUtils { * @param jdbcConfig */ public static void editSavedConnections(DatabaseConfiguration jdbcConfig) { - logger.debug("Edit SavedConnections called with: {}", jdbcConfig); - + if(logger.isDebugEnabled()) { + logger.debug("Edit SavedConnections called with: {}", jdbcConfig); + } + try { ObjectMapper mapper = new ObjectMapper(); String savedConnectionFile = getExtensionFilePath(); @@ -263,6 +310,8 @@ public class DatabaseUtils { File dir = ((FileProjectManager) ProjectManager.singleton).getWorkspaceDir(); String fileSep = System.getProperty("file.separator"); String filename = dir.getPath() + fileSep + DATABASE_EXTENSION_DIR + fileSep + SETTINGS_FILE_NAME; + + // logger.info("** extension file name: {} **", filename); return filename; } public static String getExtensionFolder(){ @@ -272,17 +321,17 @@ public class DatabaseUtils { return filename; } - public static DatabaseColumnType getDbColumnType(int dbType) { + public static DatabaseColumnType getDbColumnType(int dbColumnType) { - switch (dbType) { + switch (dbColumnType) { case java.sql.Types.BIGINT: return DatabaseColumnType.NUMBER; case java.sql.Types.FLOAT: - return DatabaseColumnType.STRING; + return DatabaseColumnType.FLOAT; case java.sql.Types.REAL: - return DatabaseColumnType.STRING; + return DatabaseColumnType.DOUBLE; case java.sql.Types.DOUBLE: - return DatabaseColumnType.STRING; + return DatabaseColumnType.DOUBLE; case java.sql.Types.NUMERIC: return DatabaseColumnType.NUMBER; case java.sql.Types.DECIMAL: @@ -294,7 +343,7 @@ public class DatabaseUtils { case java.sql.Types.LONGVARCHAR: return DatabaseColumnType.STRING; case java.sql.Types.DATE: - return DatabaseColumnType.DATETIME; + return DatabaseColumnType.DATE; case java.sql.Types.TIME: return DatabaseColumnType.DATETIME; case java.sql.Types.TIMESTAMP: @@ -323,6 +372,11 @@ public class DatabaseUtils { return DatabaseColumnType.STRING; case java.sql.Types.REF: return DatabaseColumnType.STRING; + case java.sql.Types.BOOLEAN: + return DatabaseColumnType.BOOLEAN; + case java.sql.Types.INTEGER: + return DatabaseColumnType.NUMBER; + default: return DatabaseColumnType.STRING; } diff --git a/extensions/database/src/com/google/refine/extension/database/SavedConnectionContainer.java b/extensions/database/src/com/google/refine/extension/database/SavedConnectionContainer.java index c63e4247e..15a45f731 100644 --- a/extensions/database/src/com/google/refine/extension/database/SavedConnectionContainer.java +++ b/extensions/database/src/com/google/refine/extension/database/SavedConnectionContainer.java @@ -31,6 +31,7 @@ package com.google.refine.extension.database; import java.util.List; public class SavedConnectionContainer { + private List savedConnections; public List getSavedConnections() { return savedConnections; @@ -41,7 +42,7 @@ public class SavedConnectionContainer { this.savedConnections = savedConnections; } - List savedConnections; + public SavedConnectionContainer(List savedConnections) { super(); diff --git a/extensions/database/src/com/google/refine/extension/database/cmd/ConnectCommand.java b/extensions/database/src/com/google/refine/extension/database/cmd/ConnectCommand.java index 594949635..014b30930 100644 --- a/extensions/database/src/com/google/refine/extension/database/cmd/ConnectCommand.java +++ b/extensions/database/src/com/google/refine/extension/database/cmd/ConnectCommand.java @@ -56,11 +56,10 @@ public class ConnectCommand extends DatabaseCommand { public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - logger.debug("ConnectCommand::Post"); DatabaseConfiguration databaseConfiguration = getJdbcConfiguration(request); - - logger.info("ConnectCommand::Post::{}", databaseConfiguration); - + if(logger.isDebugEnabled()) { + logger.debug("ConnectCommand::Post::{}", databaseConfiguration); + } // ProjectManager.singleton.setBusy(true); try { @@ -73,11 +72,11 @@ public class ConnectCommand extends DatabaseCommand { try { DatabaseInfo databaseInfo = DatabaseService.get(databaseConfiguration.getDatabaseType()) .connect(databaseConfiguration); + String databaseInfoString = mapperObj.writeValueAsString(databaseInfo); response.setStatus(HttpStatus.SC_OK); writer.object(); writer.key("code"); writer.value("ok"); - String databaseInfoString = mapperObj.writeValueAsString(databaseInfo); writer.key("databaseInfo"); writer.value(databaseInfoString); @@ -96,9 +95,10 @@ public class ConnectCommand extends DatabaseCommand { } catch (Exception e) { logger.error("ConnectCommand::Post::Exception::{}", e); throw new ServletException(e); - } finally { - // ProjectManager.singleton.setBusy(false); - } + } +// finally { +// // ProjectManager.singleton.setBusy(false); +// } } diff --git a/extensions/database/src/com/google/refine/extension/database/cmd/DatabaseCommand.java b/extensions/database/src/com/google/refine/extension/database/cmd/DatabaseCommand.java index 75989f331..ccd967d95 100644 --- a/extensions/database/src/com/google/refine/extension/database/cmd/DatabaseCommand.java +++ b/extensions/database/src/com/google/refine/extension/database/cmd/DatabaseCommand.java @@ -38,11 +38,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.refine.commands.Command; - import com.google.refine.extension.database.DatabaseConfiguration; import com.google.refine.extension.database.DatabaseServiceException; public abstract class DatabaseCommand extends Command { + + private static final Logger logger = LoggerFactory.getLogger("DatabaseCommand"); /** * @@ -55,12 +56,22 @@ public abstract class DatabaseCommand extends Command { jdbcConfig.setConnectionName(request.getParameter("connectionName")); jdbcConfig.setDatabaseType(request.getParameter("databaseType")); jdbcConfig.setDatabaseHost(request.getParameter("databaseServer")); - jdbcConfig.setDatabasePort(Integer.parseInt(request.getParameter("databasePort"))); + + String dbPort = request.getParameter("databasePort"); + if(dbPort != null) { + try { + jdbcConfig.setDatabasePort(Integer.parseInt(dbPort)); + }catch(NumberFormatException nfe) {} + } + jdbcConfig.setDatabaseUser(request.getParameter("databaseUser")); jdbcConfig.setDatabasePassword(request.getParameter("databasePassword")); jdbcConfig.setDatabaseName(request.getParameter("initialDatabase")); jdbcConfig.setDatabaseSchema(request.getParameter("initialSchema")); - + + if(logger.isDebugEnabled()) { + logger.debug("JDBC Configuration: {}", jdbcConfig); + } return jdbcConfig; } /** diff --git a/extensions/database/src/com/google/refine/extension/database/cmd/ExecuteQueryCommand.java b/extensions/database/src/com/google/refine/extension/database/cmd/ExecuteQueryCommand.java index 20e78585c..90a0dd387 100644 --- a/extensions/database/src/com/google/refine/extension/database/cmd/ExecuteQueryCommand.java +++ b/extensions/database/src/com/google/refine/extension/database/cmd/ExecuteQueryCommand.java @@ -55,12 +55,14 @@ public class ExecuteQueryCommand extends DatabaseCommand { @Override public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - logger.debug("QueryCommand::Post"); + DatabaseConfiguration databaseConfiguration = getJdbcConfiguration(request); String query = request.getParameter("queryString"); - logger.info("QueryCommand::Post::DatabaseConfiguration::{}::Query::{} " ,databaseConfiguration, query); - + if(logger.isDebugEnabled()) { + logger.debug("QueryCommand::Post::DatabaseConfiguration::{}::Query::{} " ,databaseConfiguration, query); + } + //ProjectManager.singleton.setBusy(true); try { @@ -76,10 +78,15 @@ public class ExecuteQueryCommand extends DatabaseCommand { response.setStatus(HttpStatus.SC_OK); String jsonStr = mapperObj.writeValueAsString(databaseInfo); - logger.debug("QueryCommand::Post::Result::{} " ,jsonStr); + + if(logger.isDebugEnabled()) { + logger.debug("QueryCommand::Post::Result::{} " ,jsonStr); + } + writer.object(); - writer.key("code"); writer.value("ok"); + writer.key("code"); + writer.value("ok"); writer.key("QueryResult"); writer.value(jsonStr); writer.endObject(); @@ -98,9 +105,10 @@ public class ExecuteQueryCommand extends DatabaseCommand { } catch (Exception e) { logger.error("QueryCommand::Post::Exception::{}", e); throw new ServletException(e); - } finally { - // ProjectManager.singleton.setBusy(false); - } + } +// finally { +// // ProjectManager.singleton.setBusy(false); +// } } diff --git a/extensions/database/src/com/google/refine/extension/database/cmd/SavedConnectionCommand.java b/extensions/database/src/com/google/refine/extension/database/cmd/SavedConnectionCommand.java index a1e68ae3e..12ba7b4e3 100644 --- a/extensions/database/src/com/google/refine/extension/database/cmd/SavedConnectionCommand.java +++ b/extensions/database/src/com/google/refine/extension/database/cmd/SavedConnectionCommand.java @@ -54,7 +54,10 @@ public class SavedConnectionCommand extends DatabaseCommand { @Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - logger.info("SavedConnectionCommand::Get::connectionName::{}", request.getParameter("connectionName")); + + if(logger.isDebugEnabled()) { + logger.debug("SavedConnectionCommand::Get::connectionName::{}", request.getParameter("connectionName")); + } String connectionName = request.getParameter("connectionName"); try { @@ -79,7 +82,10 @@ public class SavedConnectionCommand extends DatabaseCommand { public void doDelete(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - logger.info("SavedConnectionCommand::Delete Connection: {}", request.getParameter("connectionName")); + + if(logger.isDebugEnabled()) { + logger.debug("SavedConnectionCommand::Delete Connection: {}", request.getParameter("connectionName")); + } String connectionName = request.getParameter("connectionName"); @@ -225,7 +231,7 @@ public class SavedConnectionCommand extends DatabaseCommand { } writer.endArray(); writer.endObject(); - logger.info("Saved Connection Get Response sent"); + // logger.info("Saved Connection Get Response sent"); } finally { w.flush(); w.close(); @@ -238,8 +244,10 @@ public class SavedConnectionCommand extends DatabaseCommand { @Override public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - // TODO Auto-generated method stub - logger.info("SavedConnectionCommand::Post"); + + if(logger.isDebugEnabled()) { + logger.info("doPost Connection: {}", request.getParameter("connectionName")); + } DatabaseConfiguration jdbcConfig = getJdbcConfiguration(request); @@ -262,7 +270,7 @@ public class SavedConnectionCommand extends DatabaseCommand { if(jdbcConfig.getDatabasePassword() != null) { - logger.debug("SavedConnectionCommand::Post::password::{}", jdbcConfig.getDatabasePassword()); + //logger.debug("SavedConnectionCommand::Post::password::{}", jdbcConfig.getDatabasePassword()); jdbcConfig.setDatabasePassword(DatabaseUtils.encrypt(jdbcConfig.getDatabasePassword())); } @@ -285,7 +293,11 @@ public class SavedConnectionCommand extends DatabaseCommand { public void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - logger.info("SavedConnectionCommand::Put::databaseType " + request.getParameter("databaseType")); + + if(logger.isDebugEnabled()) { + logger.debug("databaseType::{} " , request.getParameter("databaseHost")); + } + logger.info("databaseHost::{} " , request.getParameter("databaseServer")); DatabaseConfiguration jdbcConfig = getJdbcConfiguration(request); StringBuilder sb = new StringBuilder(); @@ -312,8 +324,9 @@ public class SavedConnectionCommand extends DatabaseCommand { response.sendError(HttpStatus.SC_BAD_REQUEST, sb.toString()); } - - logger.debug("SavedConnectionCommand::PUT Connection: {}", jdbcConfig.getConnectionName()); + if(logger.isDebugEnabled()) { + logger.debug("Connection Config:: {}", jdbcConfig.getConnectionName()); + } if(jdbcConfig.getDatabasePassword() != null) { jdbcConfig.setDatabasePassword(DatabaseUtils.encrypt(jdbcConfig.getDatabasePassword())); diff --git a/extensions/database/src/com/google/refine/extension/database/cmd/TestConnectCommand.java b/extensions/database/src/com/google/refine/extension/database/cmd/TestConnectCommand.java index b545645bf..b4cd0dd0b 100644 --- a/extensions/database/src/com/google/refine/extension/database/cmd/TestConnectCommand.java +++ b/extensions/database/src/com/google/refine/extension/database/cmd/TestConnectCommand.java @@ -54,9 +54,12 @@ public class TestConnectCommand extends DatabaseCommand { public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - logger.debug("TestConnectCommand::Post"); + DatabaseConfiguration databaseConfiguration = getJdbcConfiguration(request); - logger.info("TestConnectCommand::Post::{}", databaseConfiguration); + if(logger.isDebugEnabled()) { + logger.debug("TestConnectCommand::Post::{}", databaseConfiguration); + } + //ProjectManager.singleton.setBusy(true); try { diff --git a/extensions/database/src/com/google/refine/extension/database/cmd/TestQueryCommand.java b/extensions/database/src/com/google/refine/extension/database/cmd/TestQueryCommand.java index 14d1e1040..a728fc1bd 100644 --- a/extensions/database/src/com/google/refine/extension/database/cmd/TestQueryCommand.java +++ b/extensions/database/src/com/google/refine/extension/database/cmd/TestQueryCommand.java @@ -55,11 +55,14 @@ public class TestQueryCommand extends DatabaseCommand { @Override public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - logger.debug("TestQueryCommand::Post"); - + DatabaseConfiguration dbConfig = getJdbcConfiguration(request); String query = request.getParameter("query"); - logger.info("TestQueryCommand::Post::DatabaseConfiguration::{}::Query::{} " ,dbConfig, query); + + if(logger.isDebugEnabled()) { + logger.debug("TestQueryCommand::Post::DatabaseConfiguration::{}::Query::{} " ,dbConfig, query); + } + //ProjectManager.singleton.setBusy(true); try { @@ -76,7 +79,9 @@ public class TestQueryCommand extends DatabaseCommand { response.setStatus(HttpStatus.SC_OK); String jsonStr = mapperObj.writeValueAsString(databaseInfo); - logger.debug("TestQueryCommand::Post::Result::{} " ,jsonStr); + if(logger.isDebugEnabled()) { + logger.debug("TestQueryCommand::Post::Result::{} " ,jsonStr); + } writer.object(); writer.key("code"); @@ -99,9 +104,10 @@ public class TestQueryCommand extends DatabaseCommand { } catch (Exception e) { logger.error("TestQueryCommand::Post::Exception::{}", e); throw new ServletException(e); - } finally { - // ProjectManager.singleton.setBusy(false); } +// finally { +// // ProjectManager.singleton.setBusy(false); +// } } diff --git a/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManager.java b/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManager.java index b73f3059f..0bd7a4713 100644 --- a/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManager.java +++ b/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManager.java @@ -123,12 +123,16 @@ public class MariaDBConnectionManager { public Connection getConnection(DatabaseConfiguration databaseConfiguration, boolean forceNewConnection) throws DatabaseServiceException{ try { - logger.info("connection::{}, forceNewConnection: {}", connection, forceNewConnection); + // logger.info("connection::{}, forceNewConnection: {}", connection, forceNewConnection); if (connection != null && !forceNewConnection) { logger.info("connection closed::{}", connection.isClosed()); if (!connection.isClosed()) { - logger.info("Returning existing connection::{}", connection); + if(logger.isDebugEnabled()) { + logger.debug("Returning existing connection::{}", connection); + } + + return connection; } } @@ -139,7 +143,10 @@ public class MariaDBConnectionManager { connection = DriverManager.getConnection(dbURL, databaseConfiguration.getDatabaseUser(), databaseConfiguration.getDatabasePassword()); - logger.info("*** Acquired New connection for ::{} **** ", dbURL); + if(logger.isDebugEnabled()) { + logger.debug("*** Acquired New connection for ::{} **** ", dbURL); + } + return connection; diff --git a/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseService.java b/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseService.java index dd7dabb1d..f5760a6b2 100644 --- a/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseService.java +++ b/extensions/database/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseService.java @@ -68,7 +68,9 @@ public class MariaDBDatabaseService extends DatabaseService { if (instance == null) { SQLType.registerSQLDriver(DB_NAME, DB_DRIVER); instance = new MariaDBDatabaseService(); - logger.debug("MariaDBDatabaseService Instance: {}", instance); + if(logger.isDebugEnabled()) { + logger.debug("MariaDBDatabaseService Instance: {}", instance); + } } return instance; } @@ -175,19 +177,33 @@ public class MariaDBDatabaseService extends DatabaseService { @Override public String buildLimitQuery(Integer limit, Integer offset, String query) { - StringBuilder sb = new StringBuilder(); - sb.append(query); - - if(limit != null) { - sb.append(" LIMIT" + " " + limit); - } - - if(offset != null) { - sb.append(" OFFSET" + " " + offset); - } - - return sb.toString(); - } +// if(logger.isDebugEnabled()) { +// logger.info( "<<< original input query::{} >>>" , query ); +// } +// + final int len = query.length(); + String parsedQuery = len > 0 && query.endsWith(";") ? query.substring(0, len - 1) : query; + + + StringBuilder sb = new StringBuilder(); + sb.append(parsedQuery); + + if(limit != null) { + sb.append(" LIMIT" + " " + limit); + } + + if(offset != null) { + sb.append(" OFFSET" + " " + offset); + } + sb.append(";"); + String parsedQueryOut = sb.toString(); + +// if(logger.isDebugEnabled()) { +// logger.info( "<<>>" , parsedQueryOut ); +// } + + return parsedQueryOut; + } @Override public ArrayList getColumns(DatabaseConfiguration dbConfig, String query) throws DatabaseServiceException{ diff --git a/extensions/database/src/com/google/refine/extension/database/model/DatabaseColumn.java b/extensions/database/src/com/google/refine/extension/database/model/DatabaseColumn.java index bf8972c60..35620146c 100644 --- a/extensions/database/src/com/google/refine/extension/database/model/DatabaseColumn.java +++ b/extensions/database/src/com/google/refine/extension/database/model/DatabaseColumn.java @@ -36,6 +36,7 @@ public class DatabaseColumn { private String name; private int size; private DatabaseColumnType type; + private String label; public DatabaseColumnType getType() { return type; @@ -60,10 +61,6 @@ public class DatabaseColumn { } - - - private String label; - public DatabaseColumn(String name, int size, DatabaseColumnType type) { super(); this.name = name; diff --git a/extensions/database/src/com/google/refine/extension/database/mysql/MySQLConnectionManager.java b/extensions/database/src/com/google/refine/extension/database/mysql/MySQLConnectionManager.java index ff14f6379..9e81fd264 100644 --- a/extensions/database/src/com/google/refine/extension/database/mysql/MySQLConnectionManager.java +++ b/extensions/database/src/com/google/refine/extension/database/mysql/MySQLConnectionManager.java @@ -59,8 +59,7 @@ public class MySQLConnectionManager { } - - + /** * Create a new instance of this connection manager. @@ -122,8 +121,6 @@ public class MySQLConnectionManager { public Connection getConnection(DatabaseConfiguration databaseConfiguration, boolean forceNewConnection) throws DatabaseServiceException{ try { - // logger.info("connection::{}, forceNewConnection: {}", connection, forceNewConnection); - if (connection != null && !forceNewConnection) { //logger.info("connection closed::{}", connection.isClosed()); if (!connection.isClosed()) { @@ -134,14 +131,20 @@ public class MySQLConnectionManager { return connection; } } - - Class.forName(type.getClassPath()); - DriverManager.setLoginTimeout(10); String dbURL = getDatabaseUrl(databaseConfiguration); + Class.forName(type.getClassPath()); + + //logger.info("*** type.getClassPath() ::{}, {}**** ", type.getClassPath()); + + DriverManager.setLoginTimeout(10); + connection = DriverManager.getConnection(dbURL, databaseConfiguration.getDatabaseUser(), databaseConfiguration.getDatabasePassword()); - logger.debug("*** Acquired New connection for ::{} **** ", dbURL); + if(logger.isDebugEnabled()) { + logger.debug("*** Acquired New connection for ::{} **** ", dbURL); + } + return connection; diff --git a/extensions/database/src/com/google/refine/extension/database/mysql/MySQLDatabaseService.java b/extensions/database/src/com/google/refine/extension/database/mysql/MySQLDatabaseService.java index d2c7982d6..503936e16 100644 --- a/extensions/database/src/com/google/refine/extension/database/mysql/MySQLDatabaseService.java +++ b/extensions/database/src/com/google/refine/extension/database/mysql/MySQLDatabaseService.java @@ -46,7 +46,7 @@ import com.google.refine.extension.database.SQLType; import com.google.refine.extension.database.model.DatabaseColumn; import com.google.refine.extension.database.model.DatabaseInfo; import com.google.refine.extension.database.model.DatabaseRow; -import com.mysql.jdbc.ResultSetMetaData; +//import com.mysql.jdbc.ResultSetMetaData; public class MySQLDatabaseService extends DatabaseService { @@ -65,7 +65,9 @@ public class MySQLDatabaseService extends DatabaseService { if (instance == null) { SQLType.registerSQLDriver(DB_NAME, DB_DRIVER, false); instance = new MySQLDatabaseService(); - logger.debug("MySQLDatabaseService Instance: {}", instance); + if(logger.isDebugEnabled()) { + logger.debug("MySQLDatabaseService Instance: {}", instance); + } } return instance; } @@ -88,7 +90,13 @@ public class MySQLDatabaseService extends DatabaseService { Connection connection = MySQLConnectionManager.getInstance().getConnection(dbConfig, false); Statement statement = connection.createStatement(); ResultSet queryResult = statement.executeQuery(query); - ResultSetMetaData metadata = (ResultSetMetaData)queryResult.getMetaData(); + java.sql.ResultSetMetaData metadata = queryResult.getMetaData(); + + if(metadata instanceof com.mysql.jdbc.ResultSetMetaData) { + metadata = (com.mysql.jdbc.ResultSetMetaData)metadata; + } + //ResultSetMetaData metadata = (ResultSetMetaData)queryResult.getMetaData(); + int columnCount = metadata.getColumnCount(); ArrayList columns = new ArrayList(columnCount); @@ -171,8 +179,16 @@ public class MySQLDatabaseService extends DatabaseService { @Override public String buildLimitQuery(Integer limit, Integer offset, String query) { + if(logger.isDebugEnabled()) { + logger.info( "<<< original input query::{} >>>" , query ); + } + + final int len = query.length(); + String parsedQuery = len > 0 && query.endsWith(";") ? query.substring(0, len - 1) : query; + + StringBuilder sb = new StringBuilder(); - sb.append(query); + sb.append(parsedQuery); if(limit != null) { sb.append(" LIMIT" + " " + limit); @@ -181,8 +197,14 @@ public class MySQLDatabaseService extends DatabaseService { if(offset != null) { sb.append(" OFFSET" + " " + offset); } + sb.append(";"); + String parsedQueryOut = sb.toString(); - return sb.toString(); + if(logger.isDebugEnabled()) { + logger.info( "<<>>" , parsedQueryOut ); + } + + return parsedQueryOut; } @Override @@ -193,7 +215,12 @@ public class MySQLDatabaseService extends DatabaseService { Statement statement = connection.createStatement(); ResultSet queryResult = statement.executeQuery(query); - ResultSetMetaData metadata = (ResultSetMetaData) queryResult.getMetaData(); + java.sql.ResultSetMetaData metadata = queryResult.getMetaData(); + if(metadata instanceof com.mysql.jdbc.ResultSetMetaData) { + metadata = (com.mysql.jdbc.ResultSetMetaData)metadata; + } + + //ResultSetMetaData metadata = (ResultSetMetaData) queryResult.getMetaData(); int columnCount = metadata.getColumnCount(); ArrayList columns = new ArrayList(columnCount); @@ -223,7 +250,13 @@ public class MySQLDatabaseService extends DatabaseService { Statement statement = connection.createStatement(); statement.setFetchSize(10); ResultSet queryResult = statement.executeQuery(query); - ResultSetMetaData metadata = (ResultSetMetaData)queryResult.getMetaData(); + + java.sql.ResultSetMetaData metadata = queryResult.getMetaData(); + if(metadata instanceof com.mysql.jdbc.ResultSetMetaData) { + metadata = (com.mysql.jdbc.ResultSetMetaData)metadata; + } + //logger.info("metadata class::" + metadata.getClass()); + int columnCount = metadata.getColumnCount(); int index = 0; diff --git a/extensions/database/src/com/google/refine/extension/database/pgsql/PgSQLDatabaseService.java b/extensions/database/src/com/google/refine/extension/database/pgsql/PgSQLDatabaseService.java index d0e11c51a..9a6308373 100644 --- a/extensions/database/src/com/google/refine/extension/database/pgsql/PgSQLDatabaseService.java +++ b/extensions/database/src/com/google/refine/extension/database/pgsql/PgSQLDatabaseService.java @@ -65,7 +65,9 @@ public class PgSQLDatabaseService extends DatabaseService { if (instance == null) { SQLType.registerSQLDriver(DB_NAME, DB_DRIVER); instance = new PgSQLDatabaseService(); - logger.debug("PgSQLDatabaseService Instance: {}", instance); + if(logger.isDebugEnabled()) { + logger.debug("PgSQLDatabaseService Instance: {}", instance); + } } return instance; } @@ -174,8 +176,16 @@ public class PgSQLDatabaseService extends DatabaseService { @Override public String buildLimitQuery(Integer limit, Integer offset, String query) { + if(logger.isDebugEnabled()) { + logger.debug( "<<< original input query::{} >>>" , query ); + } + + final int len = query.length(); + String parsedQuery = len > 0 && query.endsWith(";") ? query.substring(0, len - 1) : query; + + StringBuilder sb = new StringBuilder(); - sb.append(query); + sb.append(parsedQuery); if(limit != null) { sb.append(" LIMIT" + " " + limit); @@ -184,8 +194,14 @@ public class PgSQLDatabaseService extends DatabaseService { if(offset != null) { sb.append(" OFFSET" + " " + offset); } + sb.append(";"); + String parsedQueryOut = sb.toString(); - return sb.toString(); + if(logger.isDebugEnabled()) { + logger.debug( "<<>>" , parsedQueryOut ); + } + + return parsedQueryOut; } @Override @@ -247,7 +263,7 @@ public class PgSQLDatabaseService extends DatabaseService { return rows; } catch (SQLException e) { - logger.error("SQLException::", e); + logger.error("SQLException::{}::{}", e); throw new DatabaseServiceException(true, e.getSQLState(), e.getErrorCode(), e.getMessage()); } } diff --git a/extensions/database/test/com/google/refine/extension/database/DBExtensionTestUtils.java b/extensions/database/test/com/google/refine/extension/database/DBExtensionTestUtils.java index 68945b595..bc06afed6 100644 --- a/extensions/database/test/com/google/refine/extension/database/DBExtensionTestUtils.java +++ b/extensions/database/test/com/google/refine/extension/database/DBExtensionTestUtils.java @@ -1,30 +1,41 @@ package com.google.refine.extension.database; +import java.io.File; +import java.io.IOException; import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.Date; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; import java.util.Map; import java.util.Random; +import org.apache.commons.lang.text.StrSubstitutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class DBExtensionTestUtils { - private static final String DB_TYPE_MYSQL = "mysql"; - private static final String DB_HOST_MYSQL = "127.0.0.1"; - private static final int DB_PORT_MYSQL = 3306; - private static final String DB_USER_MYSQL = "root"; - private static final String DB_PASSWORD_MYSQL = "secret"; - private static final String DB_NAME_MYSQL = "rxhub"; + private static final Logger logger = LoggerFactory.getLogger("DBExtensionTestUtils"); - private static final String DB_TYPE_PGSQL = "postgresql"; - private static final String DB_HOST_PGSQL = "127.0.0.1"; - private static final int DB_PORT_PGSQL = 5432; - private static final String DB_USER_PGSQL = "postgres"; - private static final String DB_PASSWORD_PGSQL = ""; - private static final String DB_NAME_PGSQL = "openrefine"; + private static final String MYSQL_DB_NAME = "mysql"; + private static final String DEFAULT_MYSQL_HOST = "127.0.0.1"; + private static final int DEFAULT_MYSQL_PORT = 3306; + private static final String DEFAULT_MYSQL_USER = "root"; + private static final String DEFAULT_MYSQL_PASSWORD = "secret"; + private static final String DEFAULT_MYSQL_DB_NAME = "testdb"; + private static final String PGSQL_DB_NAME = "postgresql"; + private static final String DEFAULT_PGSQL_HOST = "127.0.0.1"; + private static final int DEFAULT_PGSQL_PORT = 5432; + private static final String DEFAULT_PGSQL_USER = "postgres"; + private static final String DEFAULT_PGSQL_PASSWORD = ""; + private static final String DEFAULT_PGSQL_DB_NAME = "openrefine"; + + private static final String DEFAULT_TEST_TABLE = "test_data"; private static final int SAMPLE_SIZE = 500000; private static final int BATCH_SIZE = 1000; @@ -34,8 +45,142 @@ public class DBExtensionTestUtils { private Map mncMap; private Map mccMap; + + /** + * Create Test Table with one row of Data + * @param dbConfig DatabaseConfiguration to test + * @param tableName + */ + public static void initTestData(DatabaseConfiguration dbConfig, String tableName) { + + Statement stmt = null; + Connection conn = null; + try { + DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType()); + conn = dbService.getConnection(dbConfig); + stmt = conn.createStatement(); + + DatabaseMetaData dbm = conn.getMetaData(); + // check if "employee" table is there + ResultSet tables = dbm.getTables(null, null, tableName, null); + if (tables.next()) { + stmt.executeUpdate("DROP TABLE " + tableName); + //System.out.println("Drop Table Result::" + dropResult); + } + + String createSQL = " CREATE TABLE " + tableName + " ( " + + " ID INT NOT NULL, " + + " NAME VARCHAR (20) NOT NULL, " + + " CITY VARCHAR (20) NOT NULL," + + " PRIMARY KEY (ID) );"; + + + stmt.executeUpdate(createSQL); + //System.out.println("Create Table Result::" + createResult); + + String insertTableSQL = "INSERT INTO " + tableName + + "(ID, NAME, CITY) " + "VALUES" + + "(1,'frank lens','Dallas')"; + + stmt.executeUpdate(insertTableSQL); + // System.out.println("Insert Data Result::" + insertResult); + + logger.info("Database Test Init Data Created!!!"); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + + } finally { + if(stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + if(conn != null) { + try { + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + } - public void generateMySQLTestData() { + + public static void initTestData(DatabaseConfiguration dbConfig) { + + Statement stmt = null; + Connection conn = null; + try { + DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType()); + conn = dbService.getConnection(dbConfig); + stmt = conn.createStatement(); + + DatabaseMetaData dbm = conn.getMetaData(); + // check if "employee" table is there + ResultSet tables = dbm.getTables(null, null, DEFAULT_TEST_TABLE, null); + if (tables.next()) { + stmt.executeUpdate("DROP TABLE " + DEFAULT_TEST_TABLE); + //System.out.println("Drop Table Result::" + dropResult); + } + + String createSQL = " CREATE TABLE TEST_DATA( " + + " ID INT NOT NULL, " + + " NAME VARCHAR (20) NOT NULL, " + + " CITY VARCHAR (20) NOT NULL," + + " PRIMARY KEY (ID) );"; + + + stmt.executeUpdate(createSQL); + //System.out.println("Create Table Result::" + createResult); + + String insertTableSQL = "INSERT INTO TEST_DATA" + + "(ID, NAME, CITY) " + "VALUES" + + "(1,'frank lens','Dallas')"; + + stmt.executeUpdate(insertTableSQL); + // System.out.println("Insert Data Result::" + insertResult); + + logger.info("Database Test Init Data Created!!!"); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + + } finally { + if(stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + if(conn != null) { + try { + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + } + + /** + * CREATE test data in MySQL + * Table name: test_data + * @param sampleSize + * @param batchSize + */ + public void generateMySQLTestData(int sampleSize, int batchSize) { mncMap = new HashMap(); mccMap = new HashMap(); mccMap.put(0, 302); @@ -49,12 +194,12 @@ public class DBExtensionTestUtils { mncMap.put(3, 710); DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost(DB_HOST_MYSQL); - dc.setDatabaseName(DB_NAME_MYSQL); - dc.setDatabasePassword(DB_PASSWORD_MYSQL); - dc.setDatabasePort(DB_PORT_MYSQL); - dc.setDatabaseType(DB_TYPE_MYSQL); - dc.setDatabaseUser(DB_USER_MYSQL); + dc.setDatabaseHost(DEFAULT_MYSQL_HOST); + dc.setDatabaseName(DEFAULT_MYSQL_DB_NAME); + dc.setDatabasePassword(DEFAULT_MYSQL_PASSWORD); + dc.setDatabasePort(DEFAULT_MYSQL_PORT); + dc.setDatabaseType(MYSQL_DB_NAME); + dc.setDatabaseUser(DEFAULT_MYSQL_USER); dc.setUseSSL(false); String truncateTableSQL = "TRUNCATE test_data"; @@ -65,7 +210,7 @@ public class DBExtensionTestUtils { Connection conn; try { - conn = DatabaseService.get(DB_TYPE_MYSQL).getConnection(dc); + conn = DatabaseService.get(MYSQL_DB_NAME).getConnection(dc); Statement truncateStmt = conn.createStatement(); int result = truncateStmt.executeUpdate(truncateTableSQL); @@ -78,23 +223,23 @@ public class DBExtensionTestUtils { PreparedStatement stmt = conn.prepareStatement(insertTableSQL); int counter=1; - for (int i = 0; i < SAMPLE_SIZE; i++) { + for (int i = 0; i < sampleSize; i++) { stmt.setLong(1, i); - stmt.setString(2, getNextUeId(i)); - stmt.setDate(3, getNextStartDate(i)); - stmt.setDate(4, getNextEndDate(i)); + stmt.setString(2, getNextUeId()); + stmt.setDate(3, getNextStartDate()); + stmt.setDate(4, getNextEndDate()); stmt.setInt(5, rand.nextInt()); stmt.setInt(6, rand.nextInt()); stmt.setInt(7, rand.nextInt(10)); - stmt.setInt(8, getMCC(i)); - stmt.setInt(9, getMNC(i)); + stmt.setInt(8, getMCC()); + stmt.setInt(9, getMNC()); stmt.setInt(10, rand.nextInt(100)); - stmt.setString(11, getNextIMEI(i)); + stmt.setString(11, getNextIMEI()); stmt.addBatch(); //Execute batch of 1000 records - if(i%BATCH_SIZE==0){ + if(i%batchSize==0){ stmt.executeBatch(); conn.commit(); System.out.println("Batch "+(counter++)+" executed successfully"); @@ -116,8 +261,12 @@ public class DBExtensionTestUtils { } - - public void generatePgSQLTestData() { + /** + * + * @param sampleSize + * @param batchSize + */ + public void generatePgSQLTestData(int sampleSize, int batchSize) { mncMap = new HashMap(); mccMap = new HashMap(); mccMap.put(0, 302); @@ -131,12 +280,12 @@ public class DBExtensionTestUtils { mncMap.put(3, 710); DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost(DB_HOST_PGSQL); - dc.setDatabaseName(DB_NAME_PGSQL); - dc.setDatabasePassword(DB_PASSWORD_PGSQL); - dc.setDatabasePort(DB_PORT_PGSQL); - dc.setDatabaseType(DB_TYPE_PGSQL); - dc.setDatabaseUser(DB_USER_PGSQL); + dc.setDatabaseHost(DEFAULT_PGSQL_HOST); + dc.setDatabaseName(DEFAULT_PGSQL_DB_NAME); + dc.setDatabasePassword(DEFAULT_PGSQL_PASSWORD); + dc.setDatabasePort(DEFAULT_PGSQL_PORT); + dc.setDatabaseType(PGSQL_DB_NAME); + dc.setDatabaseUser(DEFAULT_PGSQL_USER); dc.setUseSSL(false); String truncateTableSQL = "TRUNCATE public.test_data"; @@ -147,7 +296,7 @@ public class DBExtensionTestUtils { Connection conn; try { - conn = DatabaseService.get(DB_TYPE_PGSQL).getConnection(dc); + conn = DatabaseService.get(PGSQL_DB_NAME).getConnection(dc); Statement truncateStmt = conn.createStatement(); int result = truncateStmt.executeUpdate(truncateTableSQL); @@ -160,23 +309,23 @@ public class DBExtensionTestUtils { PreparedStatement stmt = conn.prepareStatement(insertTableSQL); int counter=1; - for (int i = 0; i < SAMPLE_SIZE; i++) { + for (int i = 0; i < sampleSize; i++) { stmt.setLong(1, i); - stmt.setString(2, getNextUeId(i)); - stmt.setDate(3, getNextStartDate(i)); - stmt.setDate(4, getNextEndDate(i)); + stmt.setString(2, getNextUeId()); + stmt.setDate(3, getNextStartDate()); + stmt.setDate(4, getNextEndDate()); stmt.setInt(5, rand.nextInt()); stmt.setInt(6, rand.nextInt()); stmt.setInt(7, rand.nextInt(10)); - stmt.setInt(8, getMCC(i)); - stmt.setInt(9, getMNC(i)); + stmt.setInt(8, getMCC()); + stmt.setInt(9, getMNC()); stmt.setInt(10, rand.nextInt(100)); - stmt.setString(11, getNextIMEI(i)); + stmt.setString(11, getNextIMEI()); stmt.addBatch(); //Execute batch of 1000 records - if(i%BATCH_SIZE==0){ + if(i%batchSize==0){ stmt.executeBatch(); conn.commit(); System.out.println("Batch "+(counter++)+" executed successfully"); @@ -198,44 +347,35 @@ public class DBExtensionTestUtils { } - private String getNextIMEI(int index) { - index++; -// byte[] array = new byte[16]; // length is bounded by 7 -// new Random().nextBytes(array); -// String generatedString = new String(array, Charset.forName("UTF-8")); - - int n = 1000000000 + rand.nextInt(900000000); + private String getNextIMEI() { + int n = 1000000000 + rand.nextInt(900000000); return "" + n; } - private int getMNC(int index) { - index++; - // TODO Auto-generated method stub + private int getMNC() { + return mncMap.get(rand.nextInt(3)); } - private int getMCC(int index) { - index++; - // System.out.println(rand.nextInt(3)); + private int getMCC() { + return mccMap.get(rand.nextInt(3)); } - private Date getNextEndDate(int index) { - // TODO Auto-generated method stub - index++; + private Date getNextEndDate() { + return new Date(System.currentTimeMillis() + 1); } - private Date getNextStartDate(int index) { - // TODO Auto-generated method stub - index++; + private Date getNextStartDate() { + return new Date(System.currentTimeMillis()); } - private String getNextUeId(int index) { - index++; + private String getNextUeId() { + int n = 300000000 + rand.nextInt(900000000); return "" + n; @@ -243,8 +383,74 @@ public class DBExtensionTestUtils { public static void main(String[] args) { DBExtensionTestUtils testUtil = new DBExtensionTestUtils(); - // testUtil.generatePgSQLTestData(); - testUtil.generateMySQLTestData(); + testUtil.generatePgSQLTestData(SAMPLE_SIZE, BATCH_SIZE); + // testUtil.generateMySQLTestData(); + } + + + public static void cleanUpTestData(DatabaseConfiguration dbConfig) { + Statement stmt = null; + Connection conn = null; + try { + DatabaseService dbService = DatabaseService.get(dbConfig.getDatabaseType()); + conn = dbService.getConnection(dbConfig); + stmt = conn.createStatement(); + + DatabaseMetaData dbm = conn.getMetaData(); + // check if "employee" table is there + ResultSet tables = dbm.getTables(null, null, DEFAULT_TEST_TABLE, null); + if (tables.next()) { + stmt.executeUpdate("DROP TABLE " + DEFAULT_TEST_TABLE); + //System.out.println("Drop Table Result::" + dropResult); + } + + logger.info("Database Test Cleanup Done!!!"); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } finally { + if(stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + if(conn != null) { + try { + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + } + + public static File createTempDirectory(String name) + throws IOException { + File dir = File.createTempFile(name, ""); + dir.delete(); + dir.mkdir(); + return dir; + } + + public static String getJDBCUrl(DatabaseConfiguration dbConfig) { + Map substitutes = new HashMap(); + substitutes.put("dbType", dbConfig.getDatabaseType()); + substitutes.put("host", dbConfig.getDatabaseHost()); + substitutes.put("port", "" + dbConfig.getDatabasePort()); + substitutes.put("dbName", dbConfig.getDatabaseName()); + substitutes.put("useSSL", dbConfig.isUseSSL()); + String urlTemplate = "jdbc:${dbType}://${host}:${port}/${dbName}?useSSL=${useSSL}"; + StrSubstitutor strSub = new StrSubstitutor(substitutes); + return strSub.replace(urlTemplate); } } diff --git a/extensions/database/test/com/google/refine/extension/database/DBQueryResultImportReaderTest.java b/extensions/database/test/com/google/refine/extension/database/DBQueryResultImportReaderTest.java deleted file mode 100644 index ad1872f0c..000000000 --- a/extensions/database/test/com/google/refine/extension/database/DBQueryResultImportReaderTest.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.google.refine.extension.database; - -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; - -public class DBQueryResultImportReaderTest { - - @BeforeTest - public void beforeTest() { - //add test logic - } - - @Test - public void testGetNextRowOfCells() { - //add test logic - } - -} diff --git a/extensions/database/test/com/google/refine/extension/database/DBQueryResultPreviewReaderTest.java b/extensions/database/test/com/google/refine/extension/database/DBQueryResultPreviewReaderTest.java deleted file mode 100644 index 2fb4880b7..000000000 --- a/extensions/database/test/com/google/refine/extension/database/DBQueryResultPreviewReaderTest.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.google.refine.extension.database; - -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; - -public class DBQueryResultPreviewReaderTest { - - @BeforeTest - public void beforeTest() { - //add test logic - } - - @Test - public void testGetNextRowOfCells() { - //add test logic - } - -} diff --git a/extensions/database/test/com/google/refine/extension/database/DatabaseImportControllerTest.java b/extensions/database/test/com/google/refine/extension/database/DatabaseImportControllerTest.java index 988e82846..52188b611 100644 --- a/extensions/database/test/com/google/refine/extension/database/DatabaseImportControllerTest.java +++ b/extensions/database/test/com/google/refine/extension/database/DatabaseImportControllerTest.java @@ -1,28 +1,282 @@ package com.google.refine.extension.database; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONException; +import org.json.JSONObject; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; +import com.google.refine.ProjectManager; +import com.google.refine.ProjectMetadata; +import com.google.refine.RefineServlet; +import com.google.refine.extension.database.mysql.MySQLDatabaseService; +import com.google.refine.extension.database.stub.RefineDbServletStub; +import com.google.refine.importing.ImportingJob; +import com.google.refine.importing.ImportingManager; +import com.google.refine.io.FileProjectManager; +import com.google.refine.model.Project; -public class DatabaseImportControllerTest { + + + +public class DatabaseImportControllerTest extends DBExtensionTests{ + + @Mock + private HttpServletRequest request; + + @Mock + private HttpServletResponse response; - @BeforeTest - public void beforeTest() { - //add test logic - } + private Project project; + private ProjectMetadata metadata; + private ImportingJob job; + private RefineServlet servlet; + private String JSON_OPTION = "{\"mode\":\"row-based\"}}"; + + private DatabaseConfiguration testDbConfig; + private String query; + + //System under test + private DatabaseImportController SUT = null; + + @BeforeMethod + public void setUp() throws JSONException, IOException { + MockitoAnnotations.initMocks(this); + + File dir = DBExtensionTestUtils.createTempDirectory("OR_DBExtension_Test_WorkspaceDir"); + FileProjectManager.initialize(dir); + + servlet = new RefineDbServletStub(); + ImportingManager.initialize(servlet); + project = new Project(); + metadata = new ProjectMetadata(); + job = ImportingManager.createJob(); + + metadata.setName("Database Import Test Project"); + ProjectManager.singleton.registerProject(project, metadata); + SUT = new DatabaseImportController(); + + } + + @AfterMethod + public void tearDown() { + SUT = null; + request = null; + response = null; + project = null; + metadata = null; + ImportingManager.disposeJob(job.id); + job = null; + //options = null; + } + + @Test public void testDoGet() { - //add test logic + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + try { + when(response.getWriter()).thenReturn(pw); + + SUT.doGet(request, response); + + String result = sw.getBuffer().toString().trim(); + JSONObject json = new JSONObject(result); + + String code = json.getString("status"); + String message = json.getString("message"); + Assert.assertNotNull(code); + Assert.assertNotNull(message); + Assert.assertEquals(code, "error"); + Assert.assertEquals(message, "GET not implemented"); + + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } + + @Test + public void testDoPostInvalidSubCommand() { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + try { + when(request.getQueryString()).thenReturn( + "http://127.0.0.1:3333/command/core/importing-controller?controller=database/database-import-controller&subCommand=invalid-sub-command"); + + when(response.getWriter()).thenReturn(pw); + + SUT.doPost(request, response); + + String result = sw.getBuffer().toString().trim(); + JSONObject json = new JSONObject(result); + + String code = json.getString("status"); + String message = json.getString("message"); + Assert.assertNotNull(code); + Assert.assertNotNull(message); + Assert.assertEquals(code, "error"); + Assert.assertEquals(message, "No such sub command"); + + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + @Test - public void testDoPost() { - //add test logic + public void testDoPostInitializeParser() { + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + try { + when(request.getQueryString()).thenReturn( + "http://127.0.0.1:3333/command/core/importing-controller?controller=database/database-import-controller&subCommand=initialize-parser-ui"); + when(response.getWriter()).thenReturn(pw); + + SUT.doPost(request, response); + + String result = sw.getBuffer().toString().trim(); + JSONObject json = new JSONObject(result); + + String status = json.getString("status"); + //System.out.println("json::" + json); + Assert.assertEquals(status, "ok"); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } + + @Test + public void testDoPostParsePreview() { + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); - + try { + + long jobId = job.id; + + when(request.getQueryString()).thenReturn( + "http://127.0.0.1:3333/command/core/importing-controller?controller=database%2Fdatabase-import-controller&jobID=" + + jobId + "&subCommand=parse-preview"); + when(response.getWriter()).thenReturn(pw); + + + when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType()); + when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); + when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); + when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); + when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); + when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); + when(request.getParameter("query")).thenReturn(query); + when(request.getParameter("options")).thenReturn(JSON_OPTION); + + SUT.doPost(request, response); + + String result = sw.getBuffer().toString().trim(); + JSONObject json = new JSONObject(result); + + String status = json.getString("status"); + //System.out.println("json::" + json); + Assert.assertEquals(status, "ok"); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + @Test + public void testDoPostCreateProject() { + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + try { + + long jobId = job.id; + + when(request.getQueryString()).thenReturn( + "http://127.0.0.1:3333/command/core/importing-controller?controller=database%2Fdatabase-import-controller&jobID=" + + jobId + "&subCommand=create-project"); + when(response.getWriter()).thenReturn(pw); + + + when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType()); + when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); + when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); + when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); + when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); + when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); + when(request.getParameter("query")).thenReturn(query); + when(request.getParameter("options")).thenReturn(JSON_OPTION); + + + SUT.doPost(request, response); + + String result = sw.getBuffer().toString().trim(); + JSONObject json = new JSONObject(result); + + String status = json.getString("status"); + //System.out.println("json::" + json); + Assert.assertEquals(status, "ok"); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + @BeforeTest + @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) + public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, + @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, + @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { + + MockitoAnnotations.initMocks(this); + // System.out.println("beforeTest " + pgSqlDbName); + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(mySqlDbHost); + testDbConfig.setDatabaseName(mySqlDbName); + testDbConfig.setDatabasePassword(mySqlDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); + testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(mySqlDbUser); + testDbConfig.setUseSSL(false); + + //testTable = mySqlTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + + DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); + + } + } diff --git a/extensions/database/test/com/google/refine/extension/database/DatabaseServiceTest.java b/extensions/database/test/com/google/refine/extension/database/DatabaseServiceTest.java index 7d2ff0b01..80f873cba 100644 --- a/extensions/database/test/com/google/refine/extension/database/DatabaseServiceTest.java +++ b/extensions/database/test/com/google/refine/extension/database/DatabaseServiceTest.java @@ -1,58 +1,206 @@ package com.google.refine.extension.database; +import java.sql.Connection; +import java.util.List; + +import org.testng.Assert; +import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; +import com.google.refine.extension.database.mariadb.MariaDBDatabaseService; +import com.google.refine.extension.database.model.DatabaseColumn; +import com.google.refine.extension.database.model.DatabaseInfo; +import com.google.refine.extension.database.model.DatabaseRow; +import com.google.refine.extension.database.mysql.MySQLDatabaseService; +import com.google.refine.extension.database.pgsql.PgSQLDatabaseService; -public class DatabaseServiceTest { + + +public class DatabaseServiceTest extends DBExtensionTests{ + + private DatabaseConfiguration testDbConfig; + private String testTable; + + @BeforeTest - public void beforeTest() { + @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) + public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, + @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, + @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { + + + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(mySqlDbHost); + testDbConfig.setDatabaseName(mySqlDbName); + testDbConfig.setDatabasePassword(mySqlDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); + testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(mySqlDbUser); + testDbConfig.setUseSSL(false); + + testTable = mySqlTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + + DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); + DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); + DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance()); + } - @Test - public void testGetDatabaseUrl() { + @AfterSuite + public void afterSuite() { + DBExtensionTestUtils.cleanUpTestData(testDbConfig); } @Test - public void testGet() { + public void testGetDatabaseUrl() { + DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); + String dbUrl = dbService.getDatabaseUrl(testDbConfig); + System.out.println("dbUrl:" + dbUrl); + Assert.assertNotNull(dbUrl); + Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); + } + + + @Test + public void testGetPgSQLDBService() { + + DatabaseService dbService = DatabaseService.get(PgSQLDatabaseService.DB_NAME); + Assert.assertNotNull(dbService); + Assert.assertEquals(dbService.getClass(), PgSQLDatabaseService.class); + } + + @Test + public void testGetMySQLDBService() { + + DatabaseService dbService = DatabaseService.get(MySQLDatabaseService.DB_NAME); + Assert.assertNotNull(dbService); + Assert.assertEquals(dbService.getClass(), MySQLDatabaseService.class); + } + + @Test + public void testGetMariaDBSQLDBService() { + + DatabaseService dbService = DatabaseService.get(MariaDBDatabaseService.DB_NAME); + Assert.assertNotNull(dbService); + Assert.assertEquals(dbService.getClass(), MariaDBDatabaseService.class); } @Test public void testGetConnection() { - + + try { + + DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); + Connection conn = dbService.getConnection(testDbConfig); + //System.out.println("conn:" + conn); + Assert.assertNotNull(conn); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } @Test public void testTestConnection() { + try { + DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); + boolean result = dbService.testConnection(testDbConfig); + Assert.assertEquals(result, true); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } @Test public void testConnect() { + + try { + DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); + DatabaseInfo databaseInfo = dbService.connect(testDbConfig); + Assert.assertNotNull(databaseInfo); + + } catch (DatabaseServiceException e) { + e.printStackTrace(); + } } @Test public void testExecuteQuery() { + try { + DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); + DatabaseInfo databaseInfo = dbService.testQuery(testDbConfig, + "SELECT * FROM " + testTable); + + Assert.assertNotNull(databaseInfo); + + } catch (DatabaseServiceException e) { + e.printStackTrace(); + } + } @Test public void testBuildLimitQuery() { - + DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); + String limitQuery = dbService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable); + + Assert.assertNotNull(limitQuery); + + Assert.assertEquals(limitQuery, "SELECT * FROM " + testTable + " LIMIT " + 100 + " OFFSET " + 0 + ";"); } @Test public void testGetColumns() { + List dbColumns; + + try { + DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); + dbColumns = dbService.getColumns(testDbConfig,"SELECT * FROM " + testTable); + Assert.assertNotNull(dbColumns); + + int cols = dbColumns.size(); + Assert.assertEquals(cols, 3); + + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } @Test public void testGetRows() { + + try { + + DatabaseService dbService = DatabaseService.get(testDbConfig.getDatabaseType()); + List dbRows = dbService.getRows(testDbConfig, + "SELECT * FROM " + testTable); + + Assert.assertNotNull(dbRows); + Assert.assertEquals(dbRows.size(), 1); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } diff --git a/extensions/database/test/com/google/refine/extension/database/SimpleTextEncryptorTest.java b/extensions/database/test/com/google/refine/extension/database/SimpleTextEncryptorTest.java new file mode 100644 index 000000000..1ef21450f --- /dev/null +++ b/extensions/database/test/com/google/refine/extension/database/SimpleTextEncryptorTest.java @@ -0,0 +1,28 @@ +package com.google.refine.extension.database; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class SimpleTextEncryptorTest { + + @Test + public void encrypt() { + SimpleTextEncryptor textEncryptor = new SimpleTextEncryptor("WEWssa!@d445d"); + String password = "testpass"; + String encPass = textEncryptor.encrypt(password); + Assert.assertNotNull(encPass); + Assert.assertNotEquals(encPass, password); + + } + + @Test + public void decrypt() { + SimpleTextEncryptor textEncryptor = new SimpleTextEncryptor("OOEWssa!@d445d"); + String password = "testpass"; + String encPass = textEncryptor.encrypt(password); + Assert.assertNotNull(encPass); + Assert.assertNotEquals(encPass, password); + String decPass = textEncryptor.decrypt(encPass); + Assert.assertEquals(decPass, password); + } +} diff --git a/extensions/database/test/com/google/refine/extension/database/cmd/ConnectCommandTest.java b/extensions/database/test/com/google/refine/extension/database/cmd/ConnectCommandTest.java index 7b5501764..06c7e8ddd 100644 --- a/extensions/database/test/com/google/refine/extension/database/cmd/ConnectCommandTest.java +++ b/extensions/database/test/com/google/refine/extension/database/cmd/ConnectCommandTest.java @@ -1,54 +1,86 @@ package com.google.refine.extension.database.cmd; -import java.io.IOException; +import static org.mockito.Mockito.when; + import java.io.PrintWriter; import java.io.StringWriter; -import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.json.JSONException; import org.json.JSONObject; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.testng.Assert; +import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; -import com.google.refine.extension.database.pgsql.PgSQLDatabaseService; - -import static org.mockito.Mockito.when; +import com.google.refine.extension.database.DBExtensionTestUtils; +import com.google.refine.extension.database.DBExtensionTests; +import com.google.refine.extension.database.DatabaseConfiguration; +import com.google.refine.extension.database.DatabaseService; +import com.google.refine.extension.database.mysql.MySQLDatabaseService; -public class ConnectCommandTest { +public class ConnectCommandTest extends DBExtensionTests { @Mock - HttpServletRequest request; + private HttpServletRequest request; @Mock - HttpServletResponse response; + private HttpServletResponse response; + private DatabaseConfiguration testDbConfig; + // private String testTable; + + @BeforeTest - public void beforeTest() { + @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) + public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, + @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, + @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { + MockitoAnnotations.initMocks(this); + // System.out.println("beforeTest " + pgSqlDbName); + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(mySqlDbHost); + testDbConfig.setDatabaseName(mySqlDbName); + testDbConfig.setDatabasePassword(mySqlDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); + testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(mySqlDbUser); + testDbConfig.setUseSSL(false); + + //testTable = mySqlTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + + DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); + } - + + @AfterSuite + public void afterSuite() { + DBExtensionTestUtils.cleanUpTestData(testDbConfig); + + } + + @Test public void testDoPost() { - when(request.getParameter("databaseType")).thenReturn(PgSQLDatabaseService.DB_NAME); - when(request.getParameter("databaseServer")).thenReturn("127.0.0.1"); - when(request.getParameter("databasePort")).thenReturn("5432"); - when(request.getParameter("databaseUser")).thenReturn("postgres"); - when(request.getParameter("databasePassword")).thenReturn(""); - when(request.getParameter("initialDatabase")).thenReturn("postgres"); - - + when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); + when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); + when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); + when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); + when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); + when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); + StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); try { @@ -62,15 +94,11 @@ public class ConnectCommandTest { String code = json.getString("code"); Assert.assertEquals(code, "ok"); - + + String databaseInfo = json.getString("databaseInfo"); + Assert.assertNotNull(databaseInfo); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (ServletException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (JSONException e) { + } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } diff --git a/extensions/database/test/com/google/refine/extension/database/cmd/ExecuteQueryCommandTest.java b/extensions/database/test/com/google/refine/extension/database/cmd/ExecuteQueryCommandTest.java new file mode 100644 index 000000000..29e4ffb96 --- /dev/null +++ b/extensions/database/test/com/google/refine/extension/database/cmd/ExecuteQueryCommandTest.java @@ -0,0 +1,109 @@ +package com.google.refine.extension.database.cmd; + +import static org.mockito.Mockito.when; + +import java.io.PrintWriter; +import java.io.StringWriter; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONObject; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.Assert; +import org.testng.annotations.AfterSuite; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; +import org.testng.annotations.Test; + +import com.google.refine.extension.database.DBExtensionTestUtils; +import com.google.refine.extension.database.DBExtensionTests; +import com.google.refine.extension.database.DatabaseConfiguration; +import com.google.refine.extension.database.DatabaseService; +import com.google.refine.extension.database.mysql.MySQLDatabaseService; + + +public class ExecuteQueryCommandTest extends DBExtensionTests { + + @Mock + private HttpServletRequest request; + + @Mock + private HttpServletResponse response; + + + private DatabaseConfiguration testDbConfig; + private String testTable; + + + @BeforeTest + @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) + public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, + @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, + @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { + + MockitoAnnotations.initMocks(this); + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(mySqlDbHost); + testDbConfig.setDatabaseName(mySqlDbName); + testDbConfig.setDatabasePassword(mySqlDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); + testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(mySqlDbUser); + testDbConfig.setUseSSL(false); + + testTable = mySqlTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + + DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); + + } + + @AfterSuite + public void afterSuite() { + DBExtensionTestUtils.cleanUpTestData(testDbConfig); + + } + + + @Test + public void testDoPost() { + + when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType()); + when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); + when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); + when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); + when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); + when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); + when(request.getParameter("queryString")).thenReturn("SELECT count(*) FROM " + testTable); + + + StringWriter sw = new StringWriter(); + + PrintWriter pw = new PrintWriter(sw); + + try { + when(response.getWriter()).thenReturn(pw); + ExecuteQueryCommand executeQueryCommand = new ExecuteQueryCommand(); + + executeQueryCommand.doPost(request, response); + + String result = sw.getBuffer().toString().trim(); + JSONObject json = new JSONObject(result); + + String code = json.getString("code"); + Assert.assertEquals(code, "ok"); + + String queryResult = json.getString("QueryResult"); + Assert.assertNotNull(queryResult); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + +} diff --git a/extensions/database/test/com/google/refine/extension/database/cmd/QueryCommandTest.java b/extensions/database/test/com/google/refine/extension/database/cmd/QueryCommandTest.java deleted file mode 100644 index 62f677e1a..000000000 --- a/extensions/database/test/com/google/refine/extension/database/cmd/QueryCommandTest.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.google.refine.extension.database.cmd; - -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.json.JSONException; -import org.json.JSONObject; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.testng.Assert; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; - -import com.google.refine.extension.database.pgsql.PgSQLDatabaseService; - - -public class QueryCommandTest { - - @Mock - HttpServletRequest request; - - @Mock - HttpServletResponse response; - - @BeforeTest - public void beforeTest() { - MockitoAnnotations.initMocks(this); - } - - @Test - public void testDoPost() { - - when(request.getParameter("databaseType")).thenReturn(PgSQLDatabaseService.DB_NAME); - when(request.getParameter("databaseServer")).thenReturn("127.0.0.1"); - when(request.getParameter("databasePort")).thenReturn("5432"); - when(request.getParameter("databaseUser")).thenReturn("postgres"); - when(request.getParameter("databasePassword")).thenReturn(""); - when(request.getParameter("initialDatabase")).thenReturn("openrefine"); - when(request.getParameter("queryString")).thenReturn("SELECT count(*) FROM call_data"); - - - StringWriter sw = new StringWriter(); - - PrintWriter pw = new PrintWriter(sw); - - try { - when(response.getWriter()).thenReturn(pw); - ExecuteQueryCommand connectCommand = new ExecuteQueryCommand(); - - connectCommand.doPost(request, response); - - String result = sw.getBuffer().toString().trim(); - JSONObject json = new JSONObject(result); - - String code = json.getString("code"); - Assert.assertEquals(code, "ok"); - - - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (ServletException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (JSONException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - } - -} diff --git a/extensions/database/test/com/google/refine/extension/database/cmd/SavedConnectionCommandTest.java b/extensions/database/test/com/google/refine/extension/database/cmd/SavedConnectionCommandTest.java index 3bf6bce13..bfa4dd30d 100644 --- a/extensions/database/test/com/google/refine/extension/database/cmd/SavedConnectionCommandTest.java +++ b/extensions/database/test/com/google/refine/extension/database/cmd/SavedConnectionCommandTest.java @@ -2,72 +2,178 @@ package com.google.refine.extension.database.cmd; import static org.mockito.Mockito.when; +import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; -import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.AfterSuite; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; -import com.google.refine.extension.database.pgsql.PgSQLDatabaseService; +import com.google.refine.ProjectManager; +import com.google.refine.ProjectMetadata; +import com.google.refine.RefineServlet; +import com.google.refine.extension.database.DBExtensionTestUtils; +import com.google.refine.extension.database.DBExtensionTests; +import com.google.refine.extension.database.DatabaseConfiguration; +import com.google.refine.extension.database.DatabaseImportController; +import com.google.refine.extension.database.DatabaseService; +import com.google.refine.extension.database.mysql.MySQLDatabaseService; +import com.google.refine.extension.database.stub.RefineDbServletStub; +import com.google.refine.importing.ImportingJob; +import com.google.refine.importing.ImportingManager; +import com.google.refine.io.FileProjectManager; +import com.google.refine.model.Project; -public class SavedConnectionCommandTest { +public class SavedConnectionCommandTest extends DBExtensionTests{ @Mock - HttpServletRequest request; + private HttpServletRequest request; @Mock - HttpServletResponse response; + private HttpServletResponse response; - @BeforeTest - public void beforeTest() { + private DatabaseConfiguration testDbConfig; + + private Project project; + private ProjectMetadata metadata; + //private ImportingJob job; + private RefineServlet servlet; + + // private String JSON_OPTION = "{\"mode\":\"row-based\"}}"; + + + //System under test + private SavedConnectionCommand SUT = null; + + @BeforeMethod + public void setUp() throws JSONException, IOException { MockitoAnnotations.initMocks(this); - } - - @Test - public void testDoPost() { - when(request.getParameter("connectionName")).thenReturn("test-db-name"); - when(request.getParameter("databaseType")).thenReturn(PgSQLDatabaseService.DB_NAME); - when(request.getParameter("databaseServer")).thenReturn("127.0.0.1"); - when(request.getParameter("databasePort")).thenReturn("5432"); - when(request.getParameter("databaseUser")).thenReturn("postgres"); - when(request.getParameter("databasePassword")).thenReturn(""); - when(request.getParameter("initialDatabase")).thenReturn("openrefine"); - when(request.getParameter("queryString")).thenReturn("SELECT count(*) FROM call_data"); + File dir = DBExtensionTestUtils.createTempDirectory("OR_DBExtension_Test_WorkspaceDir"); + FileProjectManager.initialize(dir); + + servlet = new RefineDbServletStub(); + ImportingManager.initialize(servlet); + project = new Project(); + metadata = new ProjectMetadata(); + //job = ImportingManager.createJob(); + + metadata.setName("Save DB Config Test Project"); + ProjectManager.singleton.registerProject(project, metadata); + SUT = new SavedConnectionCommand(); + + } + + @AfterMethod + public void tearDown() { + SUT = null; + request = null; + response = null; + project = null; + metadata = null; + // ImportingManager.disposeJob(job.id); + // job = null; + //options = null; + } + + @BeforeTest + @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) + public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, + @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, + @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { + + // MockitoAnnotations.initMocks(this); + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(mySqlDbHost); + testDbConfig.setDatabaseName(mySqlDbName); + testDbConfig.setDatabasePassword(mySqlDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); + testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(mySqlDbUser); + testDbConfig.setUseSSL(false); + + DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); + + } + + @AfterSuite + public void afterSuite() { + + + } + + private void saveDatabaseConfiguration(String savedDbName) { + + when(request.getParameter("connectionName")).thenReturn(savedDbName); + when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); + when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); + when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); + when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); + when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); + when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + try { + when(response.getWriter()).thenReturn(pw); + + SUT.doPost(request, response); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + + @Test + public void testDoPost() { + + when(request.getParameter("connectionName")).thenReturn("test-db-name"); + when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); + when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); + when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); + when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); + when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); + when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); + StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); try { when(response.getWriter()).thenReturn(pw); - SavedConnectionCommand scCommand = new SavedConnectionCommand(); - scCommand.doPost(request, response); + SUT.doPost(request, response); String result = sw.getBuffer().toString().trim(); - System.out.println("result:" + result); + JSONObject json = new JSONObject(result); - - String code = json.getString("code"); - Assert.assertEquals(code, "ok"); + + JSONArray savedConnections = json.getJSONArray("savedConnections"); + Assert.assertNotNull(savedConnections); + + int len = savedConnections.length(); + + Assert.assertEquals(len, 1); + - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (ServletException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (JSONException e) { + } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } @@ -76,17 +182,141 @@ public class SavedConnectionCommandTest { @Test public void testDoGet() { - //add test logic + String testDbName = "testLocalDb"; + //add saved connection + saveDatabaseConfiguration(testDbName); + + + when(request.getParameter("connectionName")).thenReturn(testDbName); + when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); + when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); + when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); + when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); + when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); + when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + try { + when(response.getWriter()).thenReturn(pw); + + SUT.doGet(request, response); + + JSONObject json = new JSONObject(sw.getBuffer().toString().trim()); + + JSONArray savedConnections = json.getJSONArray("savedConnections"); + Assert.assertNotNull(savedConnections); + + Assert.assertEquals(savedConnections.length(), 1); + + JSONObject sc = (JSONObject)savedConnections.get(0); + // System.out.println("sc" + sc); + String connName = sc.getString("connectionName"); + Assert.assertEquals(connName, testDbName); + + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } @Test public void testDoPut() { - //add test logic + String testDbName = "testLocalDb"; + saveDatabaseConfiguration(testDbName); + + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + try { + when(response.getWriter()).thenReturn(pw); + + //modify database config + String newHost = "localhost"; + when(request.getParameter("connectionName")).thenReturn(testDbName); + when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); + when(request.getParameter("databaseServer")).thenReturn(newHost); + when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); + when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); + when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); + when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); + + SUT.doPut(request, response); + + JSONObject json = new JSONObject(sw.getBuffer().toString().trim()); + JSONArray savedConnections = json.getJSONArray("savedConnections"); + Assert.assertNotNull(savedConnections); + + Assert.assertEquals(savedConnections.length(), 1); + + JSONObject sc = (JSONObject)savedConnections.get(0); + System.out.println("sc" + sc); + String newDbHost = sc.getString("databaseHost"); + Assert.assertEquals(newDbHost, newHost); + + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } @Test - public void testDoDelete() { - //add test logic + public void testDoDeleteValidConnectionName() { + String testDbName = "testLocalDb"; + saveDatabaseConfiguration(testDbName); + + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + try { + when(response.getWriter()).thenReturn(pw); + when(request.getParameter("connectionName")).thenReturn(testDbName); + SUT.doDelete(request, response); + + JSONObject json = new JSONObject(sw.getBuffer().toString().trim()); + JSONArray savedConnections = json.getJSONArray("savedConnections"); + Assert.assertNotNull(savedConnections); + + Assert.assertEquals(savedConnections.length(), 0); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + @Test + public void testDoDeleteInValidConnectionName() { + String testDbName = "testLocalDb"; + saveDatabaseConfiguration(testDbName); + + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + try { + when(response.getWriter()).thenReturn(pw); + + when(request.getParameter("connectionName")).thenReturn("noDbName"); + + SUT.doDelete(request, response); + + String result = sw.getBuffer().toString().trim(); + + JSONObject json = new JSONObject(); + + Assert.assertNotNull(json); + + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } } diff --git a/extensions/database/test/com/google/refine/extension/database/cmd/TestConnectCommandTest.java b/extensions/database/test/com/google/refine/extension/database/cmd/TestConnectCommandTest.java index 6c7313569..cb4f8f8bd 100644 --- a/extensions/database/test/com/google/refine/extension/database/cmd/TestConnectCommandTest.java +++ b/extensions/database/test/com/google/refine/extension/database/cmd/TestConnectCommandTest.java @@ -2,48 +2,81 @@ package com.google.refine.extension.database.cmd; import static org.mockito.Mockito.when; -import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; -import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.json.JSONException; import org.json.JSONObject; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.testng.Assert; +import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; -import com.google.refine.extension.database.pgsql.PgSQLDatabaseService; +import com.google.refine.extension.database.DBExtensionTestUtils; +import com.google.refine.extension.database.DBExtensionTests; +import com.google.refine.extension.database.DatabaseConfiguration; +import com.google.refine.extension.database.DatabaseService; +import com.google.refine.extension.database.mysql.MySQLDatabaseService; -public class TestConnectCommandTest { +public class TestConnectCommandTest extends DBExtensionTests{ @Mock - HttpServletRequest request; + private HttpServletRequest request; @Mock - HttpServletResponse response; - - @BeforeTest - public void beforeTest() { - MockitoAnnotations.initMocks(this); - } + private HttpServletResponse response; + private DatabaseConfiguration testDbConfig; + // private String testTable; + + + @BeforeTest + @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) + public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, + @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, + @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { + + MockitoAnnotations.initMocks(this); + + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(mySqlDbHost); + testDbConfig.setDatabaseName(mySqlDbName); + testDbConfig.setDatabasePassword(mySqlDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); + testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(mySqlDbUser); + testDbConfig.setUseSSL(false); + + //testTable = mySqlTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + + DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); + + } + + @AfterSuite + public void afterSuite() { + DBExtensionTestUtils.cleanUpTestData(testDbConfig); + + } + @Test public void testDoPost() { - when(request.getParameter("databaseType")).thenReturn(PgSQLDatabaseService.DB_NAME); - when(request.getParameter("databaseServer")).thenReturn("127.0.0.1"); - when(request.getParameter("databasePort")).thenReturn("5432"); - when(request.getParameter("databaseUser")).thenReturn("postgres"); - when(request.getParameter("databasePassword")).thenReturn(""); - when(request.getParameter("initialDatabase")).thenReturn("postgres"); + when(request.getParameter("databaseType")).thenReturn(MySQLDatabaseService.DB_NAME); + when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); + when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); + when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); + when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); + when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); StringWriter sw = new StringWriter(); @@ -63,16 +96,10 @@ public class TestConnectCommandTest { Assert.assertEquals(code, "ok"); - } catch (IOException e) { + } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); - } catch (ServletException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (JSONException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + } } diff --git a/extensions/database/test/com/google/refine/extension/database/cmd/TestQueryCommandTest.java b/extensions/database/test/com/google/refine/extension/database/cmd/TestQueryCommandTest.java new file mode 100644 index 000000000..6b604e6d0 --- /dev/null +++ b/extensions/database/test/com/google/refine/extension/database/cmd/TestQueryCommandTest.java @@ -0,0 +1,109 @@ +package com.google.refine.extension.database.cmd; + +import static org.mockito.Mockito.when; + +import java.io.PrintWriter; +import java.io.StringWriter; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONObject; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.Assert; +import org.testng.annotations.AfterSuite; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; +import org.testng.annotations.Test; + +import com.google.refine.extension.database.DBExtensionTestUtils; +import com.google.refine.extension.database.DBExtensionTests; +import com.google.refine.extension.database.DatabaseConfiguration; +import com.google.refine.extension.database.DatabaseService; +import com.google.refine.extension.database.mysql.MySQLDatabaseService; + + +public class TestQueryCommandTest extends DBExtensionTests { + + @Mock + private HttpServletRequest request; + + @Mock + private HttpServletResponse response; + + + private DatabaseConfiguration testDbConfig; + private String testTable; + + + @BeforeTest + @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) + public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, + @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, + @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { + + MockitoAnnotations.initMocks(this); + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(mySqlDbHost); + testDbConfig.setDatabaseName(mySqlDbName); + testDbConfig.setDatabasePassword(mySqlDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); + testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(mySqlDbUser); + testDbConfig.setUseSSL(false); + + testTable = mySqlTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + + DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); + + } + + @AfterSuite + public void afterSuite() { + DBExtensionTestUtils.cleanUpTestData(testDbConfig); + + } + + + @Test + public void testDoPost() { + + when(request.getParameter("databaseType")).thenReturn(testDbConfig.getDatabaseType()); + when(request.getParameter("databaseServer")).thenReturn(testDbConfig.getDatabaseHost()); + when(request.getParameter("databasePort")).thenReturn("" + testDbConfig.getDatabasePort()); + when(request.getParameter("databaseUser")).thenReturn(testDbConfig.getDatabaseUser()); + when(request.getParameter("databasePassword")).thenReturn(testDbConfig.getDatabasePassword()); + when(request.getParameter("initialDatabase")).thenReturn(testDbConfig.getDatabaseName()); + when(request.getParameter("query")).thenReturn("SELECT count(*) FROM " + testTable); + + + StringWriter sw = new StringWriter(); + + PrintWriter pw = new PrintWriter(sw); + + try { + when(response.getWriter()).thenReturn(pw); + TestQueryCommand executeQueryCommand = new TestQueryCommand(); + + executeQueryCommand.doPost(request, response); + + String result = sw.getBuffer().toString().trim(); + JSONObject json = new JSONObject(result); + + String code = json.getString("code"); + Assert.assertEquals(code, "ok"); + + String queryResult = json.getString("QueryResult"); + Assert.assertNotNull(queryResult); + + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + +} diff --git a/extensions/database/test/com/google/refine/extension/database/mariadb/MariaDBConnectionManagerTest.java b/extensions/database/test/com/google/refine/extension/database/mariadb/MariaDBConnectionManagerTest.java index 1e204cf34..c5cb4a6c8 100644 --- a/extensions/database/test/com/google/refine/extension/database/mariadb/MariaDBConnectionManagerTest.java +++ b/extensions/database/test/com/google/refine/extension/database/mariadb/MariaDBConnectionManagerTest.java @@ -1,39 +1,64 @@ package com.google.refine.extension.database.mariadb; import java.sql.Connection; -import java.sql.SQLException; +import org.mockito.MockitoAnnotations; import org.testng.Assert; +import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; +import com.google.refine.extension.database.DBExtensionTestUtils; +import com.google.refine.extension.database.DBExtensionTests; import com.google.refine.extension.database.DatabaseConfiguration; import com.google.refine.extension.database.DatabaseService; import com.google.refine.extension.database.DatabaseServiceException; -public class MariaDBConnectionManagerTest { - - - private static final int DATABASE_PORT = 3306; +public class MariaDBConnectionManagerTest extends DBExtensionTests { + + + + private DatabaseConfiguration testDbConfig; + + @BeforeTest - public void beforeTest() { - DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); - } + @Parameters({ "mariaDbName", "mariaDbHost", "mariaDbPort", "mariaDbUser", "mariaDbPassword", "mariaDbTestTable"}) + public void beforeTest(@Optional(DEFAULT_MARIADB_NAME) String mariaDbName, @Optional(DEFAULT_MARIADB_HOST) String mariaDbHost, + @Optional(DEFAULT_MARIADB_PORT) String mariaDbPort, @Optional(DEFAULT_MARIADB_USER) String mariaDbUser, + @Optional(DEFAULT_MARIADB_PASSWORD) String mariaDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariaDbTestTable) { + + MockitoAnnotations.initMocks(this); + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(mariaDbHost); + testDbConfig.setDatabaseName(mariaDbName); + testDbConfig.setDatabasePassword(mariaDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(mariaDbPort)); + testDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(mariaDbUser); + testDbConfig.setUseSSL(false); + +// testTable = mariaDbTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + + DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); + + } + + @AfterSuite + public void afterSuite() { + DBExtensionTestUtils.cleanUpTestData(testDbConfig); + + } @Test public void testTestConnection() { - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost("127.0.0.1"); - dc.setDatabaseName("test"); - dc.setDatabasePassword("secret"); - dc.setDatabasePort(DATABASE_PORT); - dc.setDatabaseType(MariaDBDatabaseService.DB_NAME); - dc.setDatabaseUser("root"); - dc.setUseSSL(false); + try { - boolean conn = MariaDBConnectionManager.getInstance().testConnection(dc); + boolean conn = MariaDBConnectionManager.getInstance().testConnection(testDbConfig); Assert.assertEquals(conn, true); } catch (DatabaseServiceException e) { @@ -44,16 +69,9 @@ public class MariaDBConnectionManagerTest { @Test public void testGetConnection() { - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost("127.0.0.1"); - dc.setDatabaseName("test"); - dc.setDatabasePassword("secret"); - dc.setDatabasePort(DATABASE_PORT); - dc.setDatabaseType(MariaDBDatabaseService.DB_NAME); - dc.setDatabaseUser("root"); - dc.setUseSSL(false); + try { - Connection conn = MariaDBConnectionManager.getInstance().getConnection(dc, true); + Connection conn = MariaDBConnectionManager.getInstance().getConnection(testDbConfig, true); Assert.assertNotNull(conn); } catch (DatabaseServiceException e) { @@ -64,17 +82,9 @@ public class MariaDBConnectionManagerTest { @Test public void testShutdown() { - - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost("127.0.0.1"); - dc.setDatabaseName("test"); - dc.setDatabasePassword("secret"); - dc.setDatabasePort(DATABASE_PORT); - dc.setDatabaseType(MariaDBDatabaseService.DB_NAME); - dc.setDatabaseUser("root"); - dc.setUseSSL(false); + try { - Connection conn = MariaDBConnectionManager.getInstance().getConnection(dc, true); + Connection conn = MariaDBConnectionManager.getInstance().getConnection(testDbConfig, true); Assert.assertNotNull(conn); MariaDBConnectionManager.getInstance().shutdown(); @@ -83,10 +93,7 @@ public class MariaDBConnectionManagerTest { Assert.assertEquals(conn.isClosed(), true); } - } catch (DatabaseServiceException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (SQLException e) { + } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } diff --git a/extensions/database/test/com/google/refine/extension/database/mariadb/MariaDBDatabaseServiceTest.java b/extensions/database/test/com/google/refine/extension/database/mariadb/MariaDBDatabaseServiceTest.java index ed7ab02aa..18b052b1b 100644 --- a/extensions/database/test/com/google/refine/extension/database/mariadb/MariaDBDatabaseServiceTest.java +++ b/extensions/database/test/com/google/refine/extension/database/mariadb/MariaDBDatabaseServiceTest.java @@ -1,58 +1,178 @@ package com.google.refine.extension.database.mariadb; +import java.sql.Connection; +import java.util.List; + +import org.mockito.MockitoAnnotations; +import org.testng.Assert; +import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; -public class MariaDBDatabaseServiceTest { +import com.google.refine.extension.database.DBExtensionTestUtils; +import com.google.refine.extension.database.DBExtensionTests; +import com.google.refine.extension.database.DatabaseConfiguration; +import com.google.refine.extension.database.DatabaseService; +import com.google.refine.extension.database.DatabaseServiceException; +import com.google.refine.extension.database.model.DatabaseColumn; +import com.google.refine.extension.database.model.DatabaseInfo; +import com.google.refine.extension.database.model.DatabaseRow; + + +public class MariaDBDatabaseServiceTest extends DBExtensionTests{ + + + + private DatabaseConfiguration testDbConfig; + + private String testTable; + @BeforeTest - public void beforeTest() { - //add test logic - } + @Parameters({ "mariaDbName", "mariaDbHost", "mariaDbPort", "mariaDbUser", "mariaDbPassword", "mariaDbTestTable"}) + public void beforeTest(@Optional(DEFAULT_MARIADB_NAME) String mariaDbName, @Optional(DEFAULT_MARIADB_HOST) String mariaDbHost, + @Optional(DEFAULT_MARIADB_PORT) String mariaDbPort, @Optional(DEFAULT_MARIADB_USER) String mariaDbUser, + @Optional(DEFAULT_MARIADB_PASSWORD) String mariaDbPassword, @Optional(DEFAULT_TEST_TABLE) String mariaDbTestTable) { + + MockitoAnnotations.initMocks(this); + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(mariaDbHost); + testDbConfig.setDatabaseName(mariaDbName); + testDbConfig.setDatabasePassword(mariaDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(mariaDbPort)); + testDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(mariaDbUser); + testDbConfig.setUseSSL(false); + + testTable = mariaDbTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + + DatabaseService.DBType.registerDatabase(MariaDBDatabaseService.DB_NAME, MariaDBDatabaseService.getInstance()); + + } + @Test public void testGetDatabaseUrl() { - //add test logic + MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService)DatabaseService.get(MariaDBDatabaseService.DB_NAME); + String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig); + //System.out.println("dbUrl:" + dbUrl); + Assert.assertNotNull(dbUrl); + Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); } - @Test - public void testGetConnection() { - //add test logic - } + @Test + public void testGetConnection() { + try { + + MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService)DatabaseService.get(MariaDBDatabaseService.DB_NAME); + Connection conn = pgSqlService.getConnection(testDbConfig); + + Assert.assertNotNull(conn); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } - @Test - public void testTestConnection() { - //add test logic - } + @Test + public void testTestConnection() { + try { + MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService)DatabaseService.get(MariaDBDatabaseService.DB_NAME); + + boolean result = pgSqlService.testConnection(testDbConfig); + Assert.assertEquals(result, true); - @Test - public void testConnect() { - //add test logic - } + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } - @Test - public void testExecuteQuery() { - //add test logic - } + @Test + public void testConnect() { + try { + MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService)DatabaseService.get(MariaDBDatabaseService.DB_NAME); + DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig); + Assert.assertNotNull(databaseInfo); + + } catch (DatabaseServiceException e) { + e.printStackTrace(); + } + } + + @Test + public void testExecuteQuery() { - @Test - public void testBuildLimitQuery() { - //add test logic - } + try { - @Test - public void testGetRows() { - //add test logic - } + MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService + .get(MariaDBDatabaseService.DB_NAME); + DatabaseInfo databaseInfo = pgSqlService.testQuery(testDbConfig, "SELECT * FROM " + testTable); - @Test - public void testGetInstance() { - //add test logic - } + Assert.assertNotNull(databaseInfo); - @Test - public void testGetColumnsDatabaseConfigurationString() { - //add test logic - } + } catch (DatabaseServiceException e) { + e.printStackTrace(); + } + } + @Test + public void testBuildLimitQuery() { + MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService.get(MariaDBDatabaseService.DB_NAME); + String limitQuery = pgSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable); + Assert.assertNotNull(limitQuery); + + Assert.assertEquals(limitQuery, "SELECT * FROM " + testTable + " LIMIT " + 100 + " OFFSET " + 0); + + } + + @Test + public void testGetRows() { + try { + MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService + .get(MariaDBDatabaseService.DB_NAME); + List dbRows = pgSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable); + + Assert.assertNotNull(dbRows); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + @Test + public void testGetInstance() { + MariaDBDatabaseService instance = MariaDBDatabaseService.getInstance(); + Assert.assertNotNull(instance); + } + + @Test + public void testGetColumns() { + List dbColumns; + + try { + MariaDBDatabaseService pgSqlService = (MariaDBDatabaseService) DatabaseService + .get(MariaDBDatabaseService.DB_NAME); + + dbColumns = pgSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable); + + Assert.assertNotNull(dbColumns); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + + @AfterSuite + public void afterSuite() { + DBExtensionTestUtils.cleanUpTestData(testDbConfig); + logger.info("<>"); + } } diff --git a/extensions/database/test/com/google/refine/extension/database/mysql/MySQLConnectionManagerTest.java b/extensions/database/test/com/google/refine/extension/database/mysql/MySQLConnectionManagerTest.java index 77f885ae5..4f6ff6e5a 100644 --- a/extensions/database/test/com/google/refine/extension/database/mysql/MySQLConnectionManagerTest.java +++ b/extensions/database/test/com/google/refine/extension/database/mysql/MySQLConnectionManagerTest.java @@ -1,39 +1,60 @@ package com.google.refine.extension.database.mysql; import java.sql.Connection; -import java.sql.SQLException; +import org.mockito.MockitoAnnotations; import org.testng.Assert; +import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; +import com.google.refine.extension.database.DBExtensionTestUtils; +import com.google.refine.extension.database.DBExtensionTests; import com.google.refine.extension.database.DatabaseConfiguration; import com.google.refine.extension.database.DatabaseService; import com.google.refine.extension.database.DatabaseServiceException; -public class MySQLConnectionManagerTest { +public class MySQLConnectionManagerTest extends DBExtensionTests { + + private DatabaseConfiguration testDbConfig; - private static final int DATABASE_PORT = 3306; - @BeforeTest - public void beforeTest() { - DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); - } + @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) + public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, + @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, + @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { + + MockitoAnnotations.initMocks(this); + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(mySqlDbHost); + testDbConfig.setDatabaseName(mySqlDbName); + testDbConfig.setDatabasePassword(mySqlDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); + testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(mySqlDbUser); + testDbConfig.setUseSSL(false); + + //testTable = mySqlTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + + DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); + + } + + @AfterSuite + public void afterSuite() { + DBExtensionTestUtils.cleanUpTestData(testDbConfig); + } @Test public void testTestConnection() { - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost("127.0.0.1"); - dc.setDatabaseName("mysql"); - dc.setDatabasePassword("secret"); - dc.setDatabasePort(DATABASE_PORT); - dc.setDatabaseType(MySQLDatabaseService.DB_NAME); - dc.setDatabaseUser("root"); - dc.setUseSSL(false); + try { - boolean conn = MySQLConnectionManager.getInstance().testConnection(dc); + boolean conn = MySQLConnectionManager.getInstance().testConnection(testDbConfig); Assert.assertEquals(conn, true); } catch (DatabaseServiceException e) { @@ -44,16 +65,9 @@ public class MySQLConnectionManagerTest { @Test public void testGetConnection() { - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost("127.0.0.1"); - dc.setDatabaseName("mysql"); - dc.setDatabasePassword("secret"); - dc.setDatabasePort(DATABASE_PORT); - dc.setDatabaseType(MySQLDatabaseService.DB_NAME); - dc.setDatabaseUser("root"); - dc.setUseSSL(false); + try { - Connection conn = MySQLConnectionManager.getInstance().getConnection(dc, true); + Connection conn = MySQLConnectionManager.getInstance().getConnection(testDbConfig, true); Assert.assertNotNull(conn); } catch (DatabaseServiceException e) { @@ -64,17 +78,9 @@ public class MySQLConnectionManagerTest { @Test public void testShutdown() { - - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost("127.0.0.1"); - dc.setDatabaseName("mysql"); - dc.setDatabasePassword("secret"); - dc.setDatabasePort(DATABASE_PORT); - dc.setDatabaseType(MySQLDatabaseService.DB_NAME); - dc.setDatabaseUser("root"); - dc.setUseSSL(false); + try { - Connection conn = MySQLConnectionManager.getInstance().getConnection(dc, true); + Connection conn = MySQLConnectionManager.getInstance().getConnection(testDbConfig, true); Assert.assertNotNull(conn); MySQLConnectionManager.getInstance().shutdown(); @@ -83,10 +89,7 @@ public class MySQLConnectionManagerTest { Assert.assertEquals(conn.isClosed(), true); } - } catch (DatabaseServiceException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (SQLException e) { + } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } diff --git a/extensions/database/test/com/google/refine/extension/database/mysql/MySQLDatabaseServiceTest.java b/extensions/database/test/com/google/refine/extension/database/mysql/MySQLDatabaseServiceTest.java index 05f41ed1b..bebe5dcba 100644 --- a/extensions/database/test/com/google/refine/extension/database/mysql/MySQLDatabaseServiceTest.java +++ b/extensions/database/test/com/google/refine/extension/database/mysql/MySQLDatabaseServiceTest.java @@ -1,58 +1,178 @@ package com.google.refine.extension.database.mysql; +import java.sql.Connection; +import java.util.List; + +import org.mockito.MockitoAnnotations; +import org.testng.Assert; +import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; -public class MySQLDatabaseServiceTest { +import com.google.refine.extension.database.DBExtensionTestUtils; +import com.google.refine.extension.database.DBExtensionTests; +import com.google.refine.extension.database.DatabaseConfiguration; +import com.google.refine.extension.database.DatabaseService; +import com.google.refine.extension.database.DatabaseServiceException; +import com.google.refine.extension.database.model.DatabaseColumn; +import com.google.refine.extension.database.model.DatabaseInfo; +import com.google.refine.extension.database.model.DatabaseRow; + + + +public class MySQLDatabaseServiceTest extends DBExtensionTests{ + + private DatabaseConfiguration testDbConfig; + private String testTable; @BeforeTest - public void beforeTest() { - //add test logic + @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable"}) + public void beforeTest(@Optional(DEFAULT_MYSQL_DB_NAME) String mySqlDbName, @Optional(DEFAULT_MYSQL_HOST) String mySqlDbHost, + @Optional(DEFAULT_MYSQL_PORT) String mySqlDbPort, @Optional(DEFAULT_MYSQL_USER) String mySqlDbUser, + @Optional(DEFAULT_MYSQL_PASSWORD) String mySqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String mySqlTestTable) { + + MockitoAnnotations.initMocks(this); + + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(mySqlDbHost); + testDbConfig.setDatabaseName(mySqlDbName); + testDbConfig.setDatabasePassword(mySqlDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(mySqlDbPort)); + testDbConfig.setDatabaseType(MySQLDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(mySqlDbUser); + testDbConfig.setUseSSL(false); + + testTable = mySqlTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + + DatabaseService.DBType.registerDatabase(MySQLDatabaseService.DB_NAME, MySQLDatabaseService.getInstance()); + } + @Test public void testGetDatabaseUrl() { - //add test logic + MySQLDatabaseService pgSqlService = (MySQLDatabaseService)DatabaseService.get(MySQLDatabaseService.DB_NAME); + String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig); + //System.out.println("dbUrl:" + dbUrl); + Assert.assertNotNull(dbUrl); + Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); } + - @Test - public void testGetConnection() { - //add test logic - } + @Test + public void testGetConnection() { + try { + + MySQLDatabaseService pgSqlService = (MySQLDatabaseService)DatabaseService.get(MySQLDatabaseService.DB_NAME); + Connection conn = pgSqlService.getConnection(testDbConfig); + + Assert.assertNotNull(conn); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } - @Test - public void testTestConnection() { - //add test logic - } + @Test + public void testTestConnection() { + try { + MySQLDatabaseService pgSqlService = (MySQLDatabaseService)DatabaseService.get(MySQLDatabaseService.DB_NAME); + + boolean result = pgSqlService.testConnection(testDbConfig); + Assert.assertEquals(result, true); - @Test - public void testConnect() { - //add test logic - } + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } - @Test - public void testExecuteQuery() { - //add test logic - } + @Test + public void testConnect() { + try { + MySQLDatabaseService pgSqlService = (MySQLDatabaseService)DatabaseService.get(MySQLDatabaseService.DB_NAME); + DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig); + Assert.assertNotNull(databaseInfo); + + } catch (DatabaseServiceException e) { + e.printStackTrace(); + } + } + + @Test + public void testExecuteQuery() { - @Test - public void testBuildLimitQuery() { - //add test logic - } + try { - @Test - public void testGetRows() { - //add test logic - } + MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService + .get(MySQLDatabaseService.DB_NAME); + DatabaseInfo databaseInfo = pgSqlService.testQuery(testDbConfig, "SELECT * FROM " + testTable); - @Test - public void testGetInstance() { - //add test logic - } + Assert.assertNotNull(databaseInfo); - @Test - public void testGetColumnsDatabaseConfigurationString() { - //add test logic - } + } catch (DatabaseServiceException e) { + e.printStackTrace(); + } + } + + @Test + public void testBuildLimitQuery() { + MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService.get(MySQLDatabaseService.DB_NAME); + String limitQuery = pgSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable); + Assert.assertNotNull(limitQuery); + + Assert.assertEquals(limitQuery, "SELECT * FROM " + testTable + " LIMIT " + 100 + " OFFSET " + 0 + ";"); + + } + + @Test + public void testGetRows() { + try { + MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService + .get(MySQLDatabaseService.DB_NAME); + List dbRows = pgSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable); + + Assert.assertNotNull(dbRows); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + @Test + public void testGetInstance() { + MySQLDatabaseService instance = MySQLDatabaseService.getInstance(); + Assert.assertNotNull(instance); + } + + @Test + public void testGetColumns() { + List dbColumns; + + try { + MySQLDatabaseService pgSqlService = (MySQLDatabaseService) DatabaseService + .get(MySQLDatabaseService.DB_NAME); + + dbColumns = pgSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable); + + Assert.assertNotNull(dbColumns); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + + @AfterSuite + public void afterSuite() { + DBExtensionTestUtils.cleanUpTestData(testDbConfig); + + } } diff --git a/extensions/database/test/com/google/refine/extension/database/pgsql/PgSQLConnectionManagerTest.java b/extensions/database/test/com/google/refine/extension/database/pgsql/PgSQLConnectionManagerTest.java index a393db95b..65e005e03 100644 --- a/extensions/database/test/com/google/refine/extension/database/pgsql/PgSQLConnectionManagerTest.java +++ b/extensions/database/test/com/google/refine/extension/database/pgsql/PgSQLConnectionManagerTest.java @@ -1,39 +1,65 @@ package com.google.refine.extension.database.pgsql; import java.sql.Connection; -import java.sql.SQLException; +import org.mockito.MockitoAnnotations; import org.testng.Assert; +import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeTest; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; +import com.google.refine.extension.database.DBExtensionTestUtils; +import com.google.refine.extension.database.DBExtensionTests; import com.google.refine.extension.database.DatabaseConfiguration; import com.google.refine.extension.database.DatabaseService; import com.google.refine.extension.database.DatabaseServiceException; -public class PgSQLConnectionManagerTest { +public class PgSQLConnectionManagerTest extends DBExtensionTests { + + private DatabaseConfiguration testDbConfig; @BeforeTest - public void beforeTest() { + @Parameters({ "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable"}) + public void beforeTest(@Optional(DEFAULT_PGSQL_DB_NAME) String pgSqlDbName, @Optional(DEFAULT_PGSQL_HOST) String pgSqlDbHost, + @Optional(DEFAULT_PGSQL_PORT) String pgSqlDbPort, @Optional(DEFAULT_PGSQL_USER) String pgSqlDbUser, + @Optional(DEFAULT_PGSQL_PASSWORD) String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String pgSqlTestTable) { + + MockitoAnnotations.initMocks(this); + // System.out.println("beforeTest " + pgSqlDbName); + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(pgSqlDbHost); + testDbConfig.setDatabaseName(pgSqlDbName); + testDbConfig.setDatabasePassword(pgSqlDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(pgSqlDbPort)); + testDbConfig.setDatabaseType(PgSQLDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(pgSqlDbUser); + testDbConfig.setUseSSL(false); + + //testTable = mySqlTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance()); + + } + + + + @AfterSuite + public void afterSuite() { + DBExtensionTestUtils.cleanUpTestData(testDbConfig); } - @Test public void testTestConnection() { - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost("127.0.0.1"); - dc.setDatabaseName("enmsso"); - dc.setDatabasePort(5432); - dc.setDatabaseType(PgSQLDatabaseService.DB_NAME); - dc.setDatabaseUser("postgres"); - dc.setUseSSL(false); + try { - boolean conn = PgSQLConnectionManager.getInstance().testConnection(dc); - Assert.assertEquals(conn, true); + boolean isConnected = PgSQLConnectionManager.getInstance().testConnection(testDbConfig); + Assert.assertEquals(isConnected, true); } catch (DatabaseServiceException e) { // TODO Auto-generated catch block @@ -43,16 +69,9 @@ public class PgSQLConnectionManagerTest { @Test public void testGetConnection() { - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost("127.0.0.1"); - dc.setDatabaseName("enmsso"); - //dc.setDatabasePassword("secret"); - dc.setDatabasePort(5432); - dc.setDatabaseType(PgSQLDatabaseService.DB_NAME); - dc.setDatabaseUser("postgres"); - dc.setUseSSL(false); + try { - Connection conn = PgSQLConnectionManager.getInstance().getConnection(dc, true); + Connection conn = PgSQLConnectionManager.getInstance().getConnection(testDbConfig, true); Assert.assertNotNull(conn); } catch (DatabaseServiceException e) { @@ -63,17 +82,9 @@ public class PgSQLConnectionManagerTest { @Test public void testShutdown() { - - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost("127.0.0.1"); - dc.setDatabaseName("enmsso"); - //dc.setDatabasePassword("secret"); - dc.setDatabasePort(5432); - dc.setDatabaseType(PgSQLDatabaseService.DB_NAME); - dc.setDatabaseUser("postgres"); - dc.setUseSSL(false); + try { - Connection conn = PgSQLConnectionManager.getInstance().getConnection(dc, true); + Connection conn = PgSQLConnectionManager.getInstance().getConnection(testDbConfig, true); Assert.assertNotNull(conn); PgSQLConnectionManager.getInstance().shutdown(); @@ -82,10 +93,7 @@ public class PgSQLConnectionManagerTest { Assert.assertEquals(conn.isClosed(), true); } - } catch (DatabaseServiceException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (SQLException e) { + } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } diff --git a/extensions/database/test/com/google/refine/extension/database/pgsql/PgSQLDatabaseServiceTest.java b/extensions/database/test/com/google/refine/extension/database/pgsql/PgSQLDatabaseServiceTest.java index 50d2e17c1..d9697ccfd 100644 --- a/extensions/database/test/com/google/refine/extension/database/pgsql/PgSQLDatabaseServiceTest.java +++ b/extensions/database/test/com/google/refine/extension/database/pgsql/PgSQLDatabaseServiceTest.java @@ -1,85 +1,178 @@ package com.google.refine.extension.database.pgsql; +import java.sql.Connection; +import java.util.List; + +import org.mockito.MockitoAnnotations; import org.testng.Assert; +import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeTest; import org.testng.annotations.Optional; import org.testng.annotations.Parameters; import org.testng.annotations.Test; +import com.google.refine.extension.database.DBExtensionTestUtils; +import com.google.refine.extension.database.DBExtensionTests; import com.google.refine.extension.database.DatabaseConfiguration; import com.google.refine.extension.database.DatabaseService; +import com.google.refine.extension.database.DatabaseServiceException; +import com.google.refine.extension.database.model.DatabaseColumn; +import com.google.refine.extension.database.model.DatabaseInfo; +import com.google.refine.extension.database.model.DatabaseRow; -public class PgSQLDatabaseServiceTest { - +public class PgSQLDatabaseServiceTest extends DBExtensionTests{ + + private DatabaseConfiguration testDbConfig; + private String testTable; + + @BeforeTest - public void beforeTest() { + @Parameters({ "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable"}) + public void beforeTest(@Optional(DEFAULT_PGSQL_DB_NAME) String pgSqlDbName, @Optional(DEFAULT_PGSQL_HOST) String pgSqlDbHost, + @Optional(DEFAULT_PGSQL_PORT) String pgSqlDbPort, @Optional(DEFAULT_PGSQL_USER) String pgSqlDbUser, + @Optional(DEFAULT_PGSQL_PASSWORD) String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE) String pgSqlTestTable) { + + MockitoAnnotations.initMocks(this); + // System.out.println("beforeTest " + pgSqlDbName); + testDbConfig = new DatabaseConfiguration(); + testDbConfig.setDatabaseHost(pgSqlDbHost); + testDbConfig.setDatabaseName(pgSqlDbName); + testDbConfig.setDatabasePassword(pgSqlDbPassword); + testDbConfig.setDatabasePort(Integer.parseInt(pgSqlDbPort)); + testDbConfig.setDatabaseType(PgSQLDatabaseService.DB_NAME); + testDbConfig.setDatabaseUser(pgSqlDbUser); + testDbConfig.setUseSSL(false); + + testTable = pgSqlTestTable; + DBExtensionTestUtils.initTestData(testDbConfig); + DatabaseService.DBType.registerDatabase(PgSQLDatabaseService.DB_NAME, PgSQLDatabaseService.getInstance()); } + + @Test - @Parameters("dbName") - public void testGetDatabaseUrl(@Optional("postgres") String dbName) { - DatabaseConfiguration dc = new DatabaseConfiguration(); - dc.setDatabaseHost("127.0.0.1"); - dc.setDatabaseName(dbName); - dc.setDatabasePort(5432); - dc.setDatabaseType(PgSQLDatabaseService.DB_NAME); - dc.setDatabaseUser("postgres"); - dc.setUseSSL(false); - - String dbURL = PgSQLDatabaseService.getInstance().getDatabaseUrl(dc); - Assert.assertEquals(dbURL, "jdbc:postgresql://127.0.0.1:5432/" + dbName + "?useSSL=false"); + public void testGetDatabaseUrl() { + PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService)DatabaseService.get(PgSQLDatabaseService.DB_NAME); + String dbUrl = pgSqlService.getDatabaseUrl(testDbConfig); + System.out.println("dbUrl:" + dbUrl); + Assert.assertNotNull(dbUrl); + Assert.assertEquals(dbUrl, DBExtensionTestUtils.getJDBCUrl(testDbConfig)); + } + + + @Test + public void testGetConnection() { + try { + + PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService)DatabaseService.get(PgSQLDatabaseService.DB_NAME); + Connection conn = pgSqlService.getConnection(testDbConfig); + Assert.assertNotNull(conn); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } @Test - @Parameters("dbName") - public void testGetConnection(@Optional("postgres") String dbName) { - //add test logic + public void testTestConnection() { + try { + PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService)DatabaseService.get(PgSQLDatabaseService.DB_NAME); + + boolean result = pgSqlService.testConnection(testDbConfig); + Assert.assertEquals(result, true); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } @Test - @Parameters("dbName") - public void testTestConnection(@Optional("postgres") String dbName) { - //add test logic + public void testConnect() { + try { + PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService)DatabaseService.get(PgSQLDatabaseService.DB_NAME); + DatabaseInfo databaseInfo = pgSqlService.connect(testDbConfig); + Assert.assertNotNull(databaseInfo); + + } catch (DatabaseServiceException e) { + e.printStackTrace(); + } } - + @Test - @Parameters("dbName") - public void testConnect(@Optional("postgres") String dbName) { - //add test logic - } + public void testExecuteQuery() { - @Test - @Parameters("dbName") - public void testExecuteQuery(@Optional("postgres") String dbName) { - //add test logic + try { + + PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService + .get(PgSQLDatabaseService.DB_NAME); + DatabaseInfo databaseInfo = pgSqlService.testQuery(testDbConfig, "SELECT * FROM " + testTable); + + Assert.assertNotNull(databaseInfo); + + } catch (DatabaseServiceException e) { + e.printStackTrace(); + } } @Test public void testBuildLimitQuery() { - String query = PgSQLDatabaseService.getInstance().buildLimitQuery(100, 10, "SELECT * FROM TEST"); - Assert.assertEquals(query, "SELECT * FROM TEST LIMIT 100 OFFSET 10"); + PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService.get(PgSQLDatabaseService.DB_NAME); + String limitQuery = pgSqlService.buildLimitQuery(100, 0, "SELECT * FROM " + testTable); + Assert.assertNotNull(limitQuery); + + Assert.assertEquals(limitQuery, "SELECT * FROM " + testTable + " LIMIT " + 100 + " OFFSET " + 0 + ";"); } @Test - @Parameters("dbName") - public void testGetRows(@Optional("postgres") String dbName) { - //add test logic + public void testGetRows() { + try { + PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService + .get(PgSQLDatabaseService.DB_NAME); + List dbRows = pgSqlService.getRows(testDbConfig, "SELECT * FROM " + testTable); + + Assert.assertNotNull(dbRows); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } @Test - @Parameters("dbName") - public void testGetInstance(@Optional("postgres") String dbName) { - //add test logic + public void testGetInstance() { + PgSQLDatabaseService instance = PgSQLDatabaseService.getInstance(); + Assert.assertNotNull(instance); } @Test - @Parameters("dbName") - public void testGetColumns(@Optional("postgres") String dbName) { - //add test logic + public void testGetColumns() { + List dbColumns; + + try { + PgSQLDatabaseService pgSqlService = (PgSQLDatabaseService) DatabaseService + .get(PgSQLDatabaseService.DB_NAME); + + dbColumns = pgSqlService.getColumns(testDbConfig, "SELECT * FROM " + testTable); + + Assert.assertNotNull(dbColumns); + + } catch (DatabaseServiceException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + + @AfterSuite + public void afterClass() { + DBExtensionTestUtils.cleanUpTestData(testDbConfig); + } } diff --git a/extensions/database/test/com/google/refine/extension/database/stub/RefineDbServletStub.java b/extensions/database/test/com/google/refine/extension/database/stub/RefineDbServletStub.java new file mode 100644 index 000000000..c76e85c26 --- /dev/null +++ b/extensions/database/test/com/google/refine/extension/database/stub/RefineDbServletStub.java @@ -0,0 +1,95 @@ +/* + +Copyright 2010, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.extension.database.stub; + +import java.io.File; +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.google.refine.RefineServlet; +import com.google.refine.commands.Command; + +/** + * Exposes protected methods of com.google.refine.RefineServlet as public for unit testing + * + */ +public class RefineDbServletStub extends RefineServlet { + + private static File tempDir = null; + + //requirement of extending HttpServlet, not required for testing + private static final long serialVersionUID = 1L; + + public void wrapService(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException{ + super.service(request, response); + } + + public String wrapGetCommandName(HttpServletRequest request){ + return super.getCommandKey(request); + } + + @Override + public File getTempDir() { + if (tempDir == null) { + try { + tempDir = File.createTempFile("refine-test-dir", ""); + tempDir.deleteOnExit(); + } catch (IOException e) { + throw new RuntimeException("Failed to create temp directory",e); + } + } + return tempDir; + } + + //-------------------helper methods-------------- + /** + * Helper method for inserting a mock object + * @param commandName + * @param command + */ + public void insertCommand(String commandName, Command command ){ + registerOneCommand("core/" + commandName, command); + } + + /** + * Helper method for clearing up after testing + * @param commandName + */ + public void removeCommand( String commandName ){ + unregisterCommand("core/" + commandName); + } +} diff --git a/extensions/database/test/log4j-test.properties b/extensions/database/test/log4j-test.properties new file mode 100644 index 000000000..0ed0d9f64 --- /dev/null +++ b/extensions/database/test/log4j-test.properties @@ -0,0 +1,5 @@ +log4j.rootLogger=ERROR, console +log4j.logger.com.google.refine.extension.database=DEBUG + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=com.google.refine.logging.IndentingLayout diff --git a/extensions/database/test/testng.xml b/extensions/database/test/testng.xml index e31b4f176..870ea190f 100644 --- a/extensions/database/test/testng.xml +++ b/extensions/database/test/testng.xml @@ -1,10 +1,28 @@ - - + + + + + + + + + + + + + + + + + + + + - + + + + + - --> + \ No newline at end of file