diff --git a/main/src/com/google/refine/commands/project/ExportRowsCommand.java b/main/src/com/google/refine/commands/project/ExportRowsCommand.java index 664064b19..3270c671c 100644 --- a/main/src/com/google/refine/commands/project/ExportRowsCommand.java +++ b/main/src/com/google/refine/commands/project/ExportRowsCommand.java @@ -44,6 +44,10 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.http.HttpStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.google.refine.ProjectManager; import com.google.refine.browsing.Engine; import com.google.refine.commands.Command; @@ -52,9 +56,12 @@ import com.google.refine.exporters.Exporter; import com.google.refine.exporters.ExporterRegistry; import com.google.refine.exporters.StreamExporter; import com.google.refine.exporters.WriterExporter; +import com.google.refine.exporters.sql.SqlExporterException; + import com.google.refine.model.Project; public class ExportRowsCommand extends Command { + private static final Logger logger = LoggerFactory.getLogger("ExportRowsCommand"); @SuppressWarnings("unchecked") static public Properties getRequestParameters(HttpServletRequest request) { @@ -73,6 +80,7 @@ public class ExportRowsCommand extends Command { throws ServletException, IOException { ProjectManager.singleton.setBusy(true); + try { Project project = getProject(request); Engine engine = getEngine(request, project); @@ -100,7 +108,8 @@ public class ExportRowsCommand extends Command { ((WriterExporter) exporter).export(project, params, engine, writer); writer.close(); - } else if (exporter instanceof StreamExporter) { + } + else if (exporter instanceof StreamExporter) { response.setCharacterEncoding("UTF-8"); OutputStream stream = response.getOutputStream(); @@ -108,12 +117,17 @@ public class ExportRowsCommand extends Command { stream.close(); // } else if (exporter instanceof UrlExporter) { // ((UrlExporter) exporter).export(project, options, engine); + } else { // TODO: Should this use ServletException instead of respondException? respondException(response, new RuntimeException("Unknown exporter type")); } } catch (Exception e) { // Use generic error handling rather than our JSON handling + logger.info("error:{}", e.getMessage()); + if (e instanceof SqlExporterException) { + response.sendError(HttpStatus.SC_BAD_REQUEST, e.getMessage()); + } throw new ServletException(e); } finally { ProjectManager.singleton.setBusy(false); diff --git a/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java b/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java index 7f392fc46..82dd19279 100644 --- a/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java +++ b/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java @@ -229,12 +229,16 @@ abstract public class CustomizableTabularExporterUtilities { Map identifierSpaceToUrl = null; + //SQLExporter parameter to convert null cell value to empty string + boolean includeNullFieldValue = false; + CellFormatter() { dateFormatter = new SimpleDateFormat(fullIso8601); } CellFormatter(JSONObject options) { JSONObject reconSettings = JSONUtilities.getObject(options, "reconSettings"); + includeNullFieldValue = JSONUtilities.getBoolean(options, "nullValueToEmptyStr", false); if (reconSettings != null) { String reconOutputString = JSONUtilities.getString(reconSettings, "output", null); if ("entity-name".equals(reconOutputString)) { @@ -354,6 +358,12 @@ abstract public class CustomizableTabularExporterUtilities { } return new CellData(column.getName(), value, text, link); } + }else {//added for sql exporter + + if(includeNullFieldValue) { + return new CellData(column.getName(), "", "", ""); + } + } return null; } @@ -384,4 +394,4 @@ abstract public class CustomizableTabularExporterUtilities { } } } -} +} \ No newline at end of file diff --git a/main/src/com/google/refine/exporters/ExporterRegistry.java b/main/src/com/google/refine/exporters/ExporterRegistry.java index a8f321157..69bc92103 100644 --- a/main/src/com/google/refine/exporters/ExporterRegistry.java +++ b/main/src/com/google/refine/exporters/ExporterRegistry.java @@ -36,6 +36,8 @@ package com.google.refine.exporters; import java.util.HashMap; import java.util.Map; +import com.google.refine.exporters.sql.SqlExporter; + abstract public class ExporterRegistry { static final private Map s_formatToExporter = new HashMap(); @@ -52,6 +54,8 @@ abstract public class ExporterRegistry { s_formatToExporter.put("html", new HtmlTableExporter()); s_formatToExporter.put("template", new TemplatingExporter()); + + s_formatToExporter.put("sql", new SqlExporter()); } static public void registerExporter(String format, Exporter exporter) { diff --git a/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java new file mode 100755 index 000000000..55a62b591 --- /dev/null +++ b/main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2018, Tony Opara + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.exporters.sql; + +import java.util.List; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.refine.util.JSONUtilities; + +public class SqlCreateBuilder { + + private final static Logger logger = LoggerFactory.getLogger("SqlCreateBuilder"); + + private String table; + private List columns; + private JSONObject options; + + + public SqlCreateBuilder(String table, List columns, JSONObject options) { + this.table = table; + this.columns = columns; + this.options = options; + + } + + public String getCreateSQL() { + if(logger.isDebugEnabled()) { + logger.debug("Create SQL with columns: {}", columns); + } + StringBuffer createSB = new StringBuffer(); + + JSONArray columnOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns"); + boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false); + + + + int count = columnOptionArray.length(); + + for (int i = 0; i < count; i++) { + JSONObject columnOptions = JSONUtilities.getObjectElement(columnOptionArray, i); + if (columnOptions != null) { + String name = JSONUtilities.getString(columnOptions, "name", null); + String type = JSONUtilities.getString(columnOptions, "type", SqlData.SQL_TYPE_VARCHAR); + String size = JSONUtilities.getString(columnOptions, "size", ""); + boolean allowNull = JSONUtilities.getBoolean(columnOptions, "allowNull", true); + String defaultValue = JSONUtilities.getString(columnOptions, "defaultValue", null); + logger.info("allowNull::{}" , allowNull); + + String allowNullStr = "NULL"; + if(!allowNull) { + allowNullStr = "NOT NULL"; + } + + + if (name != null) { + if(trimColNames) { + String trimmedCol = name.replaceAll("\\s", ""); + createSB.append( trimmedCol + " "); + }else{ + createSB.append(name + " "); + } + + if (type.equals(SqlData.SQL_TYPE_VARCHAR)) { + if (size.isEmpty()) { + size = "255"; + } + createSB.append(type + "(" + size + ")"); + + } else if (type.equals(SqlData.SQL_TYPE_CHAR)) { + if (size.isEmpty()) { + size = "10"; + } + createSB.append(type + "(" + size + ")"); + + } else if (type.equals(SqlData.SQL_TYPE_INT) || type.equals(SqlData.SQL_TYPE_INTEGER)) { + if (size.isEmpty()) { + createSB.append(type); + } else { + createSB.append(type + "(" + size + ")"); + } + + } else if (type.equals(SqlData.SQL_TYPE_NUMERIC)) { + if (size.isEmpty()) { + createSB.append(type); + } else { + createSB.append(type + "(" + size + ")"); + } + } else { + createSB.append(type); + } + + createSB.append(" " + allowNullStr); + if(defaultValue != null && !defaultValue.isEmpty()) { + if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) { + createSB.append(" DEFAULT " + "'" + defaultValue + "'"); + }else { + try { + Integer.parseInt(defaultValue); + }catch(NumberFormatException nfe) { + throw new SqlExporterException(defaultValue + " is not compatible with column type :" + type); + } + createSB.append(" DEFAULT " + defaultValue); + } + + } + + if (i < count - 1) { + createSB.append(","); + } + createSB.append("\n"); + } + } + } + + StringBuffer sql = new StringBuffer(); + + boolean includeDrop = JSONUtilities.getBoolean(options, "includeDropStatement", false); + boolean addIfExist = options == null ? false : JSONUtilities.getBoolean(options, "includeIfExistWithDropStatement", true); + if (includeDrop) { + if(addIfExist) { + sql.append("DROP TABLE IF EXISTS " + table + ";\n"); + }else { + sql.append("DROP TABLE " + table + ";\n"); + } + + } + + sql.append("CREATE TABLE ").append(table); + sql.append(" (").append("\n"); + sql.append(createSB.toString()); + sql.append(")").append(";" + "\n"); + + String createSQL = sql.toString(); + if(logger.isDebugEnabled()){ + logger.debug("Create SQL Generated Successfully...{}", createSQL); + } + return createSQL; + } + + + +} diff --git a/main/src/com/google/refine/exporters/sql/SqlData.java b/main/src/com/google/refine/exporters/sql/SqlData.java new file mode 100755 index 000000000..b72921701 --- /dev/null +++ b/main/src/com/google/refine/exporters/sql/SqlData.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2018, Tony Opara + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.exporters.sql; + + +public class SqlData { + + private String columnName; + private Object value; + private String text; + public static final String SQL_TYPE_VARCHAR = "VARCHAR"; + public static final String SQL_TYPE_CHAR = "CHAR"; + public static final String SQL_TYPE_TEXT = "TEXT"; + public static final String SQL_TYPE_INTEGER = "INTEGER"; + public static final String SQL_TYPE_INT = "INT"; + public static final String SQL_TYPE_NUMERIC = "NUMERIC"; + public static final String SQL_TYPE_DATE = "DATE"; + public static final String SQL_TYPE_TIMESTAMP = "TIMESTAMP"; + + + + public SqlData(String columnName, Object value, String text) { + this.columnName = columnName; + this.value = value; + this.text = text; + + } + + + public String getColumnName() { + return columnName; + } + + + public Object getValue() { + return value; + } + + + public String getText() { + return text; + } + + + @Override + public String toString() { + return "SqlData [columnName=" + columnName + ", value=" + value + ", text=" + text + "]"; + } + +} diff --git a/main/src/com/google/refine/exporters/sql/SqlExporter.java b/main/src/com/google/refine/exporters/sql/SqlExporter.java new file mode 100755 index 000000000..808cf45e7 --- /dev/null +++ b/main/src/com/google/refine/exporters/sql/SqlExporter.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2018, Tony Opara + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.exporters.sql; + +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.refine.ProjectManager; +import com.google.refine.browsing.Engine; +import com.google.refine.exporters.CustomizableTabularExporterUtilities; +import com.google.refine.exporters.TabularSerializer; +import com.google.refine.exporters.WriterExporter; +import com.google.refine.model.Project; +import com.google.refine.util.JSONUtilities; + +public class SqlExporter implements WriterExporter { + + private static final Logger logger = LoggerFactory.getLogger("SqlExporter"); + public static final String NO_COL_SELECTED_ERROR = "****NO COLUMNS SELECTED****"; + public static final String NO_OPTIONS_PRESENT_ERROR = "****NO OPTIONS PRESENT****"; + //JSON Property names + public static final String JSON_INCLUDE_STRUCTURE = "includeStructure"; + public static final String JSON_INCLUDE_CONTENT = "includeContent"; + public static final String JSON_TABLE_NAME = "tableName"; + + private List columnNames = new ArrayList(); + private List> sqlDataList = new ArrayList>(); + private JSONObject sqlOptions; + + + @Override + public String getContentType() { + return "text/plain"; + } + + @Override + public void export(final Project project, Properties params, Engine engine, final Writer writer) + throws IOException { + if(logger.isDebugEnabled()) { + logger.debug("export sql with params: {}", params); + } + + TabularSerializer serializer = new TabularSerializer() { + + @Override + public void startFile(JSONObject options) { + sqlOptions = options; + //logger.info("setting options::{}", sqlOptions); + } + + @Override + public void endFile() { + try { + if (columnNames.isEmpty()) { + logger.error("No Columns Selected!!"); + throw new SqlExporterException(NO_COL_SELECTED_ERROR); + + } + if (sqlOptions == null) { + logger.error("No Options Selected!!"); + throw new SqlExporterException(NO_OPTIONS_PRESENT_ERROR); + + } + String tableName = ProjectManager.singleton.getProjectMetadata(project.id).getName(); + + Object tableNameManual = sqlOptions.get(JSON_TABLE_NAME); + + if (tableNameManual != null && !tableNameManual.toString().isEmpty()) { + tableName = tableNameManual.toString(); + } + + SqlCreateBuilder createBuilder = new SqlCreateBuilder(tableName, columnNames, sqlOptions); + SqlInsertBuilder insertBuilder = new SqlInsertBuilder(tableName, columnNames, sqlDataList, + sqlOptions); + + final boolean includeStructure = sqlOptions == null ? true + : JSONUtilities.getBoolean(sqlOptions, JSON_INCLUDE_STRUCTURE, true); + + final boolean includeContent = sqlOptions == null ? true + : JSONUtilities.getBoolean(sqlOptions, JSON_INCLUDE_CONTENT, true); + + if (includeStructure) { + String sqlCreateStr = createBuilder.getCreateSQL(); + writer.write(sqlCreateStr); + } + + if (includeContent) { + String sqlInsertStr = insertBuilder.getInsertSQL(); + writer.write(sqlInsertStr); + } + + if (logger.isDebugEnabled()) { + logger.debug("sqlOptions::{}", sqlOptions); + } + + columnNames = new ArrayList(); + sqlDataList = new ArrayList>(); + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public void addRow(List cells, boolean isHeader) { + + if (isHeader) { + for (CellData cellData : cells) { + columnNames.add(cellData.text); + } + + } else { + ArrayList values = new ArrayList<>(); + for (CellData cellData : cells) { + + if (cellData != null) { + if(cellData.text == null || cellData.text.isEmpty()) { + values.add(new SqlData(cellData.columnName, "", "")); + }else { + values.add(new SqlData(cellData.columnName, cellData.value, cellData.text)); + } + + } + + } + sqlDataList.add(values); + } + + } + }; + + CustomizableTabularExporterUtilities.exportRows(project, engine, params, serializer); + } + + +} diff --git a/main/src/com/google/refine/exporters/sql/SqlExporterException.java b/main/src/com/google/refine/exporters/sql/SqlExporterException.java new file mode 100644 index 000000000..39bdf2cd6 --- /dev/null +++ b/main/src/com/google/refine/exporters/sql/SqlExporterException.java @@ -0,0 +1,16 @@ +package com.google.refine.exporters.sql; + + +public class SqlExporterException extends RuntimeException { + + /** + * + */ + private static final long serialVersionUID = -2640105469265805010L; + + public SqlExporterException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + +} diff --git a/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java new file mode 100755 index 000000000..d526a968d --- /dev/null +++ b/main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2018, Tony Opara + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.exporters.sql; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.refine.util.JSONUtilities; + +public class SqlInsertBuilder { + + private static final Logger logger = LoggerFactory.getLogger("SQLInsertBuilder"); + + private String table; + + private List columns; + + private List> sqlDataList; + + private JSONObject options; + + + /** + * + * @param table + * @param columns + * @param rows + * @param options + * @param sqlErrors + */ + public SqlInsertBuilder(String table, List columns, List> rows, JSONObject options + ) { + this.table = table; + this.columns = columns; + this.sqlDataList = rows; + this.options = options; + //logger.info("Column Size:{}", columns.size()); + + } + + /** + * Get Insert Sql + * @return + */ + public String getInsertSQL(){ + if(logger.isDebugEnabled()) { + logger.debug("Insert SQL with columns: {}", columns); + } + + JSONArray colOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns"); + Map colOptionsMap = new HashMap(); + if(colOptionArray != null) { + colOptionArray.forEach(c -> { + JSONObject json = (JSONObject)c; + colOptionsMap.put("" + json.get("name"), json); + }); + } + + boolean nullValueNull = options == null ? true : JSONUtilities.getBoolean(options, "convertNulltoEmptyString", true); + + StringBuffer values = new StringBuffer(); + + int idx = 0; + for(ArrayList sqlRow : sqlDataList) { + StringBuilder rowValue = new StringBuilder(); + + //int fieldCount = 0; + for(SqlData val : sqlRow) { + + JSONObject jsonOb = colOptionsMap.get(val.getColumnName()); + String type = (String)jsonOb.get("type"); + + String defaultValue = JSONUtilities.getString(jsonOb, "defaultValue", null); + + boolean allowNullChkBox = JSONUtilities.getBoolean(jsonOb, "defaultValue", true);; + if(type == null) { + type = SqlData.SQL_TYPE_VARCHAR; + } + //Character Types + if(type.equals(SqlData.SQL_TYPE_VARCHAR) || type.equals(SqlData.SQL_TYPE_CHAR) || type.equals(SqlData.SQL_TYPE_TEXT)) { + + if((val.getText() == null || val.getText().isEmpty()) ) { + + handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, true); + + }else { + rowValue.append("'" + val.getText() + "'"); + + } + + }else if(type.equals(SqlData.SQL_TYPE_INT) || type.equals(SqlData.SQL_TYPE_INTEGER) || type.equals(SqlData.SQL_TYPE_NUMERIC)) {//Numeric Types : INT, NUMERIC + + if((val.getText() == null || val.getText().isEmpty())) { + + handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, false); + + }else {//value not null + + try { + Integer.parseInt(val.getText()); + } catch (NumberFormatException nfe) { + throw new SqlExporterException( + val.getText() + " is not compatible with column type :" + type); + } + + rowValue.append(val.getText()); + + } + + }else if(type.equals(SqlData.SQL_TYPE_DATE) || type.equals(SqlData.SQL_TYPE_TIMESTAMP)) { + if((val.getText() == null || val.getText().isEmpty())) { + handleNullField(allowNullChkBox, defaultValue, nullValueNull, val.getColumnName(), rowValue, true); + }else { + rowValue.append("'" + val.getText() + "'"); + } + } + + rowValue.append(","); + + } + + + + idx++; + String rowValString = rowValue.toString(); +// logger.info("rowValString::" + rowValString); + rowValString = rowValString.substring(0, rowValString.length() - 1); + + values.append("( "); + values.append(rowValString); + values.append(" )"); + if(idx < sqlDataList.size()) { + values.append(","); + } + values.append("\n"); + + } + + boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false); + String colNamesWithSep = columns.stream().map(col -> col.replaceAll("\\s", "")).collect(Collectors.joining(","));; + if(!trimColNames) { + colNamesWithSep = columns.stream().collect(Collectors.joining(",")); + } + + String valuesString = values.toString(); + valuesString = valuesString.substring(0, valuesString.length() - 1); + + + StringBuffer sql = new StringBuffer(); + + sql.append("INSERT INTO ").append(table); + sql.append(" ("); + sql.append(colNamesWithSep); + sql.append(") VALUES ").append("\n"); + sql.append(valuesString); + + String sqlString = sql.toString(); + if(logger.isDebugEnabled()) { + logger.debug("Insert Statement Generated Successfully...{}", sqlString); + } + return sqlString; + } + + /** + * + * @param allowNullChkBox + * @param defaultValue + * @param nullValueNull + * @param col + * @param rowValue + * @param quote + * @param fieldCount + */ + public void handleNullField( + boolean allowNullChkBox, + String defaultValue, + boolean nullValueNull, + String col, + StringBuilder rowValue, + boolean quote + ) { + + if(allowNullChkBox) {//cell nullable + if(defaultValue != null && !defaultValue.isEmpty()) { + if(quote) { + rowValue.append("'" + defaultValue + "'"); + }else { + rowValue.append(defaultValue); + } + + }else { + if(nullValueNull) { + rowValue.append("null"); + + }else { + throw new SqlExporterException("Null value not allowed for Field :" + col); + } + + } + + }else { + if(defaultValue != null && !defaultValue.isEmpty()) { + if(quote) { + rowValue.append("'" + defaultValue + "'"); + }else { + rowValue.append(defaultValue); + } + + }else { + throw new SqlExporterException("Null value not allowed for Field :" + col); + } + + } + + } + + +} diff --git a/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java b/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java new file mode 100644 index 000000000..7ddf0e4b5 --- /dev/null +++ b/main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java @@ -0,0 +1,457 @@ +/* + +Copyright 2018, Tony Opara. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.tests.exporters.sql; + +import static org.junit.Assert.assertNotEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.List; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import com.google.refine.ProjectManager; +import com.google.refine.browsing.Engine; +import com.google.refine.exporters.sql.SqlCreateBuilder; +import com.google.refine.exporters.sql.SqlExporter; +import com.google.refine.exporters.sql.SqlInsertBuilder; +import com.google.refine.model.Cell; +import com.google.refine.model.Column; +import com.google.refine.model.ModelException; +import com.google.refine.model.Project; +import com.google.refine.model.Row; +import com.google.refine.model.medadata.ProjectMetadata; +import com.google.refine.tests.ProjectManagerStub; +import com.google.refine.tests.RefineTest; + + +public class SqlExporterTests extends RefineTest { + + private static final String TEST_PROJECT_NAME = "SQL_EXPORTER_TEST_PROJECT"; + + @Override + @BeforeTest + public void init() { + logger = LoggerFactory.getLogger(this.getClass()); + } + + //dependencies + StringWriter writer; + ProjectMetadata projectMetadata; + Project project; + Engine engine; + Properties options; + SqlCreateBuilder sqlCreateBuilder; + SqlInsertBuilder sqlInsertBuilder; + + //System Under Test + SqlExporter SUT; + + @BeforeMethod + public void SetUp(){ + SUT = new SqlExporter(); + writer = new StringWriter(); + ProjectManager.singleton = new ProjectManagerStub(); + projectMetadata = new ProjectMetadata(); + project = new Project(); + projectMetadata.setName(TEST_PROJECT_NAME); + ProjectManager.singleton.registerProject(project, projectMetadata); + engine = new Engine(project); + options = mock(Properties.class); + } + + @AfterMethod + public void TearDown(){ + SUT = null; + writer = null; + ProjectManager.singleton.deleteProject(project.id); + project = null; + projectMetadata = null; + engine = null; + options = null; + sqlCreateBuilder = null; + sqlInsertBuilder = null; + } + + @Test + public void testExportSimpleSql(){ + createGrid(2, 2); + String tableName = "sql_table_test"; + String optionsString = createOptionsFromProject(tableName, null,null).toString(); + when(options.getProperty("options")).thenReturn(optionsString); + + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + // logger.info("result = \n" + result); + Assert.assertNotNull(result); + assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR); + boolean checkResult = result.contains("CREATE TABLE " + tableName); + //logger.info("checkResult1 =" + checkResult); + checkResult = result.contains("INSERT INTO " + tableName); + // logger.info("checkResult2 =" + checkResult); + Assert.assertEquals(checkResult, true); + + } + + @Test + public void testExportSqlNoSchema(){ + createGrid(2, 2); + String tableName = "sql_table_test"; + JSONObject optionsJson = createOptionsFromProject(tableName, null,null); + optionsJson.put("includeStructure", false); + when(options.getProperty("options")).thenReturn(optionsJson.toString()); + // logger.info("Options = " + optionsJson.toString()); + + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + //logger.info("result = \n" + result); + Assert.assertNotNull(result); + assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR); + boolean checkResult = result.contains("CREATE TABLE " + tableName); + Assert.assertEquals(checkResult, false); + + checkResult = result.contains("INSERT INTO " + tableName); + Assert.assertEquals(checkResult, true); + + } + + @Test + public void testExportSqlNoContent(){ + createGrid(2, 2); + String tableName = "sql_table_test"; + JSONObject optionsJson = createOptionsFromProject(tableName, null, null); + optionsJson.put("includeContent", false); + when(options.getProperty("options")).thenReturn(optionsJson.toString()); + //logger.info("Options = " + optionsJson.toString()); + + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + // logger.info("result = \n" + result); + Assert.assertNotNull(result); + assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR); + boolean checkResult = result.contains("CREATE TABLE " + tableName); + Assert.assertEquals(checkResult, true); + + checkResult = result.contains("INSERT INTO " + tableName); + Assert.assertEquals(checkResult, false); + + } + + @Test + public void testExportSqlIncludeSchemaWithDropStmt(){ + createGrid(2, 2); + String tableName = "sql_table_test"; + JSONObject optionsJson = createOptionsFromProject(tableName, null, null); + optionsJson.put("includeStructure", true); + optionsJson.put("includeDropStatement", true); + + when(options.getProperty("options")).thenReturn(optionsJson.toString()); + //logger.info("Options = " + optionsJson.toString()); + + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + //logger.info("result = " + result); + + Assert.assertNotNull(result); +// assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR); +// assertNotEquals(writer.toString(), SqlExporter.NO_COL_SELECTED_ERROR); + + boolean checkResult = result.contains("CREATE TABLE " + tableName); + Assert.assertEquals(checkResult, true); + //logger.info("checkResult1 = " + checkResult); + + checkResult = result.contains("INSERT INTO " + tableName ); + Assert.assertEquals(checkResult, true); + //logger.info("checkResult2 = " + checkResult); + + checkResult = result.contains("DROP TABLE IF EXISTS " + tableName + ";"); + Assert.assertEquals(checkResult, true); + //logger.info("checkResult3 = " + checkResult); + + } + + @Test + public void testGetCreateSql(){ + createGrid(3,3); + String tableName = "sql_table_test"; + String type = "CHAR"; + String size = "2"; + JSONObject optionsJson = createOptionsFromProject(tableName, type, size); + // logger.info("Options:: = " + optionsJson.toString()); + List columns = project.columnModel.columns.stream().map(col -> col.getName()).collect(Collectors.toList()); + + sqlCreateBuilder = new SqlCreateBuilder(tableName, columns, optionsJson); + String createSql = sqlCreateBuilder.getCreateSQL(); + //logger.info("createSql = \n" + createSql); + Assert.assertNotNull(createSql); + boolean result = createSql.contains(type + "(" + size + ")"); + Assert.assertEquals(result, true); + + } + + @Test + public void testExportSqlWithNullFields(){ + int inNull = 8; + createGridWithNullFields(3, 3, inNull); + String tableName = "sql_table_test"; + JSONObject optionsJson = createOptionsFromProject(tableName, null, null); + optionsJson.put("includeStructure", true); + optionsJson.put("includeDropStatement", true); + optionsJson.put("convertNulltoEmptyString", true); + + + when(options.getProperty("options")).thenReturn(optionsJson.toString()); + //logger.info("Options = " + optionsJson.toString()); + + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + Assert.assertNotNull(result); + //logger.info("\nresult = " + result); + // logger.info("\nNull Count:" + countWordInString(result, "null")); + + int countNull = countWordInString(result, "null"); + Assert.assertEquals(countNull, inNull); + + } + + @Test + public void testExportSqlWithNotNullColumns(){ + int noOfCols = 4; + int noOfRows = 3; + createGrid(noOfRows, noOfCols); + String tableName = "sql_table_test"; + JSONObject optionsJson = createOptionsFromProject(tableName, null, null, null, false); + optionsJson.put("includeStructure", true); + optionsJson.put("includeDropStatement", true); + optionsJson.put("convertNulltoEmptyString", true); + + when(options.getProperty("options")).thenReturn(optionsJson.toString()); + try { + SUT.export(project, options, engine, writer); + } catch (IOException e) { + Assert.fail(); + } + + String result = writer.toString(); + logger.info("\nresult:={} ", result); + Assert.assertNotNull(result); + + int countNull = countWordInString(result, "NOT NULL"); + logger.info("\nNot Null Count: {}" , countNull); + Assert.assertEquals(countNull, noOfCols); + + } + + //helper methods + + public int countWordInString(String input, String word){ + if(input == null || input.isEmpty()) { + return 0; + } + int i = 0; + Pattern p = Pattern.compile(word); + Matcher m = p.matcher( input ); + while (m.find()) { + i++; + } + + return i; + + } + + protected void createColumns(int noOfColumns){ + for(int i = 0; i < noOfColumns; i++){ + try { + project.columnModel.addColumn(i, new Column(i, "column" + i), true); + } catch (ModelException e1) { + Assert.fail("Could not create column"); + } + } + } + + protected void createGrid(int noOfRows, int noOfColumns){ + createColumns(noOfColumns); + + for(int i = 0; i < noOfRows; i++){ + Row row = new Row(noOfColumns); + for(int j = 0; j < noOfColumns; j++){ + row.cells.add(new Cell("row" + i + "cell" + j, null)); + } + project.rows.add(row); + } + } + + protected void createGridWithNullFields(int noOfRows, int noOfColumns, int noOfNullFields){ + createColumns(noOfColumns); + if(noOfNullFields > (noOfColumns * noOfRows)) { + noOfNullFields = noOfColumns * noOfRows; + } + int k = 0; + for(int i = 0; i < noOfRows; i++){ + Row row = new Row(noOfColumns); + for(int j = 0; j < noOfColumns; j++){ + if(k < noOfNullFields) { + row.cells.add(new Cell("", null)); + k++; + }else { + row.cells.add(new Cell("row" + i + "cell" + j, null)); + } + + } + project.rows.add(row); + } + } + + protected JSONObject createOptionsFromProject(String tableName, String type, String size) { + + JSONObject json = new JSONObject(); + JSONArray columns = new JSONArray(); + json.put("columns", columns); + json.put("tableName", tableName); + + List cols = project.columnModel.columns; + + cols.forEach(c -> { + //logger.info("Column Name = " + c.getName()); + JSONObject columnModel = new JSONObject(); + columnModel.put("name", c.getName()); + if(type != null) { + columnModel.put("type", type); + }else { + columnModel.put("type", "VARCHAR"); + } + if(size != null) { + columnModel.put("size", size); + }else { + columnModel.put("size", "100"); + } + + if(type != null) { + columnModel.put("type", type); + } + if(size != null) { + // logger.info(" Size = " + size); + columnModel.put("size", size); + } + + columns.put(columnModel); + + }); + + return json; + } + + protected JSONObject createOptionsFromProject(String tableName, String type, String size, String defaultValue, + boolean allowNull) { + + JSONObject json = new JSONObject(); + JSONArray columns = new JSONArray(); + json.put("columns", columns); + json.put("tableName", tableName); + + List cols = project.columnModel.columns; + + cols.forEach(c -> { + //logger.info("Column Name = " + c.getName()); + JSONObject columnModel = new JSONObject(); + columnModel.put("name", c.getName()); + if(type != null) { + columnModel.put("type", type); + }else { + columnModel.put("type", "VARCHAR"); + } + if(size != null) { + columnModel.put("size", size); + }else { + columnModel.put("size", "100"); + } + + if(type != null) { + columnModel.put("type", type); + } + if(size != null) { + // logger.info(" Size = " + size); + columnModel.put("size", size); + } + + columnModel.put("defaultValue", defaultValue); + columnModel.put("allowNull", allowNull); + + columns.put(columnModel); + + }); + + return json; + } + + +} diff --git a/main/webapp/modules/core/MOD-INF/controller.js b/main/webapp/modules/core/MOD-INF/controller.js index c37ec5723..be9e4a060 100644 --- a/main/webapp/modules/core/MOD-INF/controller.js +++ b/main/webapp/modules/core/MOD-INF/controller.js @@ -151,6 +151,7 @@ function registerCommands() { RS.registerCommand(module, "authorize", new Packages.com.google.refine.commands.auth.AuthorizeCommand()); RS.registerCommand(module, "deauthorize", new Packages.com.google.refine.commands.auth.DeAuthorizeCommand()); + } function registerOperations() { @@ -468,6 +469,7 @@ function init() { "scripts/dialogs/templating-exporter-dialog.js", "scripts/dialogs/column-reordering-dialog.js", "scripts/dialogs/custom-tabular-exporter-dialog.js", + "scripts/dialogs/sql-exporter-dialog.js", "scripts/dialogs/expression-column-dialog.js", "scripts/project/edit-general-metadata-dialog.js", "scripts/dialogs/http-headers-dialog.js", @@ -507,7 +509,7 @@ function init() { "styles/dialogs/scatterplot-dialog.less", "styles/dialogs/column-reordering-dialog.less", "styles/dialogs/custom-tabular-exporter-dialog.less", - + "styles/dialogs/sql-exporter-dialog.less", "styles/reconciliation/recon-dialog.less", "styles/reconciliation/standard-service-panel.less", "styles/reconciliation/extend-data-preview-dialog.less", diff --git a/main/webapp/modules/core/langs/translation-en.json b/main/webapp/modules/core/langs/translation-en.json index c94a65b2d..34174a64e 100644 --- a/main/webapp/modules/core/langs/translation-en.json +++ b/main/webapp/modules/core/langs/translation-en.json @@ -237,7 +237,7 @@ "help": "Help", "opt-code-applied": "Option code successfully applied.", "error-apply-code": "Error applying option code", - "custom-tab-exp": "Custom Tabular Exporter", + "custom-tab-exp": "SQL Exporter", "content": "Content", "download": "Download", "upload": "Upload", @@ -274,6 +274,18 @@ "line-sep": "Line separator", "upload-to": "Upload to", "json-text": "The following JSON text encodes the options you have set in the other tabs. You can copy it out and save it for later, and paste it back in and click Apply to re-use the same options.", + "columnType": "SQL Type", + "for-include-structure-checkbox": "Include Schema", + "for-include-drop-statement-checkbox": "Include Drop Statement", + "for-include-content-checkbox": "Include Content", + "tableNameLabel": "Table Name:", + "sqlExporterTrimColumns": "Trim Column Names", + "sqlExporterIgnoreFacets": "Ignore facets and filters and export all rows", + "sqlExporterOutputEmptyRows":"Output empty row (i.e. all cells null)", + "for-include-if-exist-drop-stmt-checkbox": "Include 'If Exists' in Drop Statement", + "for-null-cell-value-to-empty-str-label": "Convert Null Value to null in Insert" + + "choose-export-destination": "Please choose the destination for project export", "export-to-local": "Export to local", "export-to-google-drive": "Export to Google Drive" @@ -345,6 +357,7 @@ "excel-xml": "Excel 2007+ (.xlsx)", "odf": "ODF spreadsheet", "custom-tabular": "Custom tabular exporter...", + "sql-export": "SQL Exporter...", "templating": "Templating...", "warning-align": "You haven't done any schema alignment yet,\n so there is no triple to export.\n\n Use the Freebase > Edit Schema Alignment Skeleton...\n command to align your data with Freebase schemas first.", "json-invalid": "The JSON you pasted is invalid", diff --git a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html new file mode 100755 index 000000000..37c20803f --- /dev/null +++ b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.html @@ -0,0 +1,131 @@ +
+
+
+
+
    +
    • +
    • +
+ +
+
+ + + + + + + + + + + +
+ + +
+
+ + + + + + + + + + + + + +
Field NameSQL TypeSize Allow Null Default
+
+ +
+
+ + +
+ + + + + + + + + + + + +
+ + + + + + +
+
+
+ +
+
\ No newline at end of file diff --git a/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js new file mode 100755 index 000000000..3bea81a01 --- /dev/null +++ b/main/webapp/modules/core/scripts/dialogs/sql-exporter-dialog.js @@ -0,0 +1,495 @@ +/* + * Copyright (c) 2017, Tony Opara + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +function SqlExporterDialog(options) { + options = options || { + format: 'sql', + encoding: 'UTF-8', + outputBlankRows: false, + columns: null + }; + + this._columnOptionMap = {}; + this._createDialog(options); + } + + + SqlExporterDialog.uploadTargets = []; + + SqlExporterDialog.prototype._createDialog = function(options) { + var self = this; + + this._dialog = $(DOM.loadHTML("core", "scripts/dialogs/sql-exporter-dialog.html")); + this._elmts = DOM.bind(this._dialog); + this._level = DialogSystem.showDialog(this._dialog); + this._elmts.dialogHeader.html($.i18n._('core-dialogs')["custom-tab-exp"]); + this._elmts.or_dialog_content.html($.i18n._('core-dialogs')["content"]); + this._elmts.or_dialog_download.html($.i18n._('core-dialogs')["download"]); + + this._elmts.selectAllButton.html($.i18n._('core-buttons')["select-all"]); + this._elmts.deselectAllButton.html($.i18n._('core-buttons')["deselect-all"]); + + this._elmts.downloadPreviewButton.html($.i18n._('core-buttons')["preview"]); + this._elmts.downloadButton.html($.i18n._('core-buttons')["download"]); + + this._elmts.cancelButton.html($.i18n._('core-buttons')["cancel"]); +// this._elmts.nextButton.html($.i18n._('core-buttons')["next"]); + + this._elmts.tableNameLabel.html($.i18n._('core-dialogs')["tableNameLabel"]); + this._elmts.includeStructureLabel.html($.i18n._('core-dialogs')["for-include-structure-checkbox"]); + this._elmts.includeDropStatementLabel.html($.i18n._('core-dialogs')["for-include-drop-statement-checkbox"]); + this._elmts.includeContentLabel.html($.i18n._('core-dialogs')["for-include-content-checkbox"]); + this._elmts.includeIfExistDropStatementLabel.html($.i18n._('core-dialogs')["for-include-if-exist-drop-stmt-checkbox"]); + + this._elmts.nullCellValueToEmptyStringLabel.html($.i18n._('core-dialogs')["for-null-cell-value-to-empty-str-label"]); + + this._elmts.sqlExportIgnoreFacetsLabel.html($.i18n._('core-dialogs')["sqlExporterIgnoreFacets"]); + this._elmts.sqlExportTrimAllColumnsLabel.html($.i18n._('core-dialogs')["sqlExporterTrimColumns"]); + this._elmts.sqlExportOutputEmptyRowsLabel.html($.i18n._('core-dialogs')["sqlExporterOutputEmptyRows"]); + + $("#sql-exporter-tabs-content").css("display", ""); + $("#sql-exporter-tabs-download").css("display", ""); + $("#sql-exporter-tabs").tabs(); + + for (var i = 0; i < theProject.columnModel.columns.length; i++) { + var column = theProject.columnModel.columns[i]; + var name = column.name; + var rowId = "sql-exporter-dialog-row" + i; + var selectBoxName = 'selectBoxRow' + i; + var sizeInputName = 'sizeInputRow' + i; + var applyAllBtnName = 'applyAllBtn' + i; + + var allowNullChkBoxName = 'allowNullChkBox' + i; + var defaultValueTextBoxName = 'defaultValueTextBox' + i; + + + var sel = $('') + .attr('type', 'checkbox') + .attr('checked', 'checked') + .addClass("columnNameCheckboxStyle") + .appendTo(columnCell); + $('') + .text(name) + .appendTo(columnCell); + + var typeCell = $('') + .attr('width', '150px') + .appendTo(row); + sel.appendTo(typeCell); + + var sizeCell = $('') + .attr('width', '20px') + .appendTo(row); + $('') + .attr('type', 'text') + .attr('size', '8px') + .attr('id', sizeInputName) + .addClass("sql-exporter-dialog-input") + .appendTo(sizeCell); + + var applyAllCell = $('') + .attr('width', '60px') + .appendTo(row); + $('') + .attr('type', 'button') + .attr('value', 'Apply All') + .attr('id', applyAllBtnName) + .attr("rowIndex", i) + .appendTo(applyAllCell); + + var nullableCell = $('') + .attr('width', '90px') + .addClass("allowNullCellStyle") + .appendTo(row); + $('') + .attr('type', 'checkbox') + .attr('checked', 'checked') + .attr('id', allowNullChkBoxName) + .attr("rowIndex", i) + .addClass("allowNullCheckboxStyle") + .appendTo(nullableCell); + + var defValueCell = $('') + .attr('width', '30px') + .appendTo(row); + $('') + .attr('type', 'text') + .attr('size', '8px') + .attr('id', defaultValueTextBoxName) + .attr("rowIndex", i) + .addClass("defaultValueTextBoxStyle") + .appendTo(defValueCell); + + + + $('#' + applyAllBtnName).on('click', function() { + var rowIndex = this.getAttribute('rowIndex'); + var typeValue = $("#selectBoxRow" + rowIndex).val(); + var sizeValue = $("#sizeInputRow" + rowIndex).val(); + + $('select.typeSelectClass').each(function() { + //alert("Value:" + this.value + " RowIndex:" + rowIndex + " TypeValue:" + typeValue + "" + this.value); + var rowId = this.getAttribute('rowIndex'); + var id = this.getAttribute('id'); + if(rowIndex !== rowId){ + $("#" + id).val(typeValue); + } + + }); + $('input.sql-exporter-dialog-input').each(function() { + var rowId = this.getAttribute('rowIndex'); + var id = this.getAttribute('id'); + if(rowIndex !== rowId){ + $("#" + id).val(sizeValue); + } + + }); + + }); + + $('#' + allowNullChkBoxName).on('click', function() { + var rowIndex = this.getAttribute('rowIndex'); + var id = this.getAttribute('id'); + var checked = $(this).is(':checked');; + + if(checked == false){ + $('#defaultValueTextBox'+ rowIndex).prop("disabled", false); + }else{ + $('#defaultValueTextBox'+ rowIndex).val(""); + $('#defaultValueTextBox'+ rowIndex).prop("disabled", true); + } + }); + + this._columnOptionMap[name] = { + name: name, + type: '', + size: '' + + }; + } + + this._elmts.allowNullToggleCheckbox.click(function() { + var checked = $(this).is(':checked'); + if(checked == true){ + $("input:checkbox[class=allowNullCheckboxStyle]").each(function () { + $(this).attr('checked', true); + }); + $("input:text[class=defaultValueTextBoxStyle]").each(function () { + $(this).attr('disabled', true); + }); + + }else{ + $("input:checkbox[class=allowNullCheckboxStyle]").each(function () { + $(this).attr('checked', false); + }); + $("input:text[class=defaultValueTextBoxStyle]").each(function () { + $(this).attr('disabled', false); + }); + } + + }); + + this._elmts.selectAllButton.click(function() { + $("input:checkbox[class=columnNameCheckboxStyle]").each(function () { + $(this).attr('checked', true); + }); + self._updateOptionCode(); + }); + this._elmts.deselectAllButton.click(function() { + $("input:checkbox[class=columnNameCheckboxStyle]").each(function () { + $(this).attr('checked', false); + }); + self._updateOptionCode(); + }); + + this._elmts.includeStructureCheckbox.click(function() { + var checked = $(this).is(':checked'); + //alert('checked ' + checked); + if(checked == true){ + $('#includeDropStatementCheckboxId').removeAttr("disabled"); + $('#includeIfExistDropStatementCheckboxId').removeAttr("disabled"); + }else{ + $('#includeDropStatementCheckboxId').attr("disabled", true); + $('#includeIfExistDropStatementCheckboxId').attr("disabled", true); + } + }); + + this._elmts.includeContentCheckbox.click(function() { + var checked = $(this).is(':checked'); + if(checked == true){ + $('#nullCellValueToEmptyStringCheckboxId').removeAttr("disabled"); + }else{ + $('#nullCellValueToEmptyStringCheckboxId').attr("disabled", true); + + } + }); + + + this._elmts.cancelButton.click(function() { self._dismiss(); }); + this._elmts.downloadButton.click(function() { self._download(); }); + this._elmts.downloadPreviewButton.click(function(evt) { self._previewDownload(); }); + + this._configureUIFromOptionCode(options); + this._updateOptionCode(); + }; + + SqlExporterDialog.prototype._configureUIFromOptionCode = function(options) { + + this._elmts.tableNameTextBox.val(theProject.metadata.name); + this._elmts.sqlExportOutputEmptyRowsCheckbox.attr('checked', 'checked'); + this._elmts.sqlExportTrimAllColumnsCheckbox.attr('checked', 'checked'); + this._elmts.nullCellValueToEmptyStringLabel.attr('checked', 'checked'); + + $("input:text[class=defaultValueTextBoxStyle]").each(function () { + $(this).prop("disabled", true); + }); + + + }; + + SqlExporterDialog.prototype._dismiss = function() { + DialogSystem.dismissUntil(this._level - 1); + }; + + SqlExporterDialog.prototype._previewDownload = function() { + this._postExport(true); + }; + + SqlExporterDialog.prototype._download = function() { + var result = this._postExport(false); + // alert("result::" + result); + if(result == true){ + this._dismiss(); + } + + }; + + SqlExporterDialog.prototype._postExport = function(preview) { + var self = this; + var exportAllRowsCheckbox = this._elmts.sqlExportAllRowsCheckbox[0].checked; + var options = this._getOptionCode(); + + if(options.columns == null || options.columns.length == 0){ + alert("Please select at least one column..."); + return false; + } + + var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-'); + + var format = options.format; + var encoding = options.encoding; + + delete options.format; + delete options.encoding; + if (preview) { + options.limit = 10; + } + + // var ext = SqlExporterDialog.formats[format].extension; + var form = self._prepareSqlExportRowsForm(format, !exportAllRowsCheckbox, "sql"); + $('') + .attr("name", "options") + .attr("value", JSON.stringify(options)) + .appendTo(form); + if (encoding) { + $('') + .attr("name", "encoding") + .attr("value", encoding) + .appendTo(form); + } + if (!preview) { + $('') + .attr("name", "contentType") + .attr("value", "application/x-unknown") // force download + .appendTo(form); + } + + // alert("form::" + form); + document.body.appendChild(form); + + window.open("about:blank", "refine-export"); + form.submit(); + + document.body.removeChild(form); + return true; + + }; + + SqlExporterDialog.prototype._prepareSqlExportRowsForm = function(format, includeEngine, ext) { + var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-'); + var form = document.createElement("form"); + $(form) + .css("display", "none") + .attr("method", "post") + .attr("action", "command/core/export-rows/" + name + ((ext) ? ("." + ext) : "")) + .attr("target", "refine-export"); + + $('') + .attr("name", "project") + .attr("value", theProject.id) + .appendTo(form); + $('') + .attr("name", "format") + .attr("value", format) + .appendTo(form); + if (includeEngine) { + $('') + .attr("name", "engine") + .attr("value", JSON.stringify(ui.browsingEngine.getJSON())) + .appendTo(form); + } + + return form; + }; + + SqlExporterDialog.prototype._selectColumn = function(columnName) { + + this._elmts.columnNameSpan.text(columnName); + var columnOptions = this._columnOptionMap[columnName]; + alert("in _selectColumn:column type::" + columnOptions.type); + + }; + + SqlExporterDialog.prototype._updateCurrentColumnOptions = function() { +// var selectedColumnName = this._elmts.columnList.find('.sql-exporter-dialog-column.selected').attr('column'); +// //alert("_updateCurrentColumnOptions::" + selectedColumnName); +// var columnOptions = this._columnOptionMap[selectedColumnName]; +// columnOptions.type= this._elmts.columnOptionPane.find('input[name="sql-exporter-type"]:checked').val(); + + }; + + SqlExporterDialog.prototype._updateOptionCode = function() { + this._elmts.optionCodeInput.val(JSON.stringify(this._getOptionCode(), null, 2)); + }; + + + SqlExporterDialog.prototype._getOptionCode = function() { + var options = { + //format: this._dialog.find('input[name="sql-exporter-download-format"]:checked').val() + }; + var unescapeJavascriptString = function(s) { + try { + return JSON.parse('"' + s + '"'); + } catch (e) { + // We're not handling the case where the user doesn't escape double quotation marks. + return s; + } + }; + + options.format = 'sql'; + options.separator = ';'; + options.encoding = 'UTF-8'; + + options.outputBlankRows = this._elmts.sqlExportOutputEmptyRowsCheckbox[0].checked; + options.includeStructure = this._elmts.includeStructureCheckbox[0].checked; + options.includeDropStatement = this._elmts.includeDropStatementCheckbox[0].checked; + options.includeContent = this._elmts.includeContentCheckbox[0].checked; + options.tableName = $.trim(this._elmts.tableNameTextBox.val().replace(/\W/g, ' ')).replace(/\s+/g, '_'); + options.trimColumnNames = this._elmts.sqlExportTrimAllColumnsCheckbox[0].checked; + + options.convertNulltoEmptyString = this._elmts.nullCellValueToEmptyStringCheckbox[0].checked; + options.includeIfExistWithDropStatement = this._elmts.includeIfExistDropStatementCheckbox[0].checked; + + + options.columns = []; + + var self = this; + this._elmts.columnListTable.find('.sql-exporter-dialog-row').each(function() { + if ($(this).find('input[type="checkbox"]')[0].checked) { + var name = this.getAttribute('column'); + var rowIndex = this.getAttribute('rowIndex'); + + var selectedValue = $('#selectBoxRow' + rowIndex).val(); + + var typeSize = 0; + if(selectedValue === 'VARCHAR' || selectedValue === 'CHAR' || selectedValue === 'INT' || selectedValue === 'NUMERIC'){ + typeSize = $('#sizeInputRow' + rowIndex).val(); + // alert("typeSize::" + typeSize); + } + + var allowNullChkBoxName = $('#allowNullChkBox' + rowIndex).is(':checked'); + var defaultValueTextBoxName = $('#defaultValueTextBox' + rowIndex).val(); + // alert("allowNullChkBoxName::" + allowNullChkBoxName); + // alert("defaultValueTextBoxName::" + defaultValueTextBoxName); + + var fullColumnOptions = self._columnOptionMap[name]; + var columnOptions = { + name: name, + type: selectedValue, + size: typeSize, + allowNull: allowNullChkBoxName, + defaultValue: defaultValueTextBoxName, + nullValueToEmptyStr: options.convertNulltoEmptyString + + }; + + // alert('checked type ' + columnIndex + ' =' + check_value); + + options.columns.push(columnOptions); + } + }); + //alert('options:' + options); + return options; + }; + \ No newline at end of file diff --git a/main/webapp/modules/core/scripts/project/exporters.js b/main/webapp/modules/core/scripts/project/exporters.js index 60d6eb286..a0ac90320 100644 --- a/main/webapp/modules/core/scripts/project/exporters.js +++ b/main/webapp/modules/core/scripts/project/exporters.js @@ -86,6 +86,11 @@ ExporterManager.MenuItems = [ "label": $.i18n._('core-project')["custom-tabular"], "click": function() { new CustomTabularExporterDialog(); } }, + { + "id" : "core/export-sql", + "label": $.i18n._('core-project')["sql-export"], + "click": function() { new SqlExporterDialog(); } + }, { "id" : "core/export-templating", "label": $.i18n._('core-project')["templating"], @@ -131,6 +136,7 @@ ExporterManager.handlers.exportRows = function(format, ext) { ExporterManager.prepareExportRowsForm = function(format, includeEngine, ext) { var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-'); + //alert("name:" + name); var form = document.createElement("form"); $(form) .css("display", "none") diff --git a/main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less b/main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less new file mode 100644 index 000000000..c090bf1dd --- /dev/null +++ b/main/webapp/modules/core/styles/dialogs/sql-exporter-dialog.less @@ -0,0 +1,101 @@ +/* + +Copyright 2010, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +@import-less url("../theme.less"); + +#sql-exporter-tabs .ui-tabs-panel { + padding: @padding_looser; + } + +.sql-exporter-tabs-content-main { + +} +.sql-exporter-columns, .sql-exporter-column-options { + /*border: 1px solid @chrome_primary;*/ + height: 8em; + padding: @padding_loose; + } +.sql-exporter-columns { + min-height: 200px; + overflow: auto; + } +.sql-exporter-selected-column { + font-weight: bold; + border: 1px solid @chrome_primary; + background: @chrome_primary; + padding: @padding_tighter; + .rounded_corners(); + } + +.sql-exporter-dialog-column { + /* border: 1px solid @chrome_primary;*/ + background: @fill_secondary; + padding: @padding_tighter; + margin: @padding_tight; + cursor: move; + .rounded_corners(); + } +.sql-exporter-dialog-column.selected { + background: @chrome_primary; + font-weight: bold; + } + +textarea.sql-exporter-code { + width: 100%; + border: 1px solid @chrome_primary; + height: 25em; + } +.sql-exporter-dialog-row { + margin-top: 2px; + padding-top: 2px; + margin-bottom: 2px; + margin-top: 2px; +} +tr.sql-exporter-dialog-row td { + border:1px dashed lightblue; + margin: 2px; +} +.sql-exporter-dialog-input { + margin:2px; + border: solid 2px lightblue; +} +.defaultValueTextBoxStyle { + margin:2px; +} +.allowNullCellStyle { + text-align: center; + vertical-align: middle; +} +.typeSelectClass { + margin-left:2px; +} \ No newline at end of file