s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Migrate TabularExporter to use JsonNode

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-11-02 14:05:33 +00:00
parent 52426b98a3
commit 2873035ee2
13 changed files with 207 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
extensions/gdata/src/com/google/refine/extension/gdata/FusionTableSerializer.java
View File

@ -4,13 +4,14 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.HttpResponseException;
import com.google.api.services.fusiontables.Fusiontables;
import com.google.refine.exporters.CustomizableTabularExporterConfiguration;
import com.google.refine.exporters.TabularSerializer;
final class FusionTableSerializer implements TabularSerializer {
@ -31,7 +32,7 @@ final class FusionTableSerializer implements TabularSerializer {
}
@Override
public void startFile(JSONObject options) {
public void startFile(JsonNode options) {
}
@Override

6
extensions/gdata/src/com/google/refine/extension/gdata/SpreadsheetSerializer.java
View File

@ -6,10 +6,11 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.api.services.sheets.v4.Sheets;
import com.google.api.services.sheets.v4.model.AppendCellsRequest;
import com.google.api.services.sheets.v4.model.BatchUpdateSpreadsheetRequest;
@ -18,6 +19,7 @@ import com.google.api.services.sheets.v4.model.ExtendedValue;
import com.google.api.services.sheets.v4.model.Request;
import com.google.api.services.sheets.v4.model.RowData;
import com.google.refine.exporters.CustomizableTabularExporterConfiguration;
import com.google.refine.exporters.TabularSerializer;
final class SpreadsheetSerializer implements TabularSerializer {
@ -44,7 +46,7 @@ final class SpreadsheetSerializer implements TabularSerializer {
}
@Override
public void startFile(JSONObject options) {
public void startFile(JsonNode options) {
}

11
main/src/com/google/refine/exporters/CsvExporter.java
View File

@ -38,12 +38,11 @@ import java.io.Writer;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.browsing.Engine;
import com.google.refine.model.Project;
@ -82,14 +81,14 @@ public class CsvExporter implements WriterExporter{
if (optionsString != null) {
try {
options = ParsingUtilities.mapper.readValue(optionsString, Configuration.class);
if (options.separator == null) {
options.separator = Character.toString(separator);
}
} catch (IOException e) {
// Ignore and keep options null.
e.printStackTrace();
}
}
if (options.separator == null) {
options.separator = Character.toString(separator);
}
final String separator = options.separator;
final String lineSeparator = options.lineSeparator;
@ -105,7 +104,7 @@ public class CsvExporter implements WriterExporter{
TabularSerializer serializer = new TabularSerializer() {
@Override
public void startFile(JSONObject options) {
public void startFile(JsonNode options) {
}
@Override

169
main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java
View File

@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.exporters;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.OffsetDateTime;
@ -47,9 +48,12 @@ import java.util.TimeZone;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.validator.routines.UrlValidator;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
@ -65,6 +69,8 @@ import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities;
abstract public class CustomizableTabularExporterUtilities {
final static private String fullIso8601 = "yyyy-MM-dd'T'HH:mm:ss'Z'";
static public void exportRows(
final Project project,
final Engine engine,
@ -72,15 +78,15 @@ abstract public class CustomizableTabularExporterUtilities {
final TabularSerializer serializer) {
String optionsString = (params != null) ? params.getProperty("options") : null;
JSONObject optionsTemp = null;
JsonNode optionsTemp = null;
if (optionsString != null) {
try {
optionsTemp = ParsingUtilities.evaluateJsonStringToObject(optionsString);
} catch (JSONException e) {
optionsTemp = ParsingUtilities.mapper.readTree(optionsString);
} catch (IOException e) {
// Ignore and keep options null.
}
}
final JSONObject options = optionsTemp;
final JsonNode options = optionsTemp;
final boolean outputColumnHeaders = options == null ? true :
JSONUtilities.getBoolean(options, "outputColumnHeaders", true);
@ -93,7 +99,7 @@ abstract public class CustomizableTabularExporterUtilities {
final Map<String, CellFormatter> columnNameToFormatter =
new HashMap<String, CustomizableTabularExporterUtilities.CellFormatter>();
JSONArray columnOptionArray = options == null ? null :
List<JsonNode> columnOptionArray = options == null ? null :
JSONUtilities.getArray(options, "columns");
if (columnOptionArray == null) {
List<Column> columns = project.columnModel.columns;
@ -105,16 +111,20 @@ abstract public class CustomizableTabularExporterUtilities {
columnNameToFormatter.put(name, new CellFormatter());
}
} else {
int count = columnOptionArray.length();
int count = columnOptionArray.size();
columnNames = new ArrayList<String>(count);
for (int i = 0; i < count; i++) {
JSONObject columnOptions = JSONUtilities.getObjectElement(columnOptionArray, i);
JsonNode columnOptions = columnOptionArray.get(i);
if (columnOptions != null) {
String name = JSONUtilities.getString(columnOptions, "name", null);
if (name != null) {
columnNames.add(name);
columnNameToFormatter.put(name, new CellFormatter(columnOptions));
try {
columnNameToFormatter.put(name, ParsingUtilities.mapper.treeToValue(columnOptions, ColumnOptions.class));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
}
}
}
@ -186,7 +196,7 @@ abstract public class CustomizableTabularExporterUtilities {
int rows;
@Override
public void startFile(JSONObject options) {
public void startFile(JsonNode options) {
}
@Override
@ -201,30 +211,62 @@ abstract public class CustomizableTabularExporterUtilities {
}
private enum ReconOutputMode {
@JsonProperty("entity-name")
ENTITY_NAME,
@JsonProperty("entity-id")
ENTITY_ID,
@JsonProperty("cell-content")
CELL_CONTENT
}
private enum DateFormatMode {
@JsonProperty("iso-8601")
ISO_8601,
@JsonProperty("locale-short")
SHORT_LOCALE,
@JsonProperty("locale-medium")
MEDIUM_LOCALE,
@JsonProperty("locale-long")
LONG_LOCALE,
@JsonProperty("locale-full")
FULL_LOCALE,
@JsonProperty("custom")
CUSTOM
}
final static private String fullIso8601 = "yyyy-MM-dd'T'HH:mm:ss'Z'";
static private class ReconSettings {
@JsonProperty("output")
ReconOutputMode outputMode = ReconOutputMode.ENTITY_NAME;
@JsonProperty("blankUnmatchedCells")
boolean blankUnmatchedCells = false;
@JsonProperty("linkToEntityPages")
boolean linkToEntityPages = true;
}
static private class CellFormatter {
ReconOutputMode recon_outputMode = ReconOutputMode.ENTITY_NAME;
boolean recon_blankUnmatchedCells = false;
boolean recon_linkToEntityPages = true;
static private class DateSettings {
@JsonProperty("format")
DateFormatMode formatMode = DateFormatMode.ISO_8601;
@JsonProperty("custom")
String custom = null;
@JsonProperty("useLocalTimeZone")
boolean useLocalTimeZone = false;
@JsonProperty("omitTime")
boolean omitTime = false;
}
DateFormatMode date_formatMode = DateFormatMode.ISO_8601;
String date_custom = null;
boolean date_useLocalTimeZone = false;
boolean date_omitTime = false;
static public class ColumnOptions extends CellFormatter {
@JsonProperty("name")
String columnName;
}
static public class CellFormatter {
@JsonProperty("reconSettings")
ReconSettings recon = new ReconSettings();
@JsonProperty("dateSettings")
DateSettings date = new DateSettings();
//SQLExporter parameter to convert null cell value to empty string
@JsonProperty("nullValueToEmptyStr")
boolean includeNullFieldValue = false;
DateFormat dateFormatter;
String[] urlSchemes = {"http","https", "ftp"};
@ -232,88 +274,65 @@ abstract public class CustomizableTabularExporterUtilities {
Map<String, String> identifierSpaceToUrl = null;
//SQLExporter parameter to convert null cell value to empty string
boolean includeNullFieldValue = false;
@JsonCreator
CellFormatter(
@JsonProperty("reconSettings")
ReconSettings reconSettings,
@JsonProperty("dateSettings")
DateSettings dateSettings,
@JsonProperty("nullValueToEmptyStr")
boolean includeNullFieldValue) {
if(reconSettings != null) {
recon = reconSettings;
}
if(dateSettings != null) {
date = dateSettings;
}
setup();
}
CellFormatter() {
dateFormatter = new SimpleDateFormat(fullIso8601);
setup();
}
CellFormatter(JSONObject options) {
JSONObject reconSettings = JSONUtilities.getObject(options, "reconSettings");
includeNullFieldValue = JSONUtilities.getBoolean(options, "nullValueToEmptyStr", false);
if (reconSettings != null) {
String reconOutputString = JSONUtilities.getString(reconSettings, "output", null);
if ("entity-name".equals(reconOutputString)) {
recon_outputMode = ReconOutputMode.ENTITY_NAME;
} else if ("entity-id".equals(reconOutputString)) {
recon_outputMode = ReconOutputMode.ENTITY_ID;
} else if ("cell-content".equals(reconOutputString)) {
recon_outputMode = ReconOutputMode.CELL_CONTENT;
private void setup() {
if (date.formatMode == DateFormatMode.CUSTOM &&
(date.custom == null || date.custom.isEmpty())) {
date.formatMode = DateFormatMode.ISO_8601;
}
recon_blankUnmatchedCells = JSONUtilities.getBoolean(reconSettings, "blankUnmatchedCells", recon_blankUnmatchedCells);
recon_linkToEntityPages = JSONUtilities.getBoolean(reconSettings, "linkToEntityPages", recon_linkToEntityPages);
}
JSONObject dateSettings = JSONUtilities.getObject(options, "dateSettings");
if (dateSettings != null) {
String dateFormatString = JSONUtilities.getString(dateSettings, "format", null);
if ("iso-8601".equals(dateFormatString)) {
date_formatMode = DateFormatMode.ISO_8601;
} else if ("locale-short".equals(dateFormatString)) {
date_formatMode = DateFormatMode.SHORT_LOCALE;
} else if ("locale-medium".equals(dateFormatString)) {
date_formatMode = DateFormatMode.MEDIUM_LOCALE;
} else if ("locale-long".equals(dateFormatString)) {
date_formatMode = DateFormatMode.LONG_LOCALE;
} else if ("locale-full".equals(dateFormatString)) {
date_formatMode = DateFormatMode.FULL_LOCALE;
} else if ("custom".equals(dateFormatString)) {
date_formatMode = DateFormatMode.CUSTOM;
}
date_custom = JSONUtilities.getString(dateSettings, "custom", null);
date_useLocalTimeZone = JSONUtilities.getBoolean(dateSettings, "useLocalTimeZone", date_useLocalTimeZone);
date_omitTime = JSONUtilities.getBoolean(dateSettings, "omitTime", date_omitTime);
if (date_formatMode == DateFormatMode.CUSTOM &&
(date_custom == null || date_custom.isEmpty())) {
date_formatMode = DateFormatMode.ISO_8601;
}
}
switch (date_formatMode) {
switch (date.formatMode) {
case SHORT_LOCALE:
dateFormatter = date_omitTime ?
dateFormatter = date.omitTime ?
SimpleDateFormat.getDateInstance(SimpleDateFormat.SHORT) :
SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.SHORT, SimpleDateFormat.SHORT);
break;
case MEDIUM_LOCALE:
dateFormatter = date_omitTime ?
dateFormatter = date.omitTime ?
SimpleDateFormat.getDateInstance(SimpleDateFormat.MEDIUM) :
SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.MEDIUM, SimpleDateFormat.MEDIUM);
break;
case LONG_LOCALE:
dateFormatter = date_omitTime ?
dateFormatter = date.omitTime ?
SimpleDateFormat.getDateInstance(SimpleDateFormat.LONG) :
SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.LONG, SimpleDateFormat.LONG);
break;
case FULL_LOCALE:
dateFormatter = date_omitTime ?
dateFormatter = date.omitTime ?
SimpleDateFormat.getDateInstance(SimpleDateFormat.FULL) :
SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.FULL, SimpleDateFormat.FULL);
break;
case CUSTOM:
dateFormatter = new SimpleDateFormat(date_custom);
dateFormatter = new SimpleDateFormat(date.custom);
break;
default:
dateFormatter = date_omitTime ?
dateFormatter = date.omitTime ?
new SimpleDateFormat("yyyy-MM-dd") :
new SimpleDateFormat(fullIso8601);
}
if (!date_useLocalTimeZone) {
if (!date.useLocalTimeZone) {
dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
}
}
@ -326,13 +345,13 @@ abstract public class CustomizableTabularExporterUtilities {
if (cell.recon != null) {
Recon recon = cell.recon;
if (recon.judgment == Recon.Judgment.Matched) {
if (recon_outputMode == ReconOutputMode.ENTITY_NAME) {
if (this.recon.outputMode == ReconOutputMode.ENTITY_NAME) {
text = recon.match.name;
} else if (recon_outputMode == ReconOutputMode.ENTITY_ID) {
} else if (this.recon.outputMode == ReconOutputMode.ENTITY_ID) {
text = recon.match.id;
} // else: output cell content
if (recon_linkToEntityPages) {
if (this.recon.linkToEntityPages) {
buildIdentifierSpaceToUrlMap();
String service = recon.service;
@ -341,7 +360,7 @@ abstract public class CustomizableTabularExporterUtilities {
link = StringUtils.replace(viewUrl, "{{id}}", recon.match.id);
}
}
} else if (recon_blankUnmatchedCells) {
} else if (this.recon.blankUnmatchedCells) {
return null;
}
}

4
main/src/com/google/refine/exporters/HtmlTableExporter.java
View File

@ -41,6 +41,8 @@ import java.util.Properties;
import org.apache.commons.lang3.StringEscapeUtils;
import org.json.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.model.Project;
@ -58,7 +60,7 @@ public class HtmlTableExporter implements WriterExporter {
TabularSerializer serializer = new TabularSerializer() {
@Override
public void startFile(JSONObject options) {
public void startFile(JsonNode options) {
try {
writer.write("<html>\n");
writer.write("<head>\n");

5
main/src/com/google/refine/exporters/OdsExporter.java
View File

@ -39,12 +39,13 @@ import java.time.OffsetDateTime;
import java.util.List;
import java.util.Properties;
import org.json.JSONObject;
import org.odftoolkit.odfdom.doc.OdfSpreadsheetDocument;
import org.odftoolkit.odfdom.doc.table.OdfTable;
import org.odftoolkit.odfdom.doc.table.OdfTableCell;
import org.odftoolkit.odfdom.doc.table.OdfTableRow;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.model.Project;
@ -73,7 +74,7 @@ public class OdsExporter implements StreamExporter {
//int rowCount = 0;
@Override
public void startFile(JSONObject options) {
public void startFile(JsonNode options) {
table = OdfTable.newTable(odfDoc);
table.setTableName(ProjectManager.singleton.getProjectMetadata(project.id).getName());
}

4
main/src/com/google/refine/exporters/TabularSerializer.java
View File

@ -2,7 +2,7 @@ package com.google.refine.exporters;
import java.util.List;
import org.json.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
/**
@ -25,7 +25,7 @@ public interface TabularSerializer {
}
}
public void startFile(JSONObject options);
public void startFile(JsonNode options);
public void endFile();

5
main/src/com/google/refine/exporters/XlsExporter.java
View File

@ -48,7 +48,8 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.util.WorkbookUtil;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.json.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
@ -79,7 +80,7 @@ public class XlsExporter implements StreamExporter {
CellStyle dateStyle;
@Override
public void startFile(JSONObject options) {
public void startFile(JsonNode options) {
s = wb.createSheet();
String sheetName = WorkbookUtil.createSafeSheetName(
ProjectManager.singleton.getProjectMetadata(project.id).getName());

16
main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java
View File

@ -29,13 +29,13 @@
package com.google.refine.exporters.sql;
import java.util.Collections;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.util.JSONUtilities;
public class SqlCreateBuilder {
@ -44,13 +44,13 @@ public class SqlCreateBuilder {
private String table;
private List<String> columns;
private JSONObject options;
private JsonNode options;
public SqlCreateBuilder(String table, List<String> columns, JSONObject options) {
public SqlCreateBuilder(String table, List<String> columns, JsonNode sqlOptions) {
this.table = table;
this.columns = columns;
this.options = options;
this.options = sqlOptions;
}
@ -60,15 +60,15 @@ public class SqlCreateBuilder {
}
StringBuffer createSB = new StringBuffer();
JSONArray columnOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns");
List<JsonNode> columnOptionArray = options == null ? Collections.emptyList() : JSONUtilities.getArray(options, "columns");
boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false);
int count = columnOptionArray.length();
int count = columnOptionArray.size();
for (int i = 0; i < count; i++) {
JSONObject columnOptions = JSONUtilities.getObjectElement(columnOptionArray, i);
JsonNode columnOptions = columnOptionArray.get(i);
if (columnOptions != null) {
String name = JSONUtilities.getString(columnOptions, "name", null);
String type = JSONUtilities.getString(columnOptions, "type", SqlData.SQL_TYPE_VARCHAR);

12
main/src/com/google/refine/exporters/sql/SqlExporter.java
View File

@ -39,6 +39,8 @@ import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.exporters.CustomizableTabularExporterUtilities;
@ -59,7 +61,7 @@ public class SqlExporter implements WriterExporter {
private List<String> columnNames = new ArrayList<String>();
private List<ArrayList<SqlData>> sqlDataList = new ArrayList<ArrayList<SqlData>>();
private JSONObject sqlOptions;
private JsonNode sqlOptions;
@Override
@ -77,7 +79,7 @@ public class SqlExporter implements WriterExporter {
TabularSerializer serializer = new TabularSerializer() {
@Override
public void startFile(JSONObject options) {
public void startFile(JsonNode options) {
sqlOptions = options;
//logger.info("setting options::{}", sqlOptions);
}
@ -97,10 +99,10 @@ public class SqlExporter implements WriterExporter {
}
String tableName = ProjectManager.singleton.getProjectMetadata(project.id).getName();
Object tableNameManual = sqlOptions.get(JSON_TABLE_NAME);
String tableNameManual = JSONUtilities.getString(sqlOptions, JSON_TABLE_NAME, null);
if (tableNameManual != null && !tableNameManual.toString().isEmpty()) {
tableName = tableNameManual.toString();
if (tableNameManual != null) {
tableName = tableNameManual;
}
SqlCreateBuilder createBuilder = new SqlCreateBuilder(tableName, columnNames, sqlOptions);

24
main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java
View File

@ -36,11 +36,10 @@ import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.lang3.math.NumberUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.util.JSONUtilities;
public class SqlInsertBuilder {
@ -53,7 +52,7 @@ public class SqlInsertBuilder {
private List<ArrayList<SqlData>> sqlDataList;
private JSONObject options;
private JsonNode options;
/**
@ -61,15 +60,15 @@ public class SqlInsertBuilder {
* @param table
* @param columns
* @param rows
* @param options
* @param sqlOptions
* @param sqlErrors
*/
public SqlInsertBuilder(String table, List<String> columns, List<ArrayList<SqlData>> rows, JSONObject options
public SqlInsertBuilder(String table, List<String> columns, List<ArrayList<SqlData>> rows, JsonNode sqlOptions
) {
this.table = table;
this.columns = columns;
this.sqlDataList = rows;
this.options = options;
this.options = sqlOptions;
//logger.info("Column Size:{}", columns.size());
}
@ -83,12 +82,11 @@ public class SqlInsertBuilder {
logger.debug("Insert SQL with columns: {}", columns);
}
JSONArray colOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns");
Map<String, JSONObject> colOptionsMap = new HashMap<String, JSONObject>();
List<JsonNode> colOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns");
Map<String, JsonNode> colOptionsMap = new HashMap<>();
if(colOptionArray != null) {
colOptionArray.forEach(c -> {
JSONObject json = (JSONObject)c;
colOptionsMap.put("" + json.get("name"), json);
colOptionArray.forEach(json -> {
colOptionsMap.put(JSONUtilities.getString(json, "name", null), json);
});
}
@ -103,8 +101,8 @@ public class SqlInsertBuilder {
//int fieldCount = 0;
for(SqlData val : sqlRow) {
JSONObject jsonOb = colOptionsMap.get(val.getColumnName());
String type = (String)jsonOb.get("type");
JsonNode jsonOb = colOptionsMap.get(val.getColumnName());
String type = JSONUtilities.getString(jsonOb, "type", null);
String defaultValue = JSONUtilities.getString(jsonOb, "defaultValue", null);

38
main/src/com/google/refine/util/JSONUtilities.java
View File

@ -47,8 +47,14 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.JsonNodeType;
import com.google.common.collect.Lists;
public class JSONUtilities {
static public JSONObject getObject(JSONObject obj, String key) {
try {
return obj.getJSONObject(key);
@ -65,6 +71,14 @@ public class JSONUtilities {
}
}
static public String getString(JsonNode obj, String key, String def) {
if (obj.get(key) != null) {
return obj.get(key).textValue();
} else {
return def;
}
}
static public int getInt(JSONObject obj, String key, int def) {
try {
return obj.getInt(key);
@ -73,6 +87,14 @@ public class JSONUtilities {
}
}
static public int getInt(JsonNode obj, String key, int def) {
if (obj.get(key) != null) {
return obj.get(key).asInt(def);
} else {
return def;
}
}
static public boolean getBoolean(JSONObject obj, String key, boolean def) {
try {
return obj.getBoolean(key);
@ -81,6 +103,14 @@ public class JSONUtilities {
}
}
static public boolean getBoolean(JsonNode obj, String key, boolean def) {
if (obj.get(key) != null) {
return obj.get(key).asBoolean(def);
} else {
return def;
}
}
static public double getDouble(JSONObject obj, String key, double def) {
try {
return obj.getDouble(key);
@ -125,6 +155,14 @@ public class JSONUtilities {
}
}
static public List<JsonNode> getArray(JsonNode obj, String key) {
if (obj.has(key) && obj.get(key).getNodeType().equals(JsonNodeType.ARRAY)) {
return Lists.newArrayList(obj.get(key).elements());
} else {
return null;
}
}
static public JSONArray arrayToJSONArray(String[] array) {
return new JSONArray(Arrays.asList(array));
}

48
main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java
View File

@ -54,6 +54,9 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.exporters.sql.SqlCreateBuilder;
@ -68,6 +71,8 @@ import com.google.refine.model.Row;
import com.google.refine.model.metadata.ProjectMetadata;
import com.google.refine.tests.ProjectManagerStub;
import com.google.refine.tests.RefineTest;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities;
public class SqlExporterTests extends RefineTest {
@ -180,14 +185,10 @@ public class SqlExporterTests extends RefineTest {
}
String result = writer.toString();
// logger.info("result = \n" + result);
Assert.assertNotNull(result);
assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR);
boolean checkResult = result.contains("CREATE TABLE " + tableName);
//logger.info("checkResult1 =" + checkResult);
checkResult = result.contains("INSERT INTO " + tableName);
// logger.info("checkResult2 =" + checkResult);
Assert.assertEquals(checkResult, true);
Assert.assertTrue(result.contains("INSERT INTO " + tableName));
}
@ -195,7 +196,7 @@ public class SqlExporterTests extends RefineTest {
public void testExportSqlNoSchema(){
createGrid(2, 2);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null,null);
ObjectNode optionsJson = (ObjectNode) createOptionsFromProject(tableName, null,null);
optionsJson.put("includeStructure", false);
when(options.getProperty("options")).thenReturn(optionsJson.toString());
// logger.info("Options = " + optionsJson.toString());
@ -222,7 +223,7 @@ public class SqlExporterTests extends RefineTest {
public void testExportSqlNoContent(){
createGrid(2, 2);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null);
ObjectNode optionsJson = (ObjectNode) createOptionsFromProject(tableName, null, null);
optionsJson.put("includeContent", false);
when(options.getProperty("options")).thenReturn(optionsJson.toString());
//logger.info("Options = " + optionsJson.toString());
@ -249,7 +250,7 @@ public class SqlExporterTests extends RefineTest {
public void testExportSqlIncludeSchemaWithDropStmt(){
createGrid(2, 2);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null);
ObjectNode optionsJson = (ObjectNode) createOptionsFromProject(tableName, null, null);
optionsJson.put("includeStructure", true);
optionsJson.put("includeDropStatement", true);
@ -289,7 +290,7 @@ public class SqlExporterTests extends RefineTest {
String tableName = "sql_table_test";
String type = "CHAR";
String size = "2";
JSONObject optionsJson = createOptionsFromProject(tableName, type, size);
JsonNode optionsJson = createOptionsFromProject(tableName, type, size);
// logger.info("Options:: = " + optionsJson.toString());
List<String> columns = project.columnModel.columns.stream().map(col -> col.getName()).collect(Collectors.toList());
@ -307,7 +308,7 @@ public class SqlExporterTests extends RefineTest {
int inNull = 8;
createGridWithNullFields(3, 3, inNull);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null);
ObjectNode optionsJson = (ObjectNode) createOptionsFromProject(tableName, null, null);
optionsJson.put("includeStructure", true);
optionsJson.put("includeDropStatement", true);
optionsJson.put("convertNulltoEmptyString", true);
@ -338,7 +339,7 @@ public class SqlExporterTests extends RefineTest {
int noOfRows = 3;
createGrid(noOfRows, noOfCols);
String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null, null, false);
ObjectNode optionsJson = createOptionsFromProject(tableName, null, null, null, false);
optionsJson.put("includeStructure", true);
optionsJson.put("includeDropStatement", true);
optionsJson.put("convertNulltoEmptyString", true);
@ -497,18 +498,16 @@ public class SqlExporterTests extends RefineTest {
return json;
}
protected JSONObject createOptionsFromProject(String tableName, String type, String size) {
JSONObject json = new JSONObject();
JSONArray columns = new JSONArray();
json.put("columns", columns);
protected JsonNode createOptionsFromProject(String tableName, String type, String size) {
ObjectNode json = ParsingUtilities.mapper.createObjectNode();
json.put("tableName", tableName);
ArrayNode columns = json.putArray("columns");
List<Column> cols = project.columnModel.columns;
cols.forEach(c -> {
//logger.info("Column Name = " + c.getName());
JSONObject columnModel = new JSONObject();
ObjectNode columnModel = ParsingUtilities.mapper.createObjectNode();
columnModel.put("name", c.getName());
if(type != null) {
columnModel.put("type", type);
@ -529,26 +528,25 @@ public class SqlExporterTests extends RefineTest {
columnModel.put("size", size);
}
columns.put(columnModel);
columns.add(columnModel);
});
return json;
}
protected JSONObject createOptionsFromProject(String tableName, String type, String size, String defaultValue,
protected ObjectNode createOptionsFromProject(String tableName, String type, String size, String defaultValue,
boolean allowNull) {
JSONObject json = new JSONObject();
JSONArray columns = new JSONArray();
json.put("columns", columns);
ObjectNode json = ParsingUtilities.mapper.createObjectNode();
ArrayNode columns = json.putArray("columns");
json.put("tableName", tableName);
List<Column> cols = project.columnModel.columns;
cols.forEach(c -> {
//logger.info("Column Name = " + c.getName());
JSONObject columnModel = new JSONObject();
ObjectNode columnModel = ParsingUtilities.mapper.createObjectNode();
columnModel.put("name", c.getName());
if(type != null) {
columnModel.put("type", type);
@ -572,7 +570,7 @@ public class SqlExporterTests extends RefineTest {
columnModel.put("defaultValue", defaultValue);
columnModel.put("allowNull", allowNull);
columns.put(columnModel);
columns.add(columnModel);
});