s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Migrate TabularExporter to use JsonNode

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-11-02 14:05:33 +00:00
parent 52426b98a3
commit 2873035ee2
13 changed files with 207 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
extensions/gdata/src/com/google/refine/extension/gdata/FusionTableSerializer.java
View File

@ -4,13 +4,14 @@ import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.json.JSONObject; import com.fasterxml.jackson.databind.JsonNode;
import com.google.api.client.http.AbstractInputStreamContent; import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpResponseException;
import com.google.api.services.fusiontables.Fusiontables; import com.google.api.services.fusiontables.Fusiontables;
import com.google.refine.exporters.CustomizableTabularExporterConfiguration;
import com.google.refine.exporters.TabularSerializer; import com.google.refine.exporters.TabularSerializer;
final class FusionTableSerializer implements TabularSerializer { final class FusionTableSerializer implements TabularSerializer {
@ -31,7 +32,7 @@ final class FusionTableSerializer implements TabularSerializer {
} }
@Override @Override
public void startFile(JSONObject options) { public void startFile(JsonNode options) {
} }
@Override @Override

6
extensions/gdata/src/com/google/refine/extension/gdata/SpreadsheetSerializer.java
View File

@ -6,10 +6,11 @@ import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.api.services.sheets.v4.Sheets; import com.google.api.services.sheets.v4.Sheets;
import com.google.api.services.sheets.v4.model.AppendCellsRequest; import com.google.api.services.sheets.v4.model.AppendCellsRequest;
import com.google.api.services.sheets.v4.model.BatchUpdateSpreadsheetRequest; import com.google.api.services.sheets.v4.model.BatchUpdateSpreadsheetRequest;
@ -18,6 +19,7 @@ import com.google.api.services.sheets.v4.model.ExtendedValue;
import com.google.api.services.sheets.v4.model.Request; import com.google.api.services.sheets.v4.model.Request;
import com.google.api.services.sheets.v4.model.RowData; import com.google.api.services.sheets.v4.model.RowData;
import com.google.refine.exporters.CustomizableTabularExporterConfiguration;
import com.google.refine.exporters.TabularSerializer; import com.google.refine.exporters.TabularSerializer;
final class SpreadsheetSerializer implements TabularSerializer { final class SpreadsheetSerializer implements TabularSerializer {
@ -44,7 +46,7 @@ final class SpreadsheetSerializer implements TabularSerializer {
} }
@Override @Override
public void startFile(JSONObject options) { public void startFile(JsonNode options) {
} }

11
main/src/com/google/refine/exporters/CsvExporter.java
View File

@ -38,12 +38,11 @@ import java.io.Writer;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine;
import com.google.refine.model.Project; import com.google.refine.model.Project;
@ -82,14 +81,14 @@ public class CsvExporter implements WriterExporter{
if (optionsString != null) { if (optionsString != null) {
try { try {
options = ParsingUtilities.mapper.readValue(optionsString, Configuration.class); options = ParsingUtilities.mapper.readValue(optionsString, Configuration.class);
if (options.separator == null) {
options.separator = Character.toString(separator);
}
} catch (IOException e) { } catch (IOException e) {
// Ignore and keep options null. // Ignore and keep options null.
e.printStackTrace(); e.printStackTrace();
} }
} }
if (options.separator == null) {
options.separator = Character.toString(separator);
}
final String separator = options.separator; final String separator = options.separator;
final String lineSeparator = options.lineSeparator; final String lineSeparator = options.lineSeparator;
@ -105,7 +104,7 @@ public class CsvExporter implements WriterExporter{
TabularSerializer serializer = new TabularSerializer() { TabularSerializer serializer = new TabularSerializer() {
@Override @Override
public void startFile(JSONObject options) { public void startFile(JsonNode options) {
} }
@Override @Override

173
main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java
View File

@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.exporters; package com.google.refine.exporters;
import java.io.IOException;
import java.text.DateFormat; import java.text.DateFormat;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
@ -47,9 +48,12 @@ import java.util.TimeZone;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.validator.routines.UrlValidator; import org.apache.commons.validator.routines.UrlValidator;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows; import com.google.refine.browsing.FilteredRows;
@ -65,6 +69,8 @@ import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities; import com.google.refine.util.ParsingUtilities;
abstract public class CustomizableTabularExporterUtilities { abstract public class CustomizableTabularExporterUtilities {
final static private String fullIso8601 = "yyyy-MM-dd'T'HH:mm:ss'Z'";
static public void exportRows( static public void exportRows(
final Project project, final Project project,
final Engine engine, final Engine engine,
@ -72,15 +78,15 @@ abstract public class CustomizableTabularExporterUtilities {
final TabularSerializer serializer) { final TabularSerializer serializer) {
String optionsString = (params != null) ? params.getProperty("options") : null; String optionsString = (params != null) ? params.getProperty("options") : null;
JSONObject optionsTemp = null; JsonNode optionsTemp = null;
if (optionsString != null) { if (optionsString != null) {
try { try {
optionsTemp = ParsingUtilities.evaluateJsonStringToObject(optionsString); optionsTemp = ParsingUtilities.mapper.readTree(optionsString);
} catch (JSONException e) { } catch (IOException e) {
// Ignore and keep options null. // Ignore and keep options null.
} }
} }
final JSONObject options = optionsTemp; final JsonNode options = optionsTemp;
final boolean outputColumnHeaders = options == null ? true : final boolean outputColumnHeaders = options == null ? true :
JSONUtilities.getBoolean(options, "outputColumnHeaders", true); JSONUtilities.getBoolean(options, "outputColumnHeaders", true);
@ -93,7 +99,7 @@ abstract public class CustomizableTabularExporterUtilities {
final Map<String, CellFormatter> columnNameToFormatter = final Map<String, CellFormatter> columnNameToFormatter =
new HashMap<String, CustomizableTabularExporterUtilities.CellFormatter>(); new HashMap<String, CustomizableTabularExporterUtilities.CellFormatter>();
JSONArray columnOptionArray = options == null ? null : List<JsonNode> columnOptionArray = options == null ? null :
JSONUtilities.getArray(options, "columns"); JSONUtilities.getArray(options, "columns");
if (columnOptionArray == null) { if (columnOptionArray == null) {
List<Column> columns = project.columnModel.columns; List<Column> columns = project.columnModel.columns;
@ -105,16 +111,20 @@ abstract public class CustomizableTabularExporterUtilities {
columnNameToFormatter.put(name, new CellFormatter()); columnNameToFormatter.put(name, new CellFormatter());
} }
} else { } else {
int count = columnOptionArray.length(); int count = columnOptionArray.size();
columnNames = new ArrayList<String>(count); columnNames = new ArrayList<String>(count);
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
JSONObject columnOptions = JSONUtilities.getObjectElement(columnOptionArray, i); JsonNode columnOptions = columnOptionArray.get(i);
if (columnOptions != null) { if (columnOptions != null) {
String name = JSONUtilities.getString(columnOptions, "name", null); String name = JSONUtilities.getString(columnOptions, "name", null);
if (name != null) { if (name != null) {
columnNames.add(name); columnNames.add(name);
columnNameToFormatter.put(name, new CellFormatter(columnOptions)); try {
columnNameToFormatter.put(name, ParsingUtilities.mapper.treeToValue(columnOptions, ColumnOptions.class));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
} }
} }
} }
@ -186,7 +196,7 @@ abstract public class CustomizableTabularExporterUtilities {
int rows; int rows;
@Override @Override
public void startFile(JSONObject options) { public void startFile(JsonNode options) {
} }
@Override @Override
@ -201,30 +211,62 @@ abstract public class CustomizableTabularExporterUtilities {
} }
private enum ReconOutputMode { private enum ReconOutputMode {
@JsonProperty("entity-name")
ENTITY_NAME, ENTITY_NAME,
@JsonProperty("entity-id")
ENTITY_ID, ENTITY_ID,
@JsonProperty("cell-content")
CELL_CONTENT CELL_CONTENT
} }
private enum DateFormatMode { private enum DateFormatMode {
@JsonProperty("iso-8601")
ISO_8601, ISO_8601,
@JsonProperty("locale-short")
SHORT_LOCALE, SHORT_LOCALE,
@JsonProperty("locale-medium")
MEDIUM_LOCALE, MEDIUM_LOCALE,
@JsonProperty("locale-long")
LONG_LOCALE, LONG_LOCALE,
@JsonProperty("locale-full")
FULL_LOCALE, FULL_LOCALE,
@JsonProperty("custom")
CUSTOM CUSTOM
} }
final static private String fullIso8601 = "yyyy-MM-dd'T'HH:mm:ss'Z'"; static private class ReconSettings {
@JsonProperty("output")
ReconOutputMode outputMode = ReconOutputMode.ENTITY_NAME;
@JsonProperty("blankUnmatchedCells")
boolean blankUnmatchedCells = false;
@JsonProperty("linkToEntityPages")
boolean linkToEntityPages = true;
}
static private class CellFormatter { static private class DateSettings {
ReconOutputMode recon_outputMode = ReconOutputMode.ENTITY_NAME; @JsonProperty("format")
boolean recon_blankUnmatchedCells = false; DateFormatMode formatMode = DateFormatMode.ISO_8601;
boolean recon_linkToEntityPages = true; @JsonProperty("custom")
String custom = null;
@JsonProperty("useLocalTimeZone")
boolean useLocalTimeZone = false;
@JsonProperty("omitTime")
boolean omitTime = false;
}
static public class ColumnOptions extends CellFormatter {
@JsonProperty("name")
String columnName;
}
static public class CellFormatter {
@JsonProperty("reconSettings")
ReconSettings recon = new ReconSettings();
@JsonProperty("dateSettings")
DateSettings date = new DateSettings();
DateFormatMode date_formatMode = DateFormatMode.ISO_8601; //SQLExporter parameter to convert null cell value to empty string
String date_custom = null; @JsonProperty("nullValueToEmptyStr")
boolean date_useLocalTimeZone = false; boolean includeNullFieldValue = false;
boolean date_omitTime = false;
DateFormat dateFormatter; DateFormat dateFormatter;
String[] urlSchemes = {"http","https", "ftp"}; String[] urlSchemes = {"http","https", "ftp"};
@ -232,88 +274,65 @@ abstract public class CustomizableTabularExporterUtilities {
Map<String, String> identifierSpaceToUrl = null; Map<String, String> identifierSpaceToUrl = null;
//SQLExporter parameter to convert null cell value to empty string @JsonCreator
boolean includeNullFieldValue = false; CellFormatter(
@JsonProperty("reconSettings")
CellFormatter() { ReconSettings reconSettings,
dateFormatter = new SimpleDateFormat(fullIso8601); @JsonProperty("dateSettings")
DateSettings dateSettings,
@JsonProperty("nullValueToEmptyStr")
boolean includeNullFieldValue) {
if(reconSettings != null) {
recon = reconSettings;
}
if(dateSettings != null) {
date = dateSettings;
}
setup();
} }
CellFormatter(JSONObject options) { CellFormatter() {
JSONObject reconSettings = JSONUtilities.getObject(options, "reconSettings"); setup();
includeNullFieldValue = JSONUtilities.getBoolean(options, "nullValueToEmptyStr", false); }
if (reconSettings != null) {
String reconOutputString = JSONUtilities.getString(reconSettings, "output", null); private void setup() {
if ("entity-name".equals(reconOutputString)) { if (date.formatMode == DateFormatMode.CUSTOM &&
recon_outputMode = ReconOutputMode.ENTITY_NAME; (date.custom == null || date.custom.isEmpty())) {
} else if ("entity-id".equals(reconOutputString)) { date.formatMode = DateFormatMode.ISO_8601;
recon_outputMode = ReconOutputMode.ENTITY_ID;
} else if ("cell-content".equals(reconOutputString)) {
recon_outputMode = ReconOutputMode.CELL_CONTENT;
}
recon_blankUnmatchedCells = JSONUtilities.getBoolean(reconSettings, "blankUnmatchedCells", recon_blankUnmatchedCells);
recon_linkToEntityPages = JSONUtilities.getBoolean(reconSettings, "linkToEntityPages", recon_linkToEntityPages);
}
JSONObject dateSettings = JSONUtilities.getObject(options, "dateSettings");
if (dateSettings != null) {
String dateFormatString = JSONUtilities.getString(dateSettings, "format", null);
if ("iso-8601".equals(dateFormatString)) {
date_formatMode = DateFormatMode.ISO_8601;
} else if ("locale-short".equals(dateFormatString)) {
date_formatMode = DateFormatMode.SHORT_LOCALE;
} else if ("locale-medium".equals(dateFormatString)) {
date_formatMode = DateFormatMode.MEDIUM_LOCALE;
} else if ("locale-long".equals(dateFormatString)) {
date_formatMode = DateFormatMode.LONG_LOCALE;
} else if ("locale-full".equals(dateFormatString)) {
date_formatMode = DateFormatMode.FULL_LOCALE;
} else if ("custom".equals(dateFormatString)) {
date_formatMode = DateFormatMode.CUSTOM;
}
date_custom = JSONUtilities.getString(dateSettings, "custom", null);
date_useLocalTimeZone = JSONUtilities.getBoolean(dateSettings, "useLocalTimeZone", date_useLocalTimeZone);
date_omitTime = JSONUtilities.getBoolean(dateSettings, "omitTime", date_omitTime);
if (date_formatMode == DateFormatMode.CUSTOM &&
(date_custom == null || date_custom.isEmpty())) {
date_formatMode = DateFormatMode.ISO_8601;
}
} }
switch (date_formatMode) { switch (date.formatMode) {
case SHORT_LOCALE: case SHORT_LOCALE:
dateFormatter = date_omitTime ? dateFormatter = date.omitTime ?
SimpleDateFormat.getDateInstance(SimpleDateFormat.SHORT) : SimpleDateFormat.getDateInstance(SimpleDateFormat.SHORT) :
SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.SHORT, SimpleDateFormat.SHORT); SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.SHORT, SimpleDateFormat.SHORT);
break; break;
case MEDIUM_LOCALE: case MEDIUM_LOCALE:
dateFormatter = date_omitTime ? dateFormatter = date.omitTime ?
SimpleDateFormat.getDateInstance(SimpleDateFormat.MEDIUM) : SimpleDateFormat.getDateInstance(SimpleDateFormat.MEDIUM) :
SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.MEDIUM, SimpleDateFormat.MEDIUM); SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.MEDIUM, SimpleDateFormat.MEDIUM);
break; break;
case LONG_LOCALE: case LONG_LOCALE:
dateFormatter = date_omitTime ? dateFormatter = date.omitTime ?
SimpleDateFormat.getDateInstance(SimpleDateFormat.LONG) : SimpleDateFormat.getDateInstance(SimpleDateFormat.LONG) :
SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.LONG, SimpleDateFormat.LONG); SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.LONG, SimpleDateFormat.LONG);
break; break;
case FULL_LOCALE: case FULL_LOCALE:
dateFormatter = date_omitTime ? dateFormatter = date.omitTime ?
SimpleDateFormat.getDateInstance(SimpleDateFormat.FULL) : SimpleDateFormat.getDateInstance(SimpleDateFormat.FULL) :
SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.FULL, SimpleDateFormat.FULL); SimpleDateFormat.getDateTimeInstance(SimpleDateFormat.FULL, SimpleDateFormat.FULL);
break; break;
case CUSTOM: case CUSTOM:
dateFormatter = new SimpleDateFormat(date_custom); dateFormatter = new SimpleDateFormat(date.custom);
break; break;
default: default:
dateFormatter = date_omitTime ? dateFormatter = date.omitTime ?
new SimpleDateFormat("yyyy-MM-dd") : new SimpleDateFormat("yyyy-MM-dd") :
new SimpleDateFormat(fullIso8601); new SimpleDateFormat(fullIso8601);
} }
if (!date_useLocalTimeZone) { if (!date.useLocalTimeZone) {
dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
} }
} }
@ -326,13 +345,13 @@ abstract public class CustomizableTabularExporterUtilities {
if (cell.recon != null) { if (cell.recon != null) {
Recon recon = cell.recon; Recon recon = cell.recon;
if (recon.judgment == Recon.Judgment.Matched) { if (recon.judgment == Recon.Judgment.Matched) {
if (recon_outputMode == ReconOutputMode.ENTITY_NAME) { if (this.recon.outputMode == ReconOutputMode.ENTITY_NAME) {
text = recon.match.name; text = recon.match.name;
} else if (recon_outputMode == ReconOutputMode.ENTITY_ID) { } else if (this.recon.outputMode == ReconOutputMode.ENTITY_ID) {
text = recon.match.id; text = recon.match.id;
} // else: output cell content } // else: output cell content
if (recon_linkToEntityPages) { if (this.recon.linkToEntityPages) {
buildIdentifierSpaceToUrlMap(); buildIdentifierSpaceToUrlMap();
String service = recon.service; String service = recon.service;
@ -341,7 +360,7 @@ abstract public class CustomizableTabularExporterUtilities {
link = StringUtils.replace(viewUrl, "{{id}}", recon.match.id); link = StringUtils.replace(viewUrl, "{{id}}", recon.match.id);
} }
} }
} else if (recon_blankUnmatchedCells) { } else if (this.recon.blankUnmatchedCells) {
return null; return null;
} }
} }

4
main/src/com/google/refine/exporters/HtmlTableExporter.java
View File

@ -41,6 +41,8 @@ import java.util.Properties;
import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.StringEscapeUtils;
import org.json.JSONObject; import org.json.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine;
import com.google.refine.model.Project; import com.google.refine.model.Project;
@ -58,7 +60,7 @@ public class HtmlTableExporter implements WriterExporter {
TabularSerializer serializer = new TabularSerializer() { TabularSerializer serializer = new TabularSerializer() {
@Override @Override
public void startFile(JSONObject options) { public void startFile(JsonNode options) {
try { try {
writer.write("<html>\n"); writer.write("<html>\n");
writer.write("<head>\n"); writer.write("<head>\n");

5
main/src/com/google/refine/exporters/OdsExporter.java
View File

@ -39,12 +39,13 @@ import java.time.OffsetDateTime;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.json.JSONObject;
import org.odftoolkit.odfdom.doc.OdfSpreadsheetDocument; import org.odftoolkit.odfdom.doc.OdfSpreadsheetDocument;
import org.odftoolkit.odfdom.doc.table.OdfTable; import org.odftoolkit.odfdom.doc.table.OdfTable;
import org.odftoolkit.odfdom.doc.table.OdfTableCell; import org.odftoolkit.odfdom.doc.table.OdfTableCell;
import org.odftoolkit.odfdom.doc.table.OdfTableRow; import org.odftoolkit.odfdom.doc.table.OdfTableRow;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine;
import com.google.refine.model.Project; import com.google.refine.model.Project;
@ -73,7 +74,7 @@ public class OdsExporter implements StreamExporter {
//int rowCount = 0; //int rowCount = 0;
@Override @Override
public void startFile(JSONObject options) { public void startFile(JsonNode options) {
table = OdfTable.newTable(odfDoc); table = OdfTable.newTable(odfDoc);
table.setTableName(ProjectManager.singleton.getProjectMetadata(project.id).getName()); table.setTableName(ProjectManager.singleton.getProjectMetadata(project.id).getName());
} }

4
main/src/com/google/refine/exporters/TabularSerializer.java
View File

@ -2,7 +2,7 @@ package com.google.refine.exporters;
import java.util.List; import java.util.List;
import org.json.JSONObject; import com.fasterxml.jackson.databind.JsonNode;
/** /**
@ -25,7 +25,7 @@ public interface TabularSerializer {
} }
} }
public void startFile(JSONObject options); public void startFile(JsonNode options);
public void endFile(); public void endFile();

5
main/src/com/google/refine/exporters/XlsExporter.java
View File

@ -48,7 +48,8 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.util.WorkbookUtil; import org.apache.poi.ss.util.WorkbookUtil;
import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.json.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine;
@ -79,7 +80,7 @@ public class XlsExporter implements StreamExporter {
CellStyle dateStyle; CellStyle dateStyle;
@Override @Override
public void startFile(JSONObject options) { public void startFile(JsonNode options) {
s = wb.createSheet(); s = wb.createSheet();
String sheetName = WorkbookUtil.createSafeSheetName( String sheetName = WorkbookUtil.createSafeSheetName(
ProjectManager.singleton.getProjectMetadata(project.id).getName()); ProjectManager.singleton.getProjectMetadata(project.id).getName());

16
main/src/com/google/refine/exporters/sql/SqlCreateBuilder.java
View File

@ -29,13 +29,13 @@
package com.google.refine.exporters.sql; package com.google.refine.exporters.sql;
import java.util.Collections;
import java.util.List; import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.util.JSONUtilities; import com.google.refine.util.JSONUtilities;
public class SqlCreateBuilder { public class SqlCreateBuilder {
@ -44,13 +44,13 @@ public class SqlCreateBuilder {
private String table; private String table;
private List<String> columns; private List<String> columns;
private JSONObject options; private JsonNode options;
public SqlCreateBuilder(String table, List<String> columns, JSONObject options) { public SqlCreateBuilder(String table, List<String> columns, JsonNode sqlOptions) {
this.table = table; this.table = table;
this.columns = columns; this.columns = columns;
this.options = options; this.options = sqlOptions;
} }
@ -60,15 +60,15 @@ public class SqlCreateBuilder {
} }
StringBuffer createSB = new StringBuffer(); StringBuffer createSB = new StringBuffer();
JSONArray columnOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns"); List<JsonNode> columnOptionArray = options == null ? Collections.emptyList() : JSONUtilities.getArray(options, "columns");
boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false); boolean trimColNames = options == null ? false : JSONUtilities.getBoolean(options, "trimColumnNames", false);
int count = columnOptionArray.length(); int count = columnOptionArray.size();
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
JSONObject columnOptions = JSONUtilities.getObjectElement(columnOptionArray, i); JsonNode columnOptions = columnOptionArray.get(i);
if (columnOptions != null) { if (columnOptions != null) {
String name = JSONUtilities.getString(columnOptions, "name", null); String name = JSONUtilities.getString(columnOptions, "name", null);
String type = JSONUtilities.getString(columnOptions, "type", SqlData.SQL_TYPE_VARCHAR); String type = JSONUtilities.getString(columnOptions, "type", SqlData.SQL_TYPE_VARCHAR);

12
main/src/com/google/refine/exporters/sql/SqlExporter.java
View File

@ -39,6 +39,8 @@ import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine;
import com.google.refine.exporters.CustomizableTabularExporterUtilities; import com.google.refine.exporters.CustomizableTabularExporterUtilities;
@ -59,7 +61,7 @@ public class SqlExporter implements WriterExporter {
private List<String> columnNames = new ArrayList<String>(); private List<String> columnNames = new ArrayList<String>();
private List<ArrayList<SqlData>> sqlDataList = new ArrayList<ArrayList<SqlData>>(); private List<ArrayList<SqlData>> sqlDataList = new ArrayList<ArrayList<SqlData>>();
private JSONObject sqlOptions; private JsonNode sqlOptions;
@Override @Override
@ -77,7 +79,7 @@ public class SqlExporter implements WriterExporter {
TabularSerializer serializer = new TabularSerializer() { TabularSerializer serializer = new TabularSerializer() {
@Override @Override
public void startFile(JSONObject options) { public void startFile(JsonNode options) {
sqlOptions = options; sqlOptions = options;
//logger.info("setting options::{}", sqlOptions); //logger.info("setting options::{}", sqlOptions);
} }
@ -97,10 +99,10 @@ public class SqlExporter implements WriterExporter {
} }
String tableName = ProjectManager.singleton.getProjectMetadata(project.id).getName(); String tableName = ProjectManager.singleton.getProjectMetadata(project.id).getName();
Object tableNameManual = sqlOptions.get(JSON_TABLE_NAME); String tableNameManual = JSONUtilities.getString(sqlOptions, JSON_TABLE_NAME, null);
if (tableNameManual != null && !tableNameManual.toString().isEmpty()) { if (tableNameManual != null) {
tableName = tableNameManual.toString(); tableName = tableNameManual;
} }
SqlCreateBuilder createBuilder = new SqlCreateBuilder(tableName, columnNames, sqlOptions); SqlCreateBuilder createBuilder = new SqlCreateBuilder(tableName, columnNames, sqlOptions);

24
main/src/com/google/refine/exporters/sql/SqlInsertBuilder.java
View File

@ -36,11 +36,10 @@ import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.lang3.math.NumberUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.util.JSONUtilities; import com.google.refine.util.JSONUtilities;
public class SqlInsertBuilder { public class SqlInsertBuilder {
@ -53,7 +52,7 @@ public class SqlInsertBuilder {
private List<ArrayList<SqlData>> sqlDataList; private List<ArrayList<SqlData>> sqlDataList;
private JSONObject options; private JsonNode options;
/** /**
@ -61,15 +60,15 @@ public class SqlInsertBuilder {
* @param table * @param table
* @param columns * @param columns
* @param rows * @param rows
* @param options * @param sqlOptions
* @param sqlErrors * @param sqlErrors
*/ */
public SqlInsertBuilder(String table, List<String> columns, List<ArrayList<SqlData>> rows, JSONObject options public SqlInsertBuilder(String table, List<String> columns, List<ArrayList<SqlData>> rows, JsonNode sqlOptions
) { ) {
this.table = table; this.table = table;
this.columns = columns; this.columns = columns;
this.sqlDataList = rows; this.sqlDataList = rows;
this.options = options; this.options = sqlOptions;
//logger.info("Column Size:{}", columns.size()); //logger.info("Column Size:{}", columns.size());
} }
@ -83,12 +82,11 @@ public class SqlInsertBuilder {
logger.debug("Insert SQL with columns: {}", columns); logger.debug("Insert SQL with columns: {}", columns);
} }
JSONArray colOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns"); List<JsonNode> colOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns");
Map<String, JSONObject> colOptionsMap = new HashMap<String, JSONObject>(); Map<String, JsonNode> colOptionsMap = new HashMap<>();
if(colOptionArray != null) { if(colOptionArray != null) {
colOptionArray.forEach(c -> { colOptionArray.forEach(json -> {
JSONObject json = (JSONObject)c; colOptionsMap.put(JSONUtilities.getString(json, "name", null), json);
colOptionsMap.put("" + json.get("name"), json);
}); });
} }
@ -103,8 +101,8 @@ public class SqlInsertBuilder {
//int fieldCount = 0; //int fieldCount = 0;
for(SqlData val : sqlRow) { for(SqlData val : sqlRow) {
JSONObject jsonOb = colOptionsMap.get(val.getColumnName()); JsonNode jsonOb = colOptionsMap.get(val.getColumnName());
String type = (String)jsonOb.get("type"); String type = JSONUtilities.getString(jsonOb, "type", null);
String defaultValue = JSONUtilities.getString(jsonOb, "defaultValue", null); String defaultValue = JSONUtilities.getString(jsonOb, "defaultValue", null);

38
main/src/com/google/refine/util/JSONUtilities.java
View File

@ -47,8 +47,14 @@ import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.JsonNodeType;
import com.google.common.collect.Lists;
public class JSONUtilities { public class JSONUtilities {
static public JSONObject getObject(JSONObject obj, String key) { static public JSONObject getObject(JSONObject obj, String key) {
try { try {
return obj.getJSONObject(key); return obj.getJSONObject(key);
@ -65,6 +71,14 @@ public class JSONUtilities {
} }
} }
static public String getString(JsonNode obj, String key, String def) {
if (obj.get(key) != null) {
return obj.get(key).textValue();
} else {
return def;
}
}
static public int getInt(JSONObject obj, String key, int def) { static public int getInt(JSONObject obj, String key, int def) {
try { try {
return obj.getInt(key); return obj.getInt(key);
@ -73,6 +87,14 @@ public class JSONUtilities {
} }
} }
static public int getInt(JsonNode obj, String key, int def) {
if (obj.get(key) != null) {
return obj.get(key).asInt(def);
} else {
return def;
}
}
static public boolean getBoolean(JSONObject obj, String key, boolean def) { static public boolean getBoolean(JSONObject obj, String key, boolean def) {
try { try {
return obj.getBoolean(key); return obj.getBoolean(key);
@ -81,6 +103,14 @@ public class JSONUtilities {
} }
} }
static public boolean getBoolean(JsonNode obj, String key, boolean def) {
if (obj.get(key) != null) {
return obj.get(key).asBoolean(def);
} else {
return def;
}
}
static public double getDouble(JSONObject obj, String key, double def) { static public double getDouble(JSONObject obj, String key, double def) {
try { try {
return obj.getDouble(key); return obj.getDouble(key);
@ -125,6 +155,14 @@ public class JSONUtilities {
} }
} }
static public List<JsonNode> getArray(JsonNode obj, String key) {
if (obj.has(key) && obj.get(key).getNodeType().equals(JsonNodeType.ARRAY)) {
return Lists.newArrayList(obj.get(key).elements());
} else {
return null;
}
}
static public JSONArray arrayToJSONArray(String[] array) { static public JSONArray arrayToJSONArray(String[] array) {
return new JSONArray(Arrays.asList(array)); return new JSONArray(Arrays.asList(array));
} }

50
main/tests/server/src/com/google/refine/tests/exporters/sql/SqlExporterTests.java
View File

@ -54,6 +54,9 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest; import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine;
import com.google.refine.exporters.sql.SqlCreateBuilder; import com.google.refine.exporters.sql.SqlCreateBuilder;
@ -68,6 +71,8 @@ import com.google.refine.model.Row;
import com.google.refine.model.metadata.ProjectMetadata; import com.google.refine.model.metadata.ProjectMetadata;
import com.google.refine.tests.ProjectManagerStub; import com.google.refine.tests.ProjectManagerStub;
import com.google.refine.tests.RefineTest; import com.google.refine.tests.RefineTest;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities;
public class SqlExporterTests extends RefineTest { public class SqlExporterTests extends RefineTest {
@ -180,14 +185,10 @@ public class SqlExporterTests extends RefineTest {
} }
String result = writer.toString(); String result = writer.toString();
// logger.info("result = \n" + result);
Assert.assertNotNull(result); Assert.assertNotNull(result);
assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR); assertNotEquals(writer.toString(), SqlExporter.NO_OPTIONS_PRESENT_ERROR);
boolean checkResult = result.contains("CREATE TABLE " + tableName); Assert.assertTrue(result.contains("INSERT INTO " + tableName));
//logger.info("checkResult1 =" + checkResult);
checkResult = result.contains("INSERT INTO " + tableName);
// logger.info("checkResult2 =" + checkResult);
Assert.assertEquals(checkResult, true);
} }
@ -195,7 +196,7 @@ public class SqlExporterTests extends RefineTest {
public void testExportSqlNoSchema(){ public void testExportSqlNoSchema(){
createGrid(2, 2); createGrid(2, 2);
String tableName = "sql_table_test"; String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null,null); ObjectNode optionsJson = (ObjectNode) createOptionsFromProject(tableName, null,null);
optionsJson.put("includeStructure", false); optionsJson.put("includeStructure", false);
when(options.getProperty("options")).thenReturn(optionsJson.toString()); when(options.getProperty("options")).thenReturn(optionsJson.toString());
// logger.info("Options = " + optionsJson.toString()); // logger.info("Options = " + optionsJson.toString());
@ -222,7 +223,7 @@ public class SqlExporterTests extends RefineTest {
public void testExportSqlNoContent(){ public void testExportSqlNoContent(){
createGrid(2, 2); createGrid(2, 2);
String tableName = "sql_table_test"; String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null); ObjectNode optionsJson = (ObjectNode) createOptionsFromProject(tableName, null, null);
optionsJson.put("includeContent", false); optionsJson.put("includeContent", false);
when(options.getProperty("options")).thenReturn(optionsJson.toString()); when(options.getProperty("options")).thenReturn(optionsJson.toString());
//logger.info("Options = " + optionsJson.toString()); //logger.info("Options = " + optionsJson.toString());
@ -249,7 +250,7 @@ public class SqlExporterTests extends RefineTest {
public void testExportSqlIncludeSchemaWithDropStmt(){ public void testExportSqlIncludeSchemaWithDropStmt(){
createGrid(2, 2); createGrid(2, 2);
String tableName = "sql_table_test"; String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null); ObjectNode optionsJson = (ObjectNode) createOptionsFromProject(tableName, null, null);
optionsJson.put("includeStructure", true); optionsJson.put("includeStructure", true);
optionsJson.put("includeDropStatement", true); optionsJson.put("includeDropStatement", true);
@ -289,7 +290,7 @@ public class SqlExporterTests extends RefineTest {
String tableName = "sql_table_test"; String tableName = "sql_table_test";
String type = "CHAR"; String type = "CHAR";
String size = "2"; String size = "2";
JSONObject optionsJson = createOptionsFromProject(tableName, type, size); JsonNode optionsJson = createOptionsFromProject(tableName, type, size);
// logger.info("Options:: = " + optionsJson.toString()); // logger.info("Options:: = " + optionsJson.toString());
List<String> columns = project.columnModel.columns.stream().map(col -> col.getName()).collect(Collectors.toList()); List<String> columns = project.columnModel.columns.stream().map(col -> col.getName()).collect(Collectors.toList());
@ -307,7 +308,7 @@ public class SqlExporterTests extends RefineTest {
int inNull = 8; int inNull = 8;
createGridWithNullFields(3, 3, inNull); createGridWithNullFields(3, 3, inNull);
String tableName = "sql_table_test"; String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null); ObjectNode optionsJson = (ObjectNode) createOptionsFromProject(tableName, null, null);
optionsJson.put("includeStructure", true); optionsJson.put("includeStructure", true);
optionsJson.put("includeDropStatement", true); optionsJson.put("includeDropStatement", true);
optionsJson.put("convertNulltoEmptyString", true); optionsJson.put("convertNulltoEmptyString", true);
@ -338,7 +339,7 @@ public class SqlExporterTests extends RefineTest {
int noOfRows = 3; int noOfRows = 3;
createGrid(noOfRows, noOfCols); createGrid(noOfRows, noOfCols);
String tableName = "sql_table_test"; String tableName = "sql_table_test";
JSONObject optionsJson = createOptionsFromProject(tableName, null, null, null, false); ObjectNode optionsJson = createOptionsFromProject(tableName, null, null, null, false);
optionsJson.put("includeStructure", true); optionsJson.put("includeStructure", true);
optionsJson.put("includeDropStatement", true); optionsJson.put("includeDropStatement", true);
optionsJson.put("convertNulltoEmptyString", true); optionsJson.put("convertNulltoEmptyString", true);
@ -497,18 +498,16 @@ public class SqlExporterTests extends RefineTest {
return json; return json;
} }
protected JSONObject createOptionsFromProject(String tableName, String type, String size) { protected JsonNode createOptionsFromProject(String tableName, String type, String size) {
ObjectNode json = ParsingUtilities.mapper.createObjectNode();
JSONObject json = new JSONObject();
JSONArray columns = new JSONArray();
json.put("columns", columns);
json.put("tableName", tableName); json.put("tableName", tableName);
ArrayNode columns = json.putArray("columns");
List<Column> cols = project.columnModel.columns; List<Column> cols = project.columnModel.columns;
cols.forEach(c -> { cols.forEach(c -> {
//logger.info("Column Name = " + c.getName()); //logger.info("Column Name = " + c.getName());
JSONObject columnModel = new JSONObject(); ObjectNode columnModel = ParsingUtilities.mapper.createObjectNode();
columnModel.put("name", c.getName()); columnModel.put("name", c.getName());
if(type != null) { if(type != null) {
columnModel.put("type", type); columnModel.put("type", type);
@ -529,26 +528,25 @@ public class SqlExporterTests extends RefineTest {
columnModel.put("size", size); columnModel.put("size", size);
} }
columns.put(columnModel); columns.add(columnModel);
}); });
return json; return json;
} }
protected JSONObject createOptionsFromProject(String tableName, String type, String size, String defaultValue, protected ObjectNode createOptionsFromProject(String tableName, String type, String size, String defaultValue,
boolean allowNull) { boolean allowNull) {
JSONObject json = new JSONObject(); ObjectNode json = ParsingUtilities.mapper.createObjectNode();
JSONArray columns = new JSONArray(); ArrayNode columns = json.putArray("columns");
json.put("columns", columns);
json.put("tableName", tableName); json.put("tableName", tableName);
List<Column> cols = project.columnModel.columns; List<Column> cols = project.columnModel.columns;
cols.forEach(c -> { cols.forEach(c -> {
//logger.info("Column Name = " + c.getName()); //logger.info("Column Name = " + c.getName());
JSONObject columnModel = new JSONObject(); ObjectNode columnModel = ParsingUtilities.mapper.createObjectNode();
columnModel.put("name", c.getName()); columnModel.put("name", c.getName());
if(type != null) { if(type != null) {
columnModel.put("type", type); columnModel.put("type", type);
@ -572,7 +570,7 @@ public class SqlExporterTests extends RefineTest {
columnModel.put("defaultValue", defaultValue); columnModel.put("defaultValue", defaultValue);
columnModel.put("allowNull", allowNull); columnModel.put("allowNull", allowNull);
columns.put(columnModel); columns.add(columnModel);
}); });