Add locale support to toDate() - fixes #729

main/src/com/google/refine/expr/functions/ToDate.java

@@ -33,12 +33,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package com.google.refine.expr.functions;
 
+import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.GregorianCalendar;
+import java.util.Locale;
 import java.util.Properties;
+import java.util.Locale.Category;
 
+import org.apache.commons.lang.StringUtils;
 import org.json.JSONException;
 import org.json.JSONWriter;
 
@@ -99,29 +103,56 @@ public class ToDate implements Function {
         }
 
         // "o, format1, format2 (optional), ..."
+        Locale locale = Locale.getDefault(Category.FORMAT);
         if (args.length>=2) {
             for (int i=1;i<args.length;i++) {
                 if (!(args[i] instanceof String)) {
                     // skip formats that aren't strings
                     continue;
                 }
-                String format  = (String) args[i];
+                String format  = StringUtils.trim((String) args[i]);
-                SimpleDateFormat formatter;
+                DateFormat formatter;
+                // Attempt to parse first string as a language tag
+                if (i == 1) {
+                    // Locale possibleLocale = Locale.forLanguageTag(format); // Java 1.7+ only
+                    Locale possibleLocale;
+                    int c = format.indexOf('_');
+                    if (c > 0) {
+                        possibleLocale = new Locale(format.substring(0, c),format.substring(c+1));
+                    } else {
+                        possibleLocale = new Locale(format);
+                    }
+                    boolean valid = false;
+                    for (Locale l : DateFormat.getAvailableLocales()) {
+                        if (l.equals(possibleLocale)) {
+                            locale = possibleLocale;
+                            valid = true;
+                            break;
+                        }
+                    }
+                    if (valid) { // If we got a valid locale
+                        if (args.length == 2) { // No format strings to try, process using default
+                          formatter = DateFormat.getDateInstance(DateFormat.DEFAULT, locale);
+                          formatter.setLenient(true);
+                          GregorianCalendar date = parse(o1, formatter);
+                          if (date != null) {
+                              return date;
+                          } else {
+                              return new EvalError("Unable to parse as date");
+                          }
+                        }
+                        continue; // Don't try to process locale string as a format string if it was valid
+                    }
+                }
                 try {
-                    formatter = new SimpleDateFormat(format);
+                    formatter = new SimpleDateFormat(format,locale);
                 } catch (IllegalArgumentException e) {
                     return new EvalError("Unknown date format");
                 }
-                Date date = null;
+                formatter.setLenient(true);
-                try {
+                GregorianCalendar date = parse(o1, formatter);
-                    date = formatter.parse(o1);
-                } catch (java.text.ParseException e) {
-                    continue;
-                }
                 if (date != null) {
-                    GregorianCalendar c = new GregorianCalendar();
+                    return date;
-                    c.setTime(date);
-                    return c;
                 }
             }
             return new EvalError("Unable to parse as date");
@@ -131,6 +162,18 @@ public class ToDate implements Function {
     }
 
 
+    private GregorianCalendar parse(String o1, DateFormat formatter) {
+        try {
+            Date date = formatter.parse(o1);
+            GregorianCalendar c = new GregorianCalendar();
+            c.setTime(date);
+            return c;
+        } catch (java.text.ParseException e) {
+            return null;
+        }
+    }
+
+
     @Override
     public void write(JSONWriter writer, Properties options)
     throws JSONException {

main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java

@@ -131,6 +131,13 @@ public class ToFromConversionTests extends RefineTest {
       Assert.assertEquals(invoke("toDate", "2012-03-01","yyyy-MM-dd"),CalendarParser.parse("2012-03-01"));
       // Multiple format strings should get tried sequentially until one succeeds or all are exhausted
       Assert.assertEquals(invoke("toDate", "2012-03-01","MMM","yyyy-MM-dd"), CalendarParser.parse("2012-03-01"));
+      // First string can be a locale identifier instead of a format string
+      Assert.assertEquals(invoke("toDate", "2013-06-01","zh"), CalendarParser.parse("2013-06-01")); 
+      Assert.assertEquals(invoke("toDate", "01-六月-2013","zh","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01")); 
+      Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_HK","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01")); 
+      Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_TW","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01")); 
+      Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_CN","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01")); 
+
         // Date
         // Calendar
         // String