From 54f177cacc2ef206259732b23d0fd8e68199ebc1 Mon Sep 17 00:00:00 2001 From: Kelven Yang Date: Tue, 30 Nov 2010 09:40:35 -0800 Subject: [PATCH] 1) Make generic dao be able to persist UTF-8 string for internationalization support 2) Undo gson Unicode escape in API response object to avoid double escaping which can break Javascript from getting correct text content 3) Correct API layer in dealing with character encoding 4) Remove double escape in cloud.core.js --- core/.classpath | 2 +- server/.classpath | 4 +- server/src/com/cloud/api/ApiGsonHelper.java | 1 + server/src/com/cloud/api/ApiServer.java | 14 +++--- server/src/com/cloud/api/ApiServlet.java | 12 ++--- .../api/response/ApiResponseSerializer.java | 19 ++++++-- .../cloud/servlet/ConsoleProxyServlet.java | 7 +-- ui/scripts/cloud.core.js | 4 +- .../com/cloud/utils/db/GenericDaoBase.java | 44 +++++++++++++++++-- 9 files changed, 77 insertions(+), 30 deletions(-) diff --git a/core/.classpath b/core/.classpath index 17a6103c14d..1867836084a 100644 --- a/core/.classpath +++ b/core/.classpath @@ -11,7 +11,6 @@ - @@ -39,5 +38,6 @@ + diff --git a/server/.classpath b/server/.classpath index 8a0c5b39e0c..fe6302cf152 100644 --- a/server/.classpath +++ b/server/.classpath @@ -13,13 +13,13 @@ - - + + diff --git a/server/src/com/cloud/api/ApiGsonHelper.java b/server/src/com/cloud/api/ApiGsonHelper.java index 9877d27bdba..4e11d4b1977 100644 --- a/server/src/com/cloud/api/ApiGsonHelper.java +++ b/server/src/com/cloud/api/ApiGsonHelper.java @@ -14,3 +14,4 @@ public class ApiGsonHelper { return s_gBuilder; } } + diff --git a/server/src/com/cloud/api/ApiServer.java b/server/src/com/cloud/api/ApiServer.java index 7613b6db72a..0120fbf02c5 100755 --- a/server/src/com/cloud/api/ApiServer.java +++ b/server/src/com/cloud/api/ApiServer.java @@ -274,9 +274,9 @@ public class ApiServer implements HttpRequestHandler { response.setStatusCode(se.getErrorCode()); response.setReasonPhrase(se.getDescription()); BasicHttpEntity body = new BasicHttpEntity(); - body.setContentType("text/xml; charset=UTF-8"); + body.setContentType("text/xml"); String responseStr = ""+se.getErrorCode()+" : "+se.getDescription()+""; - body.setContent(new ByteArrayInputStream(responseStr.getBytes())); + body.setContent(new ByteArrayInputStream(responseStr.getBytes("UTF-8"))); response.setEntity(body); sb.append(" " + se.getErrorCode() + " " + responseStr.length()); @@ -714,19 +714,19 @@ public class ApiServer implements HttpRequestHandler { BasicHttpEntity body = new BasicHttpEntity(); if (BaseCmd.RESPONSE_TYPE_JSON.equalsIgnoreCase(responseType)) { // JSON response - body.setContentType("text/javascript; charset=UTF-8"); + body.setContentType("text/javascript"); if (responseText == null) { - body.setContent(new ByteArrayInputStream("{ \"error\" : { \"description\" : \"Internal Server Error\" } }".getBytes())); + body.setContent(new ByteArrayInputStream("{ \"error\" : { \"description\" : \"Internal Server Error\" } }".getBytes("UTF-8"))); } } else { - body.setContentType("text/xml; charset=UTF-8"); + body.setContentType("text/xml"); if (responseText == null) { - body.setContent(new ByteArrayInputStream("Internal Server Error".getBytes())); + body.setContent(new ByteArrayInputStream("Internal Server Error".getBytes("UTF-8"))); } } if (responseText != null) { - body.setContent(new ByteArrayInputStream(responseText.getBytes())); + body.setContent(new ByteArrayInputStream(responseText.getBytes("UTF-8"))); } resp.setEntity(body); } catch (Exception ex) { diff --git a/server/src/com/cloud/api/ApiServlet.java b/server/src/com/cloud/api/ApiServlet.java index 1e6e87eba11..0ba8243be90 100755 --- a/server/src/com/cloud/api/ApiServlet.java +++ b/server/src/com/cloud/api/ApiServlet.java @@ -320,14 +320,10 @@ public class ApiServlet extends HttpServlet { } else { resp.setContentType("text/xml"); } - resp.setStatus(isError? HttpServletResponse.SC_INTERNAL_SERVER_ERROR : HttpServletResponse.SC_OK); - byte[] respBytes = response.getBytes(); - resp.setContentLength(respBytes.length); - OutputStream os = resp.getOutputStream(); - os.write(respBytes); - os.flush(); - os.close(); - resp.flushBuffer(); + resp.setStatus(isError? HttpServletResponse.SC_INTERNAL_SERVER_ERROR : HttpServletResponse.SC_OK); + + // use getWriter() instead of manually manipulate encoding to have better localization support + resp.getWriter().print(response); } catch (IOException ioex) { if (s_logger.isTraceEnabled()) { s_logger.trace("exception writing response: " + ioex); diff --git a/server/src/com/cloud/api/response/ApiResponseSerializer.java b/server/src/com/cloud/api/response/ApiResponseSerializer.java index ca1852eaaf6..85e60baa3d6 100644 --- a/server/src/com/cloud/api/response/ApiResponseSerializer.java +++ b/server/src/com/cloud/api/response/ApiResponseSerializer.java @@ -4,10 +4,10 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.util.ArrayList; import java.util.Date; -import java.util.LinkedList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.log4j.Logger; @@ -28,6 +28,16 @@ public class ApiResponseSerializer { return toXMLSerializedString(result); } } + + private static final Pattern s_unicodeEscapePattern = Pattern.compile("\\\\u([0-9A-Fa-f]{4})"); + public static String unescape(String escaped) { + String str = escaped; + Matcher matcher = s_unicodeEscapePattern.matcher(str); + while(matcher.find()) { + str = str.replaceAll("\\" + matcher.group(0), Character.toString((char)Integer.parseInt(matcher.group(1), 16))); + } + return str; + } private static String toJSONSerializedString(ResponseObject result) { if (result != null) { @@ -39,10 +49,12 @@ public class ApiResponseSerializer { List responses = ((ListResponse)result).getResponses(); if ((responses != null) && !responses.isEmpty()) { int count = responses.size(); - String jsonStr = gson.toJson(responses.get(0)); + String jsonStr = gson.toJson(responses.get(0));; + jsonStr = unescape(jsonStr); sb.append("{ \"" + responses.get(0).getObjectName() + "\" : [ " + jsonStr); for (int i = 1; i < count; i++) { jsonStr = gson.toJson(responses.get(i)); + jsonStr = unescape(jsonStr); sb.append(", " + jsonStr); } sb.append(" ] }"); @@ -54,6 +66,7 @@ public class ApiResponseSerializer { } else { String jsonStr = gson.toJson(result); if ((jsonStr != null) && !"".equals(jsonStr)) { + jsonStr = unescape(jsonStr); if (result instanceof AsyncJobResponse || result instanceof CreateCmdResponse) { sb.append(jsonStr); } else { diff --git a/server/src/com/cloud/servlet/ConsoleProxyServlet.java b/server/src/com/cloud/servlet/ConsoleProxyServlet.java index b43bae7fdb7..7876a343a2b 100644 --- a/server/src/com/cloud/servlet/ConsoleProxyServlet.java +++ b/server/src/com/cloud/servlet/ConsoleProxyServlet.java @@ -66,15 +66,16 @@ public class ConsoleProxyServlet extends HttpServlet { } @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) { + protected void doGet(HttpServletRequest req, HttpServletResponse resp) { + try { String userId = null; String account = null; Account accountObj = null; - + Map params = new HashMap(); params.putAll(req.getParameterMap()); - + HttpSession session = req.getSession(false); if(session == null) { if(verifyRequest(params)) { diff --git a/ui/scripts/cloud.core.js b/ui/scripts/cloud.core.js index fff1902cc93..aa0637821d1 100644 --- a/ui/scripts/cloud.core.js +++ b/ui/scripts/cloud.core.js @@ -537,11 +537,11 @@ function createURL(url) { } function fromdb(val) { - return sanitizeXSS(unescape(noNull(val))); + return sanitizeXSS(noNull(val)); } function todb(val) { - return encodeURIComponent(escape(val)); + return encodeURIComponent(val); } var midmenuItemCount = 20; diff --git a/utils/src/com/cloud/utils/db/GenericDaoBase.java b/utils/src/com/cloud/utils/db/GenericDaoBase.java index dfa6d72c78a..a5dab05daa9 100755 --- a/utils/src/com/cloud/utils/db/GenericDaoBase.java +++ b/utils/src/com/cloud/utils/db/GenericDaoBase.java @@ -18,6 +18,7 @@ package com.cloud.utils.db; import java.io.Serializable; +import java.io.UnsupportedEncodingException; import java.lang.reflect.Field; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; @@ -448,7 +449,20 @@ public abstract class GenericDaoBase implements Gene try { final Class type = field.getType(); if (type == String.class) { - field.set(entity, rs.getString(index)); + byte[] bytes = rs.getBytes(index); + if(bytes != null) { + try { + field.set(entity, new String(bytes, "UTF-8")); + } catch (IllegalArgumentException e) { + assert(false); + throw new CloudRuntimeException("IllegalArgumentException when converting UTF-8 data"); + } catch (UnsupportedEncodingException e) { + assert(false); + throw new CloudRuntimeException("UnsupportedEncodingException when converting UTF-8 data"); + } + } else { + field.set(entity, null); + } } else if (type == long.class) { field.setLong(entity, rs.getLong(index)); } else if (type == Long.class) { @@ -560,7 +574,16 @@ public abstract class GenericDaoBase implements Gene @DB(txn=false) @SuppressWarnings("unchecked") protected M getObject(Class type, ResultSet rs, int index) throws SQLException { if (type == String.class) { - return (M)rs.getString(index); + byte[] bytes = rs.getBytes(index); + if(bytes != null) { + try { + return (M)new String(bytes, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new CloudRuntimeException("UnsupportedEncodingException exception while converting UTF-8 data"); + } + } else { + return (M)null; + } } else if (type == int.class) { return (M)new Integer(rs.getInt(index)); } else if (type == Integer.class) { @@ -1197,10 +1220,23 @@ public abstract class GenericDaoBase implements Gene final Column column = attr.field.getAnnotation(Column.class); final int length = column != null ? column.length() : 255; + // to support generic localization, utilize MySql UTF-8 support if (length < str.length()) { - pstmt.setString(j, str.substring(0, column.length())); + try { + pstmt.setBytes(j, str.substring(0, column.length()).getBytes("UTF-8")); + } catch (UnsupportedEncodingException e) { + // no-way it can't support UTF-8 encoding + assert(false); + throw new CloudRuntimeException("UnsupportedEncodingException when saving string as UTF-8 data"); + } } else { - pstmt.setString(j, str); + try { + pstmt.setBytes(j, str.getBytes("UTF-8")); + } catch (UnsupportedEncodingException e) { + // no-way it can't support UTF-8 encoding + assert(false); + throw new CloudRuntimeException("UnsupportedEncodingException when saving string as UTF-8 data"); + } } } else if (attr.field.getType() == Date.class) { final Date date = (Date)value;