1 : /* ***** BEGIN LICENSE BLOCK *****
2 : * Version: MPL 1.1/GPL 2.0/LGPL 2.1
3 : *
4 : * The contents of this file are subject to the Mozilla Public License Version
5 : * 1.1 (the "License"); you may not use this file except in compliance with
6 : * the License. You may obtain a copy of the License at
7 : * http://www.mozilla.org/MPL/
8 : *
9 : * Software distributed under the License is distributed on an "AS IS" basis,
10 : * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
11 : * for the specific language governing rights and limitations under the
12 : * License.
13 : *
14 : * The Original Code is Mozilla.
15 : *
16 : * The Initial Developer of the Original Code is
17 : * Netscape Communications Corporation.
18 : * Portions created by the Initial Developer are Copyright (C) 2002
19 : * the Initial Developer. All Rights Reserved.
20 : *
21 : * Contributor(s):
22 : * Darin Fisher <darin@netscape.com>
23 : * Brian Stell <bstell@ix.netcom.com>
24 : * Frank Tang <ftang@netscape.com>
25 : * Brendan Eich <brendan@mozilla.org>
26 : * Sergei Dolgov <sergei_d@fi.fi.tartu.ee>
27 : * Jungshik Shin <jshin@i18nl10n.com>
28 : *
29 : * Alternatively, the contents of this file may be used under the terms of
30 : * either the GNU General Public License Version 2 or later (the "GPL"), or
31 : * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
32 : * in which case the provisions of the GPL or the LGPL are applicable instead
33 : * of those above. If you wish to allow use of your version of this file only
34 : * under the terms of either the GPL or the LGPL, and not to allow others to
35 : * use your version of this file under the terms of the MPL, indicate your
36 : * decision by deleting the provisions above and replace them with the notice
37 : * and other provisions required by the GPL or the LGPL. If you do not delete
38 : * the provisions above, a recipient may use your version of this file under
39 : * the terms of any one of the MPL, the GPL or the LGPL.
40 : *
41 : * ***** END LICENSE BLOCK ***** */
42 :
43 : #include "xpcom-private.h"
44 :
45 : //-----------------------------------------------------------------------------
46 : // XP_MACOSX or ANDROID
47 : //-----------------------------------------------------------------------------
48 : #if defined(XP_MACOSX) || defined(ANDROID)
49 :
50 : #include "nsAString.h"
51 : #include "nsReadableUtils.h"
52 : #include "nsString.h"
53 :
54 : nsresult
55 : NS_CopyNativeToUnicode(const nsACString &input, nsAString &output)
56 : {
57 : CopyUTF8toUTF16(input, output);
58 : return NS_OK;
59 : }
60 :
61 : nsresult
62 : NS_CopyUnicodeToNative(const nsAString &input, nsACString &output)
63 : {
64 : CopyUTF16toUTF8(input, output);
65 : return NS_OK;
66 : }
67 :
68 : void
69 : NS_StartupNativeCharsetUtils()
70 : {
71 : }
72 :
73 : void
74 : NS_ShutdownNativeCharsetUtils()
75 : {
76 : }
77 :
78 :
79 : //-----------------------------------------------------------------------------
80 : // XP_UNIX
81 : //-----------------------------------------------------------------------------
82 : #elif defined(XP_UNIX)
83 :
84 : #include <stdlib.h> // mbtowc, wctomb
85 : #include <locale.h> // setlocale
86 : #include "mozilla/Mutex.h"
87 : #include "nscore.h"
88 : #include "nsAString.h"
89 : #include "nsReadableUtils.h"
90 :
91 : using namespace mozilla;
92 :
93 : //
94 : // choose a conversion library. we used to use mbrtowc/wcrtomb under Linux,
95 : // but that doesn't work for non-BMP characters whether we use '-fshort-wchar'
96 : // or not (see bug 206811 and
97 : // news://news.mozilla.org:119/bajml3$fvr1@ripley.netscape.com). we now use
98 : // iconv for all platforms where nltypes.h and nllanginfo.h are present
99 : // along with iconv.
100 : //
101 : #if defined(HAVE_ICONV) && defined(HAVE_NL_TYPES_H) && defined(HAVE_LANGINFO_CODESET)
102 : #define USE_ICONV 1
103 : #else
104 : #define USE_STDCONV 1
105 : #endif
106 :
107 : static void
108 0 : isolatin1_to_utf16(const char **input, PRUint32 *inputLeft, PRUnichar **output, PRUint32 *outputLeft)
109 : {
110 0 : while (*inputLeft && *outputLeft) {
111 0 : **output = (unsigned char) **input;
112 0 : (*input)++;
113 0 : (*inputLeft)--;
114 0 : (*output)++;
115 0 : (*outputLeft)--;
116 : }
117 0 : }
118 :
119 : static void
120 0 : utf16_to_isolatin1(const PRUnichar **input, PRUint32 *inputLeft, char **output, PRUint32 *outputLeft)
121 : {
122 0 : while (*inputLeft && *outputLeft) {
123 0 : **output = (unsigned char) **input;
124 0 : (*input)++;
125 0 : (*inputLeft)--;
126 0 : (*output)++;
127 0 : (*outputLeft)--;
128 : }
129 0 : }
130 :
131 : //-----------------------------------------------------------------------------
132 : // conversion using iconv
133 : //-----------------------------------------------------------------------------
134 : #if defined(USE_ICONV)
135 : #include <nl_types.h> // CODESET
136 : #include <langinfo.h> // nl_langinfo
137 : #include <iconv.h> // iconv_open, iconv, iconv_close
138 : #include <errno.h>
139 : #include "plstr.h"
140 :
141 : #if defined(HAVE_ICONV_WITH_CONST_INPUT)
142 : #define ICONV_INPUT(x) (x)
143 : #else
144 : #define ICONV_INPUT(x) ((char **)x)
145 : #endif
146 :
147 : // solaris definitely needs this, but we'll enable it by default
148 : // just in case... but we know for sure that iconv(3) in glibc
149 : // doesn't need this.
150 : #if !defined(__GLIBC__)
151 : #define ENABLE_UTF8_FALLBACK_SUPPORT
152 : #endif
153 :
154 : #define INVALID_ICONV_T ((iconv_t) -1)
155 :
156 : static inline size_t
157 1332325 : xp_iconv(iconv_t converter,
158 : const char **input,
159 : size_t *inputLeft,
160 : char **output,
161 : size_t *outputLeft)
162 : {
163 1332325 : size_t res, outputAvail = outputLeft ? *outputLeft : 0;
164 1332325 : res = iconv(converter, ICONV_INPUT(input), inputLeft, output, outputLeft);
165 1332325 : if (res == (size_t) -1) {
166 : // on some platforms (e.g., linux) iconv will fail with
167 : // E2BIG if it cannot convert _all_ of its input. it'll
168 : // still adjust all of the in/out params correctly, so we
169 : // can ignore this error. the assumption is that we will
170 : // be called again to complete the conversion.
171 0 : if ((errno == E2BIG) && (*outputLeft < outputAvail))
172 0 : res = 0;
173 : }
174 1332325 : return res;
175 : }
176 :
177 : static inline void
178 888094 : xp_iconv_reset(iconv_t converter)
179 : {
180 : // NOTE: the man pages on Solaris claim that you can pass NULL
181 : // for all parameter to reset the converter, but beware the
182 : // evil Solaris crash if you go down this route >:-)
183 :
184 888094 : const char *zero_char_in_ptr = NULL;
185 888094 : char *zero_char_out_ptr = NULL;
186 888094 : size_t zero_size_in = 0,
187 888094 : zero_size_out = 0;
188 :
189 : xp_iconv(converter, &zero_char_in_ptr,
190 : &zero_size_in,
191 : &zero_char_out_ptr,
192 888094 : &zero_size_out);
193 888094 : }
194 :
195 : static inline iconv_t
196 2730 : xp_iconv_open(const char **to_list, const char **from_list)
197 : {
198 : iconv_t res;
199 : const char **from_name;
200 : const char **to_name;
201 :
202 : // try all possible combinations to locate a converter.
203 2730 : to_name = to_list;
204 5460 : while (*to_name) {
205 2730 : if (**to_name) {
206 2730 : from_name = from_list;
207 5460 : while (*from_name) {
208 2730 : if (**from_name) {
209 2730 : res = iconv_open(*to_name, *from_name);
210 2730 : if (res != INVALID_ICONV_T)
211 2730 : return res;
212 : }
213 0 : from_name++;
214 : }
215 : }
216 0 : to_name++;
217 : }
218 :
219 0 : return INVALID_ICONV_T;
220 : }
221 :
222 : /*
223 : * PRUnichar[] is NOT a UCS-2 array BUT a UTF-16 string. Therefore, we
224 : * have to use UTF-16 with iconv(3) on platforms where it's supported.
225 : * However, the way UTF-16 and UCS-2 are interpreted varies across platforms
226 : * and implementations of iconv(3). On Tru64, it also depends on the environment
227 : * variable. To avoid the trouble arising from byte-swapping
228 : * (bug 208809), we have to try UTF-16LE/BE and UCS-2LE/BE before falling
229 : * back to UTF-16 and UCS-2 and variants. We assume that UTF-16 and UCS-2
230 : * on systems without UTF-16LE/BE and UCS-2LE/BE have the native endianness,
231 : * which isn't the case of glibc 2.1.x, for which we use 'UNICODELITTLE'
232 : * and 'UNICODEBIG'. It's also not true of Tru64 V4 when the environment
233 : * variable ICONV_BYTEORDER is set to 'big-endian', about which not much
234 : * can be done other than adding a note in the release notes. (bug 206811)
235 : */
236 : static const char *UTF_16_NAMES[] = {
237 : #if defined(IS_LITTLE_ENDIAN)
238 : "UTF-16LE",
239 : #if defined(__GLIBC__)
240 : "UNICODELITTLE",
241 : #endif
242 : "UCS-2LE",
243 : #else
244 : "UTF-16BE",
245 : #if defined(__GLIBC__)
246 : "UNICODEBIG",
247 : #endif
248 : "UCS-2BE",
249 : #endif
250 : "UTF-16",
251 : "UCS-2",
252 : "UCS2",
253 : "UCS_2",
254 : "ucs-2",
255 : "ucs2",
256 : "ucs_2",
257 : NULL
258 : };
259 :
260 : #if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
261 : static const char *UTF_8_NAMES[] = {
262 : "UTF-8",
263 : "UTF8",
264 : "UTF_8",
265 : "utf-8",
266 : "utf8",
267 : "utf_8",
268 : NULL
269 : };
270 : #endif
271 :
272 : static const char *ISO_8859_1_NAMES[] = {
273 : "ISO-8859-1",
274 : #if !defined(__GLIBC__)
275 : "ISO8859-1",
276 : "ISO88591",
277 : "ISO_8859_1",
278 : "ISO8859_1",
279 : "iso-8859-1",
280 : "iso8859-1",
281 : "iso88591",
282 : "iso_8859_1",
283 : "iso8859_1",
284 : #endif
285 : NULL
286 : };
287 :
288 : class nsNativeCharsetConverter
289 : {
290 : public:
291 : nsNativeCharsetConverter();
292 : ~nsNativeCharsetConverter();
293 :
294 : nsresult NativeToUnicode(const char **input , PRUint32 *inputLeft,
295 : PRUnichar **output, PRUint32 *outputLeft);
296 : nsresult UnicodeToNative(const PRUnichar **input , PRUint32 *inputLeft,
297 : char **output, PRUint32 *outputLeft);
298 :
299 : static void GlobalInit();
300 : static void GlobalShutdown();
301 : static bool IsNativeUTF8();
302 :
303 : private:
304 : static iconv_t gNativeToUnicode;
305 : static iconv_t gUnicodeToNative;
306 : #if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
307 : static iconv_t gNativeToUTF8;
308 : static iconv_t gUTF8ToNative;
309 : static iconv_t gUnicodeToUTF8;
310 : static iconv_t gUTF8ToUnicode;
311 : #endif
312 : static Mutex *gLock;
313 : static bool gInitialized;
314 : static bool gIsNativeUTF8;
315 :
316 : static void LazyInit();
317 :
318 444047 : static void Lock() { if (gLock) gLock->Lock(); }
319 444047 : static void Unlock() { if (gLock) gLock->Unlock(); }
320 : };
321 :
322 : iconv_t nsNativeCharsetConverter::gNativeToUnicode = INVALID_ICONV_T;
323 : iconv_t nsNativeCharsetConverter::gUnicodeToNative = INVALID_ICONV_T;
324 : #if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
325 : iconv_t nsNativeCharsetConverter::gNativeToUTF8 = INVALID_ICONV_T;
326 : iconv_t nsNativeCharsetConverter::gUTF8ToNative = INVALID_ICONV_T;
327 : iconv_t nsNativeCharsetConverter::gUnicodeToUTF8 = INVALID_ICONV_T;
328 : iconv_t nsNativeCharsetConverter::gUTF8ToUnicode = INVALID_ICONV_T;
329 : #endif
330 : Mutex *nsNativeCharsetConverter::gLock = nsnull;
331 : bool nsNativeCharsetConverter::gInitialized = false;
332 : bool nsNativeCharsetConverter::gIsNativeUTF8 = false;
333 :
334 : void
335 1365 : nsNativeCharsetConverter::LazyInit()
336 : {
337 1365 : const char *blank_list[] = { "", NULL };
338 1365 : const char **native_charset_list = blank_list;
339 1365 : const char *native_charset = nl_langinfo(CODESET);
340 1365 : if (native_charset == nsnull) {
341 0 : NS_ERROR("native charset is unknown");
342 : // fallback to ISO-8859-1
343 0 : native_charset_list = ISO_8859_1_NAMES;
344 : }
345 : else
346 1365 : native_charset_list[0] = native_charset;
347 :
348 : // Most, if not all, Unixen supporting UTF-8 and nl_langinfo(CODESET)
349 : // return 'UTF-8' (or 'utf-8')
350 1365 : if (!PL_strcasecmp(native_charset, "UTF-8"))
351 1365 : gIsNativeUTF8 = true;
352 :
353 1365 : gNativeToUnicode = xp_iconv_open(UTF_16_NAMES, native_charset_list);
354 1365 : gUnicodeToNative = xp_iconv_open(native_charset_list, UTF_16_NAMES);
355 :
356 : #if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
357 : if (gNativeToUnicode == INVALID_ICONV_T) {
358 : gNativeToUTF8 = xp_iconv_open(UTF_8_NAMES, native_charset_list);
359 : gUTF8ToUnicode = xp_iconv_open(UTF_16_NAMES, UTF_8_NAMES);
360 : NS_ASSERTION(gNativeToUTF8 != INVALID_ICONV_T, "no native to utf-8 converter");
361 : NS_ASSERTION(gUTF8ToUnicode != INVALID_ICONV_T, "no utf-8 to utf-16 converter");
362 : }
363 : if (gUnicodeToNative == INVALID_ICONV_T) {
364 : gUnicodeToUTF8 = xp_iconv_open(UTF_8_NAMES, UTF_16_NAMES);
365 : gUTF8ToNative = xp_iconv_open(native_charset_list, UTF_8_NAMES);
366 : NS_ASSERTION(gUnicodeToUTF8 != INVALID_ICONV_T, "no utf-16 to utf-8 converter");
367 : NS_ASSERTION(gUTF8ToNative != INVALID_ICONV_T, "no utf-8 to native converter");
368 : }
369 : #else
370 1365 : NS_ASSERTION(gNativeToUnicode != INVALID_ICONV_T, "no native to utf-16 converter");
371 1365 : NS_ASSERTION(gUnicodeToNative != INVALID_ICONV_T, "no utf-16 to native converter");
372 : #endif
373 :
374 : /*
375 : * On Solaris 8 (and newer?), the iconv modules converting to UCS-2
376 : * prepend a byte order mark unicode character (BOM, u+FEFF) during
377 : * the first use of the iconv converter. The same is the case of
378 : * glibc 2.2.9x and Tru64 V5 (see bug 208809) when 'UTF-16' is used.
379 : * However, we use 'UTF-16LE/BE' in both cases, instead so that we
380 : * should be safe. But just in case...
381 : *
382 : * This dummy conversion gets rid of the BOMs and fixes bug 153562.
383 : */
384 1365 : char dummy_input[1] = { ' ' };
385 : char dummy_output[4];
386 :
387 1365 : if (gNativeToUnicode != INVALID_ICONV_T) {
388 1365 : const char *input = dummy_input;
389 1365 : size_t input_left = sizeof(dummy_input);
390 1365 : char *output = dummy_output;
391 1365 : size_t output_left = sizeof(dummy_output);
392 :
393 1365 : xp_iconv(gNativeToUnicode, &input, &input_left, &output, &output_left);
394 : }
395 : #if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
396 : if (gUTF8ToUnicode != INVALID_ICONV_T) {
397 : const char *input = dummy_input;
398 : size_t input_left = sizeof(dummy_input);
399 : char *output = dummy_output;
400 : size_t output_left = sizeof(dummy_output);
401 :
402 : xp_iconv(gUTF8ToUnicode, &input, &input_left, &output, &output_left);
403 : }
404 : #endif
405 :
406 1365 : gInitialized = true;
407 1365 : }
408 :
409 : void
410 1365 : nsNativeCharsetConverter::GlobalInit()
411 : {
412 1365 : gLock = new Mutex("nsNativeCharsetConverter.gLock");
413 1365 : }
414 :
415 : void
416 1365 : nsNativeCharsetConverter::GlobalShutdown()
417 : {
418 1365 : if (gLock) {
419 1365 : delete gLock;
420 1365 : gLock = nsnull;
421 : }
422 :
423 1365 : if (gNativeToUnicode != INVALID_ICONV_T) {
424 1365 : iconv_close(gNativeToUnicode);
425 1365 : gNativeToUnicode = INVALID_ICONV_T;
426 : }
427 :
428 1365 : if (gUnicodeToNative != INVALID_ICONV_T) {
429 1365 : iconv_close(gUnicodeToNative);
430 1365 : gUnicodeToNative = INVALID_ICONV_T;
431 : }
432 :
433 : #if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
434 : if (gNativeToUTF8 != INVALID_ICONV_T) {
435 : iconv_close(gNativeToUTF8);
436 : gNativeToUTF8 = INVALID_ICONV_T;
437 : }
438 : if (gUTF8ToNative != INVALID_ICONV_T) {
439 : iconv_close(gUTF8ToNative);
440 : gUTF8ToNative = INVALID_ICONV_T;
441 : }
442 : if (gUnicodeToUTF8 != INVALID_ICONV_T) {
443 : iconv_close(gUnicodeToUTF8);
444 : gUnicodeToUTF8 = INVALID_ICONV_T;
445 : }
446 : if (gUTF8ToUnicode != INVALID_ICONV_T) {
447 : iconv_close(gUTF8ToUnicode);
448 : gUTF8ToUnicode = INVALID_ICONV_T;
449 : }
450 : #endif
451 :
452 1365 : gInitialized = false;
453 1365 : }
454 :
455 444047 : nsNativeCharsetConverter::nsNativeCharsetConverter()
456 : {
457 444047 : Lock();
458 444047 : if (!gInitialized)
459 1365 : LazyInit();
460 444047 : }
461 :
462 444047 : nsNativeCharsetConverter::~nsNativeCharsetConverter()
463 : {
464 : // reset converters for next time
465 444047 : if (gNativeToUnicode != INVALID_ICONV_T)
466 444047 : xp_iconv_reset(gNativeToUnicode);
467 444047 : if (gUnicodeToNative != INVALID_ICONV_T)
468 444047 : xp_iconv_reset(gUnicodeToNative);
469 : #if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
470 : if (gNativeToUTF8 != INVALID_ICONV_T)
471 : xp_iconv_reset(gNativeToUTF8);
472 : if (gUTF8ToNative != INVALID_ICONV_T)
473 : xp_iconv_reset(gUTF8ToNative);
474 : if (gUnicodeToUTF8 != INVALID_ICONV_T)
475 : xp_iconv_reset(gUnicodeToUTF8);
476 : if (gUTF8ToUnicode != INVALID_ICONV_T)
477 : xp_iconv_reset(gUTF8ToUnicode);
478 : #endif
479 444047 : Unlock();
480 444047 : }
481 :
482 : nsresult
483 215382 : nsNativeCharsetConverter::NativeToUnicode(const char **input,
484 : PRUint32 *inputLeft,
485 : PRUnichar **output,
486 : PRUint32 *outputLeft)
487 : {
488 215382 : size_t res = 0;
489 215382 : size_t inLeft = (size_t) *inputLeft;
490 215382 : size_t outLeft = (size_t) *outputLeft * 2;
491 :
492 215382 : if (gNativeToUnicode != INVALID_ICONV_T) {
493 :
494 215382 : res = xp_iconv(gNativeToUnicode, input, &inLeft, (char **) output, &outLeft);
495 :
496 215382 : *inputLeft = inLeft;
497 215382 : *outputLeft = outLeft / 2;
498 215382 : if (res != (size_t) -1)
499 215382 : return NS_OK;
500 :
501 0 : NS_WARNING("conversion from native to utf-16 failed");
502 :
503 : // reset converter
504 0 : xp_iconv_reset(gNativeToUnicode);
505 : }
506 : #if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
507 : else if ((gNativeToUTF8 != INVALID_ICONV_T) &&
508 : (gUTF8ToUnicode != INVALID_ICONV_T)) {
509 : // convert first to UTF8, then from UTF8 to UCS2
510 : const char *in = *input;
511 :
512 : char ubuf[1024];
513 :
514 : // we assume we're always called with enough space in |output|,
515 : // so convert many chars at a time...
516 : while (inLeft) {
517 : char *p = ubuf;
518 : size_t n = sizeof(ubuf);
519 : res = xp_iconv(gNativeToUTF8, &in, &inLeft, &p, &n);
520 : if (res == (size_t) -1) {
521 : NS_ERROR("conversion from native to utf-8 failed");
522 : break;
523 : }
524 : NS_ASSERTION(outLeft > 0, "bad assumption");
525 : p = ubuf;
526 : n = sizeof(ubuf) - n;
527 : res = xp_iconv(gUTF8ToUnicode, (const char **) &p, &n, (char **) output, &outLeft);
528 : if (res == (size_t) -1) {
529 : NS_ERROR("conversion from utf-8 to utf-16 failed");
530 : break;
531 : }
532 : }
533 :
534 : (*input) += (*inputLeft - inLeft);
535 : *inputLeft = inLeft;
536 : *outputLeft = outLeft / 2;
537 :
538 : if (res != (size_t) -1)
539 : return NS_OK;
540 :
541 : // reset converters
542 : xp_iconv_reset(gNativeToUTF8);
543 : xp_iconv_reset(gUTF8ToUnicode);
544 : }
545 : #endif
546 :
547 : // fallback: zero-pad and hope for the best
548 : // XXX This is lame and we have to do better.
549 0 : isolatin1_to_utf16(input, inputLeft, output, outputLeft);
550 :
551 0 : return NS_OK;
552 : }
553 :
554 : nsresult
555 227484 : nsNativeCharsetConverter::UnicodeToNative(const PRUnichar **input,
556 : PRUint32 *inputLeft,
557 : char **output,
558 : PRUint32 *outputLeft)
559 : {
560 227484 : size_t res = 0;
561 227484 : size_t inLeft = (size_t) *inputLeft * 2;
562 227484 : size_t outLeft = (size_t) *outputLeft;
563 :
564 227484 : if (gUnicodeToNative != INVALID_ICONV_T) {
565 227484 : res = xp_iconv(gUnicodeToNative, (const char **) input, &inLeft, output, &outLeft);
566 :
567 227484 : if (res != (size_t) -1) {
568 227484 : *inputLeft = inLeft / 2;
569 227484 : *outputLeft = outLeft;
570 227484 : return NS_OK;
571 : }
572 :
573 0 : NS_ERROR("iconv failed");
574 :
575 : // reset converter
576 0 : xp_iconv_reset(gUnicodeToNative);
577 : }
578 : #if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
579 : else if ((gUnicodeToUTF8 != INVALID_ICONV_T) &&
580 : (gUTF8ToNative != INVALID_ICONV_T)) {
581 : const char *in = (const char *) *input;
582 :
583 : char ubuf[6]; // max utf-8 char length (really only needs to be 4 bytes)
584 :
585 : // convert one uchar at a time...
586 : while (inLeft && outLeft) {
587 : char *p = ubuf;
588 : size_t n = sizeof(ubuf), one_uchar = sizeof(PRUnichar);
589 : res = xp_iconv(gUnicodeToUTF8, &in, &one_uchar, &p, &n);
590 : if (res == (size_t) -1) {
591 : NS_ERROR("conversion from utf-16 to utf-8 failed");
592 : break;
593 : }
594 : p = ubuf;
595 : n = sizeof(ubuf) - n;
596 : res = xp_iconv(gUTF8ToNative, (const char **) &p, &n, output, &outLeft);
597 : if (res == (size_t) -1) {
598 : if (errno == E2BIG) {
599 : // not enough room for last uchar... back up and return.
600 : in -= sizeof(PRUnichar);
601 : res = 0;
602 : }
603 : else
604 : NS_ERROR("conversion from utf-8 to native failed");
605 : break;
606 : }
607 : inLeft -= sizeof(PRUnichar);
608 : }
609 :
610 : if (res != (size_t) -1) {
611 : (*input) += (*inputLeft - inLeft/2);
612 : *inputLeft = inLeft/2;
613 : *outputLeft = outLeft;
614 : return NS_OK;
615 : }
616 :
617 : // reset converters
618 : xp_iconv_reset(gUnicodeToUTF8);
619 : xp_iconv_reset(gUTF8ToNative);
620 : }
621 : #endif
622 :
623 : // fallback: truncate and hope for the best
624 0 : utf16_to_isolatin1(input, inputLeft, output, outputLeft);
625 :
626 0 : return NS_OK;
627 : }
628 :
629 : bool
630 62059 : nsNativeCharsetConverter::IsNativeUTF8()
631 : {
632 62059 : if (!gInitialized) {
633 0 : Lock();
634 0 : if (!gInitialized)
635 0 : LazyInit();
636 0 : Unlock();
637 : }
638 62059 : return gIsNativeUTF8;
639 : }
640 :
641 : #endif // USE_ICONV
642 :
643 : //-----------------------------------------------------------------------------
644 : // conversion using mb[r]towc/wc[r]tomb
645 : //-----------------------------------------------------------------------------
646 : #if defined(USE_STDCONV)
647 : #if defined(HAVE_WCRTOMB) || defined(HAVE_MBRTOWC)
648 : #include <wchar.h> // mbrtowc, wcrtomb
649 : #endif
650 :
651 : class nsNativeCharsetConverter
652 : {
653 : public:
654 : nsNativeCharsetConverter();
655 :
656 : nsresult NativeToUnicode(const char **input , PRUint32 *inputLeft,
657 : PRUnichar **output, PRUint32 *outputLeft);
658 : nsresult UnicodeToNative(const PRUnichar **input , PRUint32 *inputLeft,
659 : char **output, PRUint32 *outputLeft);
660 :
661 : static void GlobalInit();
662 : static void GlobalShutdown() { }
663 : static bool IsNativeUTF8();
664 :
665 : private:
666 : static bool gWCharIsUnicode;
667 :
668 : #if defined(HAVE_WCRTOMB) || defined(HAVE_MBRTOWC)
669 : mbstate_t ps;
670 : #endif
671 : };
672 :
673 : bool nsNativeCharsetConverter::gWCharIsUnicode = false;
674 :
675 : nsNativeCharsetConverter::nsNativeCharsetConverter()
676 : {
677 : #if defined(HAVE_WCRTOMB) || defined(HAVE_MBRTOWC)
678 : memset(&ps, 0, sizeof(ps));
679 : #endif
680 : }
681 :
682 : void
683 : nsNativeCharsetConverter::GlobalInit()
684 : {
685 : // verify that wchar_t for the current locale is actually unicode.
686 : // if it is not, then we should avoid calling mbtowc/wctomb and
687 : // just fallback on zero-pad/truncation conversion.
688 : //
689 : // this test cannot be done at build time because the encoding of
690 : // wchar_t may depend on the runtime locale. sad, but true!!
691 : //
692 : // so, if wchar_t is unicode then converting an ASCII character
693 : // to wchar_t should not change its numeric value. we'll just
694 : // check what happens with the ASCII 'a' character.
695 : //
696 : // this test is not perfect... obviously, it could yield false
697 : // positives, but then at least ASCII text would be converted
698 : // properly (or maybe just the 'a' character) -- oh well :(
699 :
700 : char a = 'a';
701 : unsigned int w = 0;
702 :
703 : int res = mbtowc((wchar_t *) &w, &a, 1);
704 :
705 : gWCharIsUnicode = (res != -1 && w == 'a');
706 :
707 : #ifdef DEBUG
708 : if (!gWCharIsUnicode)
709 : NS_WARNING("wchar_t is not unicode (unicode conversion will be lossy)");
710 : #endif
711 : }
712 :
713 : nsresult
714 : nsNativeCharsetConverter::NativeToUnicode(const char **input,
715 : PRUint32 *inputLeft,
716 : PRUnichar **output,
717 : PRUint32 *outputLeft)
718 : {
719 : if (gWCharIsUnicode) {
720 : int incr;
721 :
722 : // cannot use wchar_t here since it may have been redefined (e.g.,
723 : // via -fshort-wchar). hopefully, sizeof(tmp) is sufficient XP.
724 : unsigned int tmp = 0;
725 : while (*inputLeft && *outputLeft) {
726 : #ifdef HAVE_MBRTOWC
727 : incr = (int) mbrtowc((wchar_t *) &tmp, *input, *inputLeft, &ps);
728 : #else
729 : // XXX is this thread-safe?
730 : incr = (int) mbtowc((wchar_t *) &tmp, *input, *inputLeft);
731 : #endif
732 : if (incr < 0) {
733 : NS_WARNING("mbtowc failed: possible charset mismatch");
734 : // zero-pad and hope for the best
735 : tmp = (unsigned char) **input;
736 : incr = 1;
737 : }
738 : **output = (PRUnichar) tmp;
739 : (*input) += incr;
740 : (*inputLeft) -= incr;
741 : (*output)++;
742 : (*outputLeft)--;
743 : }
744 : }
745 : else {
746 : // wchar_t isn't unicode, so the best we can do is treat the
747 : // input as if it is isolatin1 :(
748 : isolatin1_to_utf16(input, inputLeft, output, outputLeft);
749 : }
750 :
751 : return NS_OK;
752 : }
753 :
754 : nsresult
755 : nsNativeCharsetConverter::UnicodeToNative(const PRUnichar **input,
756 : PRUint32 *inputLeft,
757 : char **output,
758 : PRUint32 *outputLeft)
759 : {
760 : if (gWCharIsUnicode) {
761 : int incr;
762 :
763 : while (*inputLeft && *outputLeft >= MB_CUR_MAX) {
764 : #ifdef HAVE_WCRTOMB
765 : incr = (int) wcrtomb(*output, (wchar_t) **input, &ps);
766 : #else
767 : // XXX is this thread-safe?
768 : incr = (int) wctomb(*output, (wchar_t) **input);
769 : #endif
770 : if (incr < 0) {
771 : NS_WARNING("mbtowc failed: possible charset mismatch");
772 : **output = (unsigned char) **input; // truncate
773 : incr = 1;
774 : }
775 : // most likely we're dead anyways if this assertion should fire
776 : NS_ASSERTION(PRUint32(incr) <= *outputLeft, "wrote beyond end of string");
777 : (*output) += incr;
778 : (*outputLeft) -= incr;
779 : (*input)++;
780 : (*inputLeft)--;
781 : }
782 : }
783 : else {
784 : // wchar_t isn't unicode, so the best we can do is treat the
785 : // input as if it is isolatin1 :(
786 : utf16_to_isolatin1(input, inputLeft, output, outputLeft);
787 : }
788 :
789 : return NS_OK;
790 : }
791 :
792 : // XXX : for now, return false
793 : bool
794 : nsNativeCharsetConverter::IsNativeUTF8()
795 : {
796 : return false;
797 : }
798 :
799 : #endif // USE_STDCONV
800 :
801 : //-----------------------------------------------------------------------------
802 : // API implementation
803 : //-----------------------------------------------------------------------------
804 :
805 : nsresult
806 215382 : NS_CopyNativeToUnicode(const nsACString &input, nsAString &output)
807 : {
808 215382 : output.Truncate();
809 :
810 215382 : PRUint32 inputLen = input.Length();
811 :
812 215382 : nsACString::const_iterator iter;
813 215382 : input.BeginReading(iter);
814 :
815 : //
816 : // OPTIMIZATION: preallocate space for largest possible result; convert
817 : // directly into the result buffer to avoid intermediate buffer copy.
818 : //
819 : // this will generally result in a larger allocation, but that seems
820 : // better than an extra buffer copy.
821 : //
822 215382 : if (!EnsureStringLength(output, inputLen))
823 0 : return NS_ERROR_OUT_OF_MEMORY;
824 215382 : nsAString::iterator out_iter;
825 215382 : output.BeginWriting(out_iter);
826 :
827 215382 : PRUnichar *result = out_iter.get();
828 215382 : PRUint32 resultLeft = inputLen;
829 :
830 215382 : const char *buf = iter.get();
831 215382 : PRUint32 bufLeft = inputLen;
832 :
833 430764 : nsNativeCharsetConverter conv;
834 215382 : nsresult rv = conv.NativeToUnicode(&buf, &bufLeft, &result, &resultLeft);
835 215382 : if (NS_SUCCEEDED(rv)) {
836 215382 : NS_ASSERTION(bufLeft == 0, "did not consume entire input buffer");
837 215382 : output.SetLength(inputLen - resultLeft);
838 : }
839 215382 : return rv;
840 : }
841 :
842 : nsresult
843 228665 : NS_CopyUnicodeToNative(const nsAString &input, nsACString &output)
844 : {
845 228665 : output.Truncate();
846 :
847 228665 : nsAString::const_iterator iter, end;
848 228665 : input.BeginReading(iter);
849 228665 : input.EndReading(end);
850 :
851 : // cannot easily avoid intermediate buffer copy.
852 : char temp[4096];
853 :
854 457330 : nsNativeCharsetConverter conv;
855 :
856 228665 : const PRUnichar *buf = iter.get();
857 228665 : PRUint32 bufLeft = Distance(iter, end);
858 684814 : while (bufLeft) {
859 227484 : char *p = temp;
860 227484 : PRUint32 tempLeft = sizeof(temp);
861 :
862 227484 : nsresult rv = conv.UnicodeToNative(&buf, &bufLeft, &p, &tempLeft);
863 227484 : if (NS_FAILED(rv)) return rv;
864 :
865 227484 : if (tempLeft < sizeof(temp))
866 227484 : output.Append(temp, sizeof(temp) - tempLeft);
867 : }
868 228665 : return NS_OK;
869 : }
870 :
871 : bool
872 62059 : NS_IsNativeUTF8()
873 : {
874 62059 : return nsNativeCharsetConverter::IsNativeUTF8();
875 : }
876 :
877 : void
878 1365 : NS_StartupNativeCharsetUtils()
879 : {
880 : //
881 : // need to initialize the locale or else charset conversion will fail.
882 : // better not delay this in case some other component alters the locale
883 : // settings.
884 : //
885 : // XXX we assume that we are called early enough that we should
886 : // always be the first to care about the locale's charset.
887 : //
888 1365 : setlocale(LC_CTYPE, "");
889 :
890 1365 : nsNativeCharsetConverter::GlobalInit();
891 1365 : }
892 :
893 : void
894 1365 : NS_ShutdownNativeCharsetUtils()
895 : {
896 1365 : nsNativeCharsetConverter::GlobalShutdown();
897 1365 : }
898 :
899 : //-----------------------------------------------------------------------------
900 : // XP_WIN
901 : //-----------------------------------------------------------------------------
902 : #elif defined(XP_WIN)
903 :
904 : #include <windows.h>
905 : #include "nsAString.h"
906 : #include "nsReadableUtils.h"
907 :
908 : nsresult
909 : NS_CopyNativeToUnicode(const nsACString &input, nsAString &output)
910 : {
911 : PRUint32 inputLen = input.Length();
912 :
913 : nsACString::const_iterator iter;
914 : input.BeginReading(iter);
915 :
916 : const char *buf = iter.get();
917 :
918 : // determine length of result
919 : PRUint32 resultLen = 0;
920 : int n = ::MultiByteToWideChar(CP_ACP, 0, buf, inputLen, NULL, 0);
921 : if (n > 0)
922 : resultLen += n;
923 :
924 : // allocate sufficient space
925 : if (!EnsureStringLength(output, resultLen))
926 : return NS_ERROR_OUT_OF_MEMORY;
927 : if (resultLen > 0) {
928 : nsAString::iterator out_iter;
929 : output.BeginWriting(out_iter);
930 :
931 : PRUnichar *result = out_iter.get();
932 :
933 : ::MultiByteToWideChar(CP_ACP, 0, buf, inputLen, result, resultLen);
934 : }
935 : return NS_OK;
936 : }
937 :
938 : nsresult
939 : NS_CopyUnicodeToNative(const nsAString &input, nsACString &output)
940 : {
941 : PRUint32 inputLen = input.Length();
942 :
943 : nsAString::const_iterator iter;
944 : input.BeginReading(iter);
945 :
946 : const PRUnichar *buf = iter.get();
947 :
948 : // determine length of result
949 : PRUint32 resultLen = 0;
950 :
951 : int n = ::WideCharToMultiByte(CP_ACP, 0, buf, inputLen, NULL, 0, NULL, NULL);
952 : if (n > 0)
953 : resultLen += n;
954 :
955 : // allocate sufficient space
956 : if (!EnsureStringLength(output, resultLen))
957 : return NS_ERROR_OUT_OF_MEMORY;
958 : if (resultLen > 0) {
959 : nsACString::iterator out_iter;
960 : output.BeginWriting(out_iter);
961 :
962 : // default "defaultChar" is '?', which is an illegal character on windows
963 : // file system. That will cause file uncreatable. Change it to '_'
964 : const char defaultChar = '_';
965 :
966 : char *result = out_iter.get();
967 :
968 : ::WideCharToMultiByte(CP_ACP, 0, buf, inputLen, result, resultLen,
969 : &defaultChar, NULL);
970 : }
971 : return NS_OK;
972 : }
973 :
974 : // moved from widget/windows/nsToolkit.cpp
975 : PRInt32
976 : NS_ConvertAtoW(const char *aStrInA, int aBufferSize, PRUnichar *aStrOutW)
977 : {
978 : return MultiByteToWideChar(CP_ACP, 0, aStrInA, -1, aStrOutW, aBufferSize);
979 : }
980 :
981 : PRInt32
982 : NS_ConvertWtoA(const PRUnichar *aStrInW, int aBufferSizeOut,
983 : char *aStrOutA, const char *aDefault)
984 : {
985 : if ((!aStrInW) || (!aStrOutA) || (aBufferSizeOut <= 0))
986 : return 0;
987 :
988 : int numCharsConverted = WideCharToMultiByte(CP_ACP, 0, aStrInW, -1,
989 : aStrOutA, aBufferSizeOut,
990 : aDefault, NULL);
991 :
992 : if (!numCharsConverted) {
993 : if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
994 : // Overflow, add missing null termination but return 0
995 : aStrOutA[aBufferSizeOut-1] = '\0';
996 : }
997 : else {
998 : // Other error, clear string and return 0
999 : aStrOutA[0] = '\0';
1000 : }
1001 : }
1002 : else if (numCharsConverted < aBufferSizeOut) {
1003 : // Add 2nd null (really necessary?)
1004 : aStrOutA[numCharsConverted] = '\0';
1005 : }
1006 :
1007 : return numCharsConverted;
1008 : }
1009 :
1010 : //-----------------------------------------------------------------------------
1011 : // XP_OS2
1012 : //-----------------------------------------------------------------------------
1013 : #elif defined(XP_OS2)
1014 :
1015 : #define INCL_DOS
1016 : #include <os2.h>
1017 : #include <uconv.h>
1018 : #include "nsAString.h"
1019 : #include "nsReadableUtils.h"
1020 : #include <ulserrno.h>
1021 : #include "nsNativeCharsetUtils.h"
1022 :
1023 : static UconvObject UnicodeConverter = NULL;
1024 :
1025 : nsresult
1026 : NS_CopyNativeToUnicode(const nsACString &input, nsAString &output)
1027 : {
1028 : PRUint32 inputLen = input.Length();
1029 :
1030 : nsACString::const_iterator iter;
1031 : input.BeginReading(iter);
1032 : const char *inputStr = iter.get();
1033 :
1034 : // determine length of result
1035 : PRUint32 resultLen = inputLen;
1036 : if (!EnsureStringLength(output, resultLen))
1037 : return NS_ERROR_OUT_OF_MEMORY;
1038 :
1039 : nsAString::iterator out_iter;
1040 : output.BeginWriting(out_iter);
1041 : UniChar *result = (UniChar*)out_iter.get();
1042 :
1043 : size_t cSubs = 0;
1044 : size_t resultLeft = resultLen;
1045 :
1046 : if (!UnicodeConverter)
1047 : NS_StartupNativeCharsetUtils();
1048 :
1049 : int unirc = ::UniUconvToUcs(UnicodeConverter, (void**)&inputStr, &inputLen,
1050 : &result, &resultLeft, &cSubs);
1051 :
1052 : NS_ASSERTION(unirc != UCONV_E2BIG, "Path too big");
1053 :
1054 : if (unirc != ULS_SUCCESS) {
1055 : output.Truncate();
1056 : return NS_ERROR_FAILURE;
1057 : }
1058 :
1059 : // Need to update string length to reflect how many bytes were actually
1060 : // written.
1061 : output.Truncate(resultLen - resultLeft);
1062 : return NS_OK;
1063 : }
1064 :
1065 : nsresult
1066 : NS_CopyUnicodeToNative(const nsAString &input, nsACString &output)
1067 : {
1068 : size_t inputLen = input.Length();
1069 :
1070 : nsAString::const_iterator iter;
1071 : input.BeginReading(iter);
1072 : UniChar* inputStr = (UniChar*) const_cast<PRUnichar*>(iter.get());
1073 :
1074 : // maximum length of unicode string of length x converted to native
1075 : // codepage is x*2
1076 : size_t resultLen = inputLen * 2;
1077 : if (!EnsureStringLength(output, resultLen))
1078 : return NS_ERROR_OUT_OF_MEMORY;
1079 :
1080 : nsACString::iterator out_iter;
1081 : output.BeginWriting(out_iter);
1082 : char *result = out_iter.get();
1083 :
1084 : size_t cSubs = 0;
1085 : size_t resultLeft = resultLen;
1086 :
1087 : if (!UnicodeConverter)
1088 : NS_StartupNativeCharsetUtils();
1089 :
1090 : int unirc = ::UniUconvFromUcs(UnicodeConverter, &inputStr, &inputLen,
1091 : (void**)&result, &resultLeft, &cSubs);
1092 :
1093 : NS_ASSERTION(unirc != UCONV_E2BIG, "Path too big");
1094 :
1095 : if (unirc != ULS_SUCCESS) {
1096 : output.Truncate();
1097 : return NS_ERROR_FAILURE;
1098 : }
1099 :
1100 : // Need to update string length to reflect how many bytes were actually
1101 : // written.
1102 : output.Truncate(resultLen - resultLeft);
1103 : return NS_OK;
1104 : }
1105 :
1106 : void
1107 : NS_StartupNativeCharsetUtils()
1108 : {
1109 : ULONG ulLength;
1110 : ULONG ulCodePage;
1111 : DosQueryCp(sizeof(ULONG), &ulCodePage, &ulLength);
1112 :
1113 : UniChar codepage[20];
1114 : int unirc = ::UniMapCpToUcsCp(ulCodePage, codepage, 20);
1115 : if (unirc == ULS_SUCCESS) {
1116 : unirc = ::UniCreateUconvObject(codepage, &UnicodeConverter);
1117 : if (unirc == ULS_SUCCESS) {
1118 : uconv_attribute_t attr;
1119 : ::UniQueryUconvObject(UnicodeConverter, &attr, sizeof(uconv_attribute_t),
1120 : NULL, NULL, NULL);
1121 : attr.options = UCONV_OPTION_SUBSTITUTE_BOTH;
1122 : attr.subchar_len=1;
1123 : attr.subchar[0]='_';
1124 : ::UniSetUconvObject(UnicodeConverter, &attr);
1125 : }
1126 : }
1127 : }
1128 :
1129 : void
1130 : NS_ShutdownNativeCharsetUtils()
1131 : {
1132 : ::UniFreeUconvObject(UnicodeConverter);
1133 : }
1134 :
1135 : #else
1136 :
1137 : #include "nsReadableUtils.h"
1138 :
1139 : nsresult
1140 : NS_CopyNativeToUnicode(const nsACString &input, nsAString &output)
1141 : {
1142 : CopyASCIItoUTF16(input, output);
1143 : return NS_OK;
1144 : }
1145 :
1146 : nsresult
1147 : NS_CopyUnicodeToNative(const nsAString &input, nsACString &output)
1148 : {
1149 : LossyCopyUTF16toASCII(input, output);
1150 : return NS_OK;
1151 : }
1152 :
1153 : void
1154 : NS_StartupNativeCharsetUtils()
1155 : {
1156 : }
1157 :
1158 : void
1159 : NS_ShutdownNativeCharsetUtils()
1160 : {
1161 : }
1162 :
1163 : #endif
|