// utf-8

/*	Copyright  (c)	Günter Woigk  2002-2003
  					mailto:kio@little-bat.de

	This file is free software

 	This program is distributed in the hope that it will be useful,
 	but WITHOUT ANY WARRANTY; without even the implied warranty of
 	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

 	Permission to use, copy, modify, distribute, and sell this software and
 	its documentation for any purpose is hereby granted without fee, provided
 	that the above copyright notice appear in all copies and that both that
 	copyright notice and this permission notice appear in supporting
 	documentation, and that the name of the copyright holder not be used
 	in advertising or publicity pertaining to distribution of the software
 	without specific, written prior permission. The copyright holder makes no
 	representations about the suitability of this software for any purpose.
 	It is provided "as is" without express or implied warranty.

 	THE COPYRIGHT HOLDER DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
 	INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
 	EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY SPECIAL, INDIRECT OR
 	CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
 	DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
 	TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 	PERFORMANCE OF THIS SOFTWARE.
	

	this module provides support for UTF-8, UCS-2 and UCS-4 unicode character encoding


	2002-03-17 kio	started work
	2002-07-30 kio	utf-html conversion
	2002-08-11 kio	removing dependencies of String to Str
	2002-06-09 kio	rework
	2003-05-25 kio	upercase, lowercase et.al.
	2003-08-10 kio	rework: removed all string functions (moved to class String), kept all char functions
*/

#define SAFE	3
#define	LOG		2

#include	"config.h"
#include	<ctype.h>
#include	"utf-8/utf-8.h"


#ifndef _INCLUDE_UNICODE_CHARACTER_NAMES
#define	_INCLUDE_UNICODE_CHARACTER_NAMES	0
#endif



/* ********************************************************************************
					CHARACTER CONVERSION
******************************************************************************** */


/* ----	convert UTF-8 char to UCS-4 -------------------------------
		stops at next non-fup or at nominal char size
		sets errno and returns replacement char on error
*/
UCS4Char UCS4CharFromUTF8 ( UTF8Char s )
{	UCS4Char c0,n; uint i;

	XXTRAP ( !s );

	c0 = uchar ( *s++ );
	if (utf8_is_7bit(c0)) return c0;		// 7-bit ascii char
	if (utf8_is_fup(c0)) goto x1;			// 0x80 - 0xBF: unexpected fups
	if (c0>=0xfe) goto x2;					// ill. codes

// longish character:
// 0xC0.. => 2 bytes:	110xxxxx 10xxxxxx
// 0xE0.. => 3 bytes:	1110xxxx 10xxxxxx 10xxxxxx
// 0xF0.. => 4 bytes:	11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
// 0xF8.. => 5 bytes:	111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
// 0xFC.. => 6 bytes:	1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
	n = 0;		// UCS-4 char code akku
	i = 0;		// UTF-8 character size

	while( c0 & (0x80u>>(++i)) )
	{
		uchar c1 = *s; if( utf8_no_fup(c1) ) goto x3;
		s++; n = (n<<6) + (c1&0x3F);
	}

// now: i = total number of digits
//      n = UCS4 char code without c0 bits
	n += (c0&(0x7f>>i)) << (6*i-6);			// n := UCS4 char code

// ill. overlong encodings:
// i=2:	1100000x 10xxxxxx										< (1<<6)
// i=3:	11100000 100xxxxx 10xxxxxx 								< (1<<11)
// i=4:	11110000 1000xxxx 10xxxxxx 10xxxxxx						< (1<<16)
// i=5:	11111000 10000xxx 10xxxxxx 10xxxxxx 10xxxxxx			< (1<<21)
// i=6:	11111100 100000xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx	< (1<<26)
	if ( n < (1u<<(i*5-4)) ) goto x4;		// ill. overlong encoding

// ok => store code
	return n;

// error => store replacement char
x1:	SetError(unexpectedfup);   return UCS4ReplacementChar;
x2:	SetError(illegalchar);     return UCS4ReplacementChar;
x3:	SetError(truncatedchar);   return UCS4ReplacementChar;
x4:	SetError(illegaloverlong); return UCS4ReplacementChar;
}


/* ----	convert UCS4 code -> UTF-8 character --------------
		returns 1 .. 6 byte 0-terminated string with UTF-8 encoding
		except for n==0, where char(0) itself is the result.
		returned string may be allocated in qstring pool
		returns UTF8ReplacementChar for n>0x7FFFFFFF
*/
UTF8Char UTF8CharFromUCS4 ( UCS4Char n )
{
// 7-bit ascii:
	if (n<0x80) return CharStr(n);	// n==0 => ""; else 1-char string

// 2-byte codes:
	if ( n<(1u<<11) ) { str s=QString(2); s[0]=0xC0+(n>>6); s[1]=0x80+(n&0x3f); return s; }

// ill. codes test:
	if ( (long)n<0 ) { SetError(illegalchar); return UTF8ReplacementChar; }		// bit 31 set

// multi-byte codes:
	uint i = 2; while (n>>(1+i*5)) i++;		// utf-8 character size
	str s = QString(i);
	char c0 = 0x80;
	while (--i>0) { c0 >>= 1; s[i] = 0x80+(n&0x3f); n >>= 6; }
	s[0] = c0 + n;
	return s;
}


void UCS4CharToUTF8 ( UCS4Char n, UTF8CharPtr& d )
{
// 7-bit ascii:
	if (n<0x80) { *d++ = n; return; }

// 2-byte codes:
	if ( n<(1u<<11) ) { *d++ = 0xC0+(n>>6); *d++ = 0x80+(n&0x3f); return; }

// ill. codes test:
	if ( (long)n<0 )			// bit 31 set
	{
		SetError(illegalchar);
		cUTF8CharPtr r=UTF8ReplacementChar;
		while(*r) *d++ = *r++;
	}

// multi-byte codes:
	uint i = 2; while (n>>(1+i*5)) i++;		// utf-8 character size
	UTF8CharPtr s = d; d += i;
	char c0 = 0x80;
	while (--i>0) { c0 >>= 1; s[i] = 0x80+(n&0x3f); n >>= 6; }
	s[0] = c0 + n;
}




/* ****************************************************************
					CONVERSION 8-bit CHAR <-> UCS4
**************************************************************** */

#define B(N)	N[0]>=0 ? N[0] \
			  : (N[0]&0xe0)==0xc0 ? (N[0]&31)*(1<<6) + (N[1]&63) \
			  : (N[0]&15)*(1<<12) + (N[1]&63)*(1<<6) + (N[2]&63)

const UCS2Table ucs2_from_mac_roman =
#include "mac_roman.h"

const UCS2Table ucs2_from_ascii_ger =
#include "ascii_ger.h"

const UCS2Table ucs2_from_rtos =
#include "rtos.h"

const UCS2Table ucs2_from_cp_437 =
#include "cp_437.h"

const UCS2Table ucs2_from_atari_st =
#include "atari_st.h"


/* ----	convert UCS4 char -> 8-bit char ------------------------------------------
*/
char UCS4CharTo8Bit ( UCS4Char n, const UCS2Table t )
{
	if (n<NELEM(t) && n==t[n]) return n;

	if ((n>>16)==0)
	{
		for ( uint i=0; i<NELEM(t); i++ ) { if (t[i]==UCS2Char(n)) return i; }
	}

	SetError(notindestcharset);
	return UCS4ReplacementChar;
}




// *********************************************************************
//				UPPERCASE / LOWERCASE
// *********************************************************************


// ----	uppercase/lowercase maps (created by "uc.txt,lc.txt.vs") -------
struct Pair { UCS2Char n; short d; };	// ucs-2 codes
#include "utf-8/uc.h"
#include "utf-8/lc.h"


UCS4Char ucs4_char_to_upper ( UCS4Char c )
{
	XXXTRAP( uc[NELEM(uc)-1].d != 0 );

	if (c>>16) return c;	// ***TODO***	characters beyond Ucs2() are not yet converted
							//				DESERET Letters 0x10400 .. 0x1044F have uc/lc

	int a = 0, e = NELEM(uc), i;
	while (a<e-1) { i=(a+e)/2; if (UCS2Char(c)<uc[i].n) e=i; else a=i; }
	return c + uc[a].d;
}


UCS4Char ucs4_char_to_lower ( UCS4Char c )
{
	XXXTRAP( lc[NELEM(lc)-1].d != 0 );

	if (c>>16) return c;	// ***TODO***	characters beyond Ucs2() are not yet converted
							//				DESERET Letters 0x10400 .. 0x1044F have uc/lc

	int a = 0, e = NELEM(lc), i;
	while (a<e-1) { i=(a+e)/2; if (UCS2Char(c)<lc[i].n) e=i; else a=i; }
	return c + lc[a].d;
}


void UTF8CharToUpper ( UTF8Char q, UTF8CharPtr& d )
{
	XXTRAP(!q);
	XXTRAP(!d);

	uchar c = *q; if(utf8_is_7bit(c)) *d++ = _to_upper(c);
	else UCS4CharToUTF8(UCS4CharToUpper(UCS4CharFromUTF8(q)),d);
}


void UTF8CharToLower ( UTF8Char q, UTF8CharPtr& d )
{
	XXTRAP(!q);
	XXTRAP(!d);

	uchar c = *q; if(utf8_is_7bit(c)) *d++ = _to_lower(c);
	else UCS4CharToUTF8(UCS4CharToLower(UCS4CharFromUTF8(q)),d);
}


UTF8Char UTF8CharToLower ( UTF8Char s )
{
	XXTRAP(!s);

	char c = *s; if(utf8_is_7bit(c)) return CharStr(_to_lower(c));

	str d = QString(6);
	ptr e = d;

	UCS4CharToUTF8(UCS4CharToLower(UCS4CharFromUTF8(s)),e);
	*e = 0;

	return d;
}


UTF8Char UTF8CharToUpper ( UTF8Char s )
{
	XXTRAP(!s);

	char c = *s; if(utf8_is_7bit(c)) return CharStr(_to_upper(c));

	str d = QString(6);
	ptr e = d;

	UCS4CharToUTF8(UCS4CharToUpper(UCS4CharFromUTF8(s)),e);
	*e = 0;

	return d;
}




/* ****************************************************************
					UNICODE INFO METHODS
**************************************************************** */


static cstr utf8_general_class_name[] =
{
		"illegal character",		// ????			(no character should have this property)
		"letter uppercase",			// Lu
		"letter lowercase",			// Ll
		"letter titlecase",			// Lt
		"letter modifier",			// Lm
		"letter other",				// Lo
		"mark nonspacing",			// Mn
		"mark spacing combining",	// Mc
		"mark enclosing",			// Me
		"number decimal digit",		// Nd
		"number letter",			// Nl
		"number other",				// No
		"punctuation connector",	// Pc
		"punctuation dash",			// Pd
		"punctuation open",			// Ps
		"punctuation close",		// Pe
		"punctuation initial quote",// Pi			(may behave like Ps or Pe depending on usage)
		"punctuation final quote",	// Pf			(may behave like Ps or Pe depending on usage)
		"punctuation other",		// Po
		"symbol math",				// Sm
		"symbol currency",			// Sc
		"symbol modifier",			// Sk
		"symbol other",				// So
		"separator space",			// Zs
		"separator line",			// Zl
		"separator paragraph",		// Zp
		"other control",			// Cc
		"other format",				// Cf
		"other surrogate",			// Cs
		"other private use",		// Co
		"other not assigned"		// Cn			(no characters in the file have this property)
};


/* ====	character info table ===============================================================================
		read from "UTF8UnicodeCharInfo.h" which is generated
		by "UTF8UnicodeCharInfo.vs" from "UTF8UnicodeCharInfo.tab"
		array items are sorted ascending by character code
*/
struct UTF8UnicodeCharInfo
{
	UCS4Char code;						// UCS-4 character code
#if _INCLUDE_UNICODE_CHARACTER_NAMES
	cstr	 name;						// tokenized name
#endif
	CharClass general_class;			// 1...
//	uchar	 canonical_combining_class;	// 1...
//	uchar	 bidirectional_class;		// 1...
//	uchar	 decomposition_mapping;
//	uchar	 decimal_digit_value;		// or 0xff
//	uchar	 digit_value;				// or 0xff
	float	 number_value;				// or NAN
//	bool	 mirrored;
//	cstr	 uppercase_mapping;			// utf-8; or NULL if no uc/lc pair or if this is the uc variant
//	cstr	 lowercase_mapping;			// utf-8; similar ""
//	cstr	 titlecase_mapping;			// utf-8; similar ""
};


//#ifndef NAN
	const float NaN = 0.0/0.0;
//#else
//	const float NaN = NAN;
//#endif


#if _INCLUDE_UNICODE_CHARACTER_NAMES
	#define UD(CODE,NAME,CAT,CAN,BIDI,DECOM,ISDEC,DEC,ISDIG,DIG,ISNUM,NUM,MIRR,UC,LC,TC)	\
	{ CODE,NAME,CharClass(CAT),ISNUM?(float)NUM:NaN }
#else
	#define UD(CODE,NAME,CAT,CAN,BIDI,DECOM,ISDEC,DEC,ISDIG,DIG,ISNUM,NUM,MIRR,UC,LC,TC)	\
	{ CODE,CharClass(CAT),ISNUM?(float)NUM:NaN }
#endif

//	static char const*const namtok[tok_cnt] = { ... };
//	UTF8UnicodeCharInfo const utf8_unicode_char_info[~13,000] = { ... };
#include "utf-8/unicode_char_info.h"

//	dummy info for invalid character codes
UTF8UnicodeCharInfo const utf8_invalid_char_info =
	UD(	UCS4ReplacementChar,	// code
		"INVALID CHARACTER",	// name
		other_cat_unknown,		// class
		0,19,1,					// canonic., bidir, decomp.
		0,0,					// isdec, value
		0,0,					// isdig, value
		0,0,					// isnum, value
		0,						// mirrored
		UTF8ReplacementChar,	// upper case mapping	((to ease uc/lc conversion of broken text))
		UTF8ReplacementChar,	// lower case mapping
		UTF8ReplacementChar);	// title case mapping



/* ==== ACCESS to UTF8_CHAR_INFO ======================================================
*/

/* ----	find unicode char info for UCS2/UCS4 character ---------------- 2002-08-11 kio ----
		returns &utf8_invalid_char_info for invalid characters
*/
static UTF8UnicodeCharInfo const* utf8_char_info ( UCS4Char n )
{
	XXXLogIn("utf8_char_info");
	XXXTRAP(utf8_unicode_char_info[511].code!=511);

	// shortcut for codes < 0x200
	if (n<512) { XXXLogOut("=%i",(int)n); return &utf8_unicode_char_info[n]; }

	// search range is [a ... [e
	uint a = 512;
	uint e = NELEM(utf8_unicode_char_info);

	while (a<e)
	{
		uint i = (a+e)/2;
		if( n < utf8_unicode_char_info[i].code ) { e=i;   continue; }
		if( n > utf8_unicode_char_info[i].code ) { a=i+1; continue; }
		XXXLogOut("=%i",(int)i);
		return &utf8_unicode_char_info[i];
	}
	XXXLogOut("=ill.");
	return &utf8_invalid_char_info;
}


/* ----	find unicode char info for UTF8 character ---------------- 2002-08-11 kio ----
		returns &utf8_invalid_char_info for invalid characters
*/
static UTF8UnicodeCharInfo const* utf8_char_info ( UTF8Char s )
{
	XXXTRAP(!s);
	char n = s[0];
	if  (utf8_is_7bit(n)) return &utf8_unicode_char_info[n];
	return utf8_char_info( UCS4CharFromUTF8(s) );
}



#if _INCLUDE_UNICODE_CHARACTER_NAMES

/* ====	GET CHARACTER NAME ============================================================
*/

/* ----	decompress character name -------------------------[2003-05-23 kio debuged]----
		decompress name from utf8_unicode_char_info[].name
		to supplied buffer.
		returns: length of name
*/
static uint decompress_name ( cstr s, str t )
{
	XXXTRAP(!s);
	XXXTRAP(!t);

	ptr  z = t;
	bool f = 0;		// space sep. req. ?

	while(*s)
	{
		if( (uchar)*s<tok_bas )	// normal char ?
		{
			if (f) *z++ = ' ';
			do { *z++ = *s++; } while( *s && (uchar)*s<tok_bas );
			f = 0;
		}
		else	// token ?
		{
			if (z>t) *z++ = ' ';	// always space before token except at start
			cptr q = namtok[(uchar)*s++-tok_bas];
			do { *z++ = *q++; } while( *q );
			f = 1;
		}
	}

	return z-t;	// len
}


/* ----	decompress character name ------------------------- 2002-08-11 kio ----
		decompress name from utf8_unicode_char_info[].name
		returns str in qstring pool
*/
static str decompressed_name_str ( cstr s )
{
	XXXLogIn("decompressed_name_str");
	str  t = QString(100);
	uint n = decompress_name(s,t);
	XXXTRAP(n>100);
	t[n] = 0;
	XXXLogOut("= %s",t);
	return t;
}


/* ----	get name for character ---------------------------- 2002-08-11 kio ----
*/
str UCS4CharName ( UCS4Char n )
{
	return decompressed_name_str( utf8_char_info(n)->name );
}

str UTF8CharName ( UTF8Char s )
{
	return decompressed_name_str( utf8_char_info(s)->name );
}
#endif




/* ====	CLASSIFY CHARACTER ==============================================
*/

CharClass UCS4CharGeneralClass ( UCS4Char n )
{
	return utf8_char_info(n)->general_class;
}

CharClass UTF8CharGeneralClass ( UTF8Char s )
{
	return utf8_char_info(s)->general_class;
}


/* ----	is char a control code ? -------------------------
		$00..$1F, $7F, $80..$9F
*/
bool UCS4CharIsControl ( UCS4Char n )
{
	return n==0x7f || (n&~0x80)<0x20;
//	return n<0x7fu ? n<0x20u : utf8_char_info(n)->general_class == other_control;
}

bool UTF8CharIsControl ( UTF8Char s )
{
	XXTRAP(!s);
	uchar n = s[0];
	if(n<0x7f) return n<0x20;
	return utf8_char_info(s)->general_class == other_control;
}


/* ----	is char a letter ? -------------------------
		also non-latin
*/
bool UCS4CharIsLetter ( UCS4Char n )
{
	if(n<0x80u) return _is_letter(n);
	CharClass m = utf8_char_info(n)->general_class;
	return m>=letter_uppercase && m<=letter_titlecase;
}

bool UTF8CharIsLetter ( UTF8Char s )
{
	XXTRAP(!s);
	uchar n = s[0];
	if(utf8_is_7bit(n)) return _is_letter(n);
	CharClass m = utf8_char_info(s)->general_class;
	return m>=letter_uppercase && m<=letter_titlecase;
}


/* ----	is char printable ? -------------------------
		also non-latin
		***NOTE*** unshure about mark_nonspacing and mark_enclosing
*/
bool UCS4CharIsPrintable ( UCS4Char n )
{
#if XXXXSAFE
	int i=0x20;
	for(;i<0x7f;i++)  TRAP(utf8_char_info(i)->general_class>separator_space);
	for(;i<0xa0;i++)  TRAP(utf8_char_info(i)->general_class<=separator_space);
	for(;i<0x100;i++) TRAP(utf8_char_info(i)->general_class>separator_space);
	for(;i<0x999;i++) if(utf8_char_info(i)->general_class>separator_space)
					  { Log(" next non-printable char = $%4x",i); break; }
#endif

	if( n < 0x700 ) return n!=0x7f && (n&~0x80)>=0x20;
	CharClass m = utf8_char_info(n)->general_class;
	return m>=letter_uppercase && m<=separator_space;
}

bool UTF8CharIsPrintable ( UTF8Char s )
{
	XXTRAP(!s);
	uchar n = s[0];
	if(utf8_is_7bit(n)) return n>=0x20 && n!=0x7f;
	CharClass m = utf8_char_info(s)->general_class;
	return m>=letter_uppercase && m<=separator_space;
}


/* ----	is char a decimal digit ? -------------------------
		also non-latin digits
*/
bool UCS4CharIsDecimalDigit ( UCS4Char n )
{
	if ( n<256 ) return _is_dec_digit(n);
	return utf8_char_info(n)->general_class == number_decimal_digit;
}

bool UTF8CharIsDecimalDigit ( UTF8Char s )
{
	XXTRAP(!s);
	uchar n=s[0];
	if (utf8_is_7bit(n)) return _is_dec_digit(n);
	return utf8_char_info(s)->general_class == number_decimal_digit;
}


/* ----	is char a number ? -------------------------
		digits, numbers & decorated numbers
*/
bool UCS4CharIsNumber ( UCS4Char n )
{
	if ( n<256 ) return _is_dec_digit(n);
	CharClass m = utf8_char_info(n)->general_class;
	return m>=number_decimal_digit && m<=number_other;
}

bool UTF8CharIsNumber ( UTF8Char s )
{
	XXTRAP(!s);
	uchar n=s[0];
	if (utf8_is_7bit(n)) return _is_dec_digit(n);
	CharClass m = utf8_char_info(s)->general_class;
	return m>=number_decimal_digit && m<=number_other;
}


/* ----	is char a uppercase letter ? -------------------------
		=> will change if converted to lowercase
*/
bool UTF8CharIsUppercase ( UTF8Char s )
{
	XXTRAP(!s);
	char c = s[0];
	if (c>=0) return _is_uppercase(c);
	return UCS4CharIsUppercase( UCS4CharFromUTF8(s) );
}


/* ----	is char a lowercase letter ? -------------------------
		=> will change if converted to uppercase
*/
bool UTF8CharIsLowercase ( UTF8Char s )
{
	XXTRAP(!s);
	char c = s[0];
	if ( utf8_is_7bit(c) ) return _is_lowercase(c);
	return UCS4CharIsLowercase( UCS4CharFromUTF8(s) );
}


/* ----	get numeric value ------------------------------
		binary .. decimal digit value, also non-latin.  -1 if NaN
*/
long UCS4CharDigitValue ( UCS4Char n )
{
	if ( n<256 ) { n = _digit_val(n); return n<10 ? (long)n : -1; };
	const UTF8UnicodeCharInfo* u = utf8_char_info(n);
	return u->general_class==number_decimal_digit ? (long)u->number_value : -1;
}

long UTF8CharDigitValue ( UTF8Char s )
{
	XXTRAP(!s);
	char c=*s;
	if ( utf8_is_7bit(c) ) { ulong n=_digit_val(c); return n<10 ? (long)n : -1; };
	const UTF8UnicodeCharInfo* u = utf8_char_info(s);
	return u->general_class==number_decimal_digit ? (long)u->number_value : -1;
}


/* ----	get numeric value ------------------------------
		digit, number & decorated number value. some fractionals.
*/
float UCS4CharNumberValue ( UCS4Char n )
{
	return utf8_char_info(n)->number_value;		// value or NaN
}

float UTF8CharNumberValue ( UTF8Char s )
{
	return utf8_char_info(s)->number_value;		// value or NaN
}










