//	utf-8

/*	Copyright  (c)	Günter Woigk  2002-2003
  					mailto:kio@little-bat.de

	This file is free software

 	This program is distributed in the hope that it will be useful,
 	but WITHOUT ANY WARRANTY; without even the implied warranty of
 	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  

 	Permission to use, copy, modify, distribute, and sell this software and
 	its documentation for any purpose is hereby granted without fee, provided
 	that the above copyright notice appear in all copies and that both that
 	copyright notice and this permission notice appear in supporting
 	documentation, and that the name of the copyright holder not be used
 	in advertising or publicity pertaining to distribution of the software
 	without specific, written prior permission. The copyright holder makes no
 	representations about the suitability of this software for any purpose.
 	It is provided "as is" without express or implied warranty.

 	THE COPYRIGHT HOLDER DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
 	INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
 	EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY SPECIAL, INDIRECT OR
 	CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
 	DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
 	TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
	PERFORMANCE OF THIS SOFTWARE.


	this module provides support for UTF-8, UCS-1, UCS-2, and UCS-4 unicode characters

	it provides support for single characters.
	whole strings are handled in '$LIB/ustring/sstring.cp'.
*/


#ifndef _utf8_h_
#define	_utf8_h_

#include "kio/kio.h"
#include "cstrings/cstrings.h"



/* ********************************************************************************
						MISC. TYPE DEFINITIONS
******************************************************************************** */

typedef unsigned char		UCS1Char;			// == uchar
typedef	UCS1Char const		cUCS1Char;			// == uchar const
typedef UCS1Char *			UCS1CharPtr;		// == uptr
typedef UCS1Char const *	cUCS1CharPtr;		// == cuptr

typedef unsigned short		UCS2Char;
typedef	UCS2Char const		cUCS2Char;
typedef UCS2Char *			UCS2CharPtr;
typedef UCS2Char const *	cUCS2CharPtr;

typedef unsigned long		UCS4Char;
typedef	UCS4Char const		cUCS4Char;
typedef UCS4Char *			UCS4CharPtr;
typedef UCS4Char const *	cUCS4CharPtr;

typedef char const *		UTF8Char;
typedef char const *		cUTF8Char;
typedef char *				UTF8CharPtr;
typedef char const *		cUTF8CharPtr;

typedef	char *				UTF8Str;			// c-style string
typedef char const *		cUTF8Str;			// c-style string literals in utf-8 encoded source files




/* ********************************************************************************
					CHARACTER ENCODING for conversion
******************************************************************************** */

enum CharEncoding
{
	// character sets:
		ucs1,			iso_latin_1 = ucs1,
		ucs2,
		ucs4,
		ascii_us,		// 7-bit character set
		ascii_ger,		// 7-bit character set
		cp_437,			// the one with the line graphics
		mac_roman,
		atari_st,
		rtos,			// a private one for kio

	// multi-char encodings,
	// conversions:
		utf8,
		html,
		url,			// no '/' encoding
		url_all,		// also encode '/'
		mime,
		escaped,
		quoted,
		uppercase,		// ?
		lowercase,		// ?
		tab1,tab2,tab3,tab4,
		tab5,tab6,tab7,tab8,tab9,
};




/* ********************************************************************************
					UNICODE CHARACTER GENERAL CLASS (CATEGORY)
******************************************************************************** */

enum CharClass
{
		other_cat_unknown=0,		// ????			(no character should have this property)
		letter_uppercase,			// Lu
		letter_lowercase,			// Ll
		letter_titlecase,			// Lt
		letter_modifier,			// Lm
		letter_other,				// Lo
		mark_nonspacing,			// Mn
		mark_spacing_combining,		// Mc
		mark_enclosing,				// Me
		number_decimal_digit,		// Nd
		number_letter,				// Nl
		number_other,				// No
		punctuation_connector,		// Pc
		punctuation_dash,			// Pd
		punctuation_open,			// Ps
		punctuation_close,			// Pe
		punctuation_initial_quote,	// Pi			(may behave like Ps or Pe depending on usage)
		punctuation_final_quote,	// Pf			(may behave like Ps or Pe depending on usage)
		punctuation_other,			// Po
		symbol_math,				// Sm
		symbol_currency,			// Sc
		symbol_modifier,			// Sk
		symbol_other,				// So
		separator_space,			// Zs
		separator_line,				// Zl
		separator_paragraph,		// Zp
		other_control,				// Cc
		other_format,				// Cf
		other_surrogate,			// Cs
		other_private_use,			// Co
		other_not_assigned			// Cn			(no characters in the file have this property)
};




/* ********************************************************************************
					CLASSIFICATION for 1st byte of utf-8 character
******************************************************************************** */

inline	bool	utf8_is_null	( char c )		{ return c==0; }
inline	bool	utf8_is_7bit	( char c )		{ return c>=0; }
inline	bool	utf8_no_7bit	( char c )		{ return c<0;  }
inline	bool	utf8_is_fup		( char c )		{ return c< char(0xc0);  }
inline	bool	utf8_no_fup		( char c )		{ return c>=char(0xc0);  }
inline	bool	utf8_is_c1		( char c )		{ return c>=0; }			// == utf8_is_7bit
inline	bool	utf8_is_c2		( char c )		{ return (c&0xe0)==0xc0; }
inline	bool	utf8_is_c3		( char c )		{ return (c&0xf0)==0xe0; }
inline	bool	utf8_is_c4		( char c )		{ return (c&0xf8)==0xf0; }
inline	bool	utf8_is_c5		( char c )		{ return (c&0xfc)==0xf8; }
inline	bool	utf8_is_c6		( char c )		{ return (c&0xfe)==0xfc; }




/* ********************************************************************************
					BASIC STUFF
******************************************************************************** */

// replacement character for illegal, truncated, unprintable etc. characters
const	UCS1Char	UCS1ReplacementChar		= '?';
const	UCS2Char	UCS2ReplacementChar		= '?';
const	UCS4Char	UCS4ReplacementChar		= '?';
const	UTF8Char	UTF8ReplacementChar		= "?";
const	size_t		UTF8ReplacementCharSize	= 1;

// nominal size of utf-8 character (bytes) based on char[0]
// fups and 0xfe & 0xff are size = 1
inline	int			UTF8CharNominalSize		( char c )				{ int n=1; if (c<-2) while(char(c<<n)<0) { n++; } return n; }
inline	int			UTF8CharNominalSize		( cUTF8CharPtr p )		{ return UTF8CharNominalSize(*p); }
inline	int			UTF8CharNominalSize		( UCS1Char n )			{ return 1 + (n>0x7f); }
inline	int			UTF8CharNominalSize		( UCS2Char n )			{ return 1 + (n>0x7f) + (n>0x7ff); }
inline	int			UTF8CharNominalSize		( UCS4Char n )
{
	if(UTF8ReplacementCharSize==1) { if (long(n)<0x80) return 1; }
	else						   { if (n<0x80) return 1; if ((long)n<0) return UTF8ReplacementCharSize; }
	int i = 11; while (n>>i) i+=5; return i/5;
}


// find next character in utf-8 string
// skip current assumed non-fup byte and find next
inline	void			UTF8SkipChar		( cUTF8CharPtr& p )					{ while ( utf8_is_fup(*++p) ) {} }
inline	cUTF8CharPtr	UTF8NextChar		( cUTF8CharPtr p )					{ while ( utf8_is_fup(*++p) ) {} return p; }

// find previous character in utf-8 string
// rskip current assumed non-fup byte and find previous
inline	void			UTF8RSkipChar		( cUTF8CharPtr& p )					{ while ( utf8_is_fup(*--p) ) {} }
inline	cUTF8CharPtr	UTF8PrevChar		( cUTF8CharPtr p )					{ while ( utf8_is_fup(*--p) ) {} return p; }

// find first character in utf-8 string
// skip fups, find next non-fup byte
inline	cUTF8CharPtr	UTF8FirstChar		( cUTF8CharPtr a )					{ while(utf8_is_fup(*a)) a++; return a; }
inline	cUTF8CharPtr	UTF8FirstChar		( cUTF8CharPtr a, cUTF8CharPtr e )	{ while(a<e&&utf8_is_fup(*a)) a++; return a; }

inline	void			UTF8SkipFups		( cUTF8CharPtr& a )					{ while (utf8_is_fup(*a)) a++; }
inline	void			UTF8SkipFups		( cUTF8CharPtr& a, cUTF8CharPtr e )	{ while(a<e&&utf8_is_fup(*a)) a++; }

// find last char in utf-8 string
// rskip current non-fup byte and find previous
// identical to UTF8PrevChar()
inline	cUTF8CharPtr	UTF8LastChar		( cUTF8CharPtr e )					{ while(utf8_is_fup(*--e)) {} return e; }
inline	cUTF8CharPtr	UTF8LastChar		( cUTF8CharPtr a, cUTF8CharPtr e )	{ while(--e>=a&&utf8_is_fup(*e)) {} return e; }


/* ----	USAGE ----

		utf-8 chars in a string may be corrupted.
		running blind through a string can result in a disaster.
		special care must be taken for the first and the last character in a string.
		also remember the 'golden rule': every non-fup byte makes a character!

Example 1: running a function on all characters of a 0-terminated c-string.
		c-style strings are 'easy', because the terminating char(0) is a valid utf-8 char
		and automatically stops the parsing of the last utf-8 character in string, even if it is truncated:

		void MyStringFoo ( char const * src )
		{
			ptr a = UTF8FirstChar ( src );			// skip preceding bogus fup bytes

			while ( *a )
			{
				MyCharFoo ( a );					// do foo with the current char
				UTF8SkipChar ( a );					// skip to next
			}
		}

Example 2: running a function on all characters of a buffer without special end marker
		there is no terminating char(0) which would automatically stop the parsing
		of the last utf-8 character, in case it is truncated:

		void MyStringFoo ( char const * bu, char const * bu_end )
		{
			cptr a = UTF8FirstChar ( bu, bu_end );	// skip preceding bogus fup bytes
			if ( a>=e ) return;						// no utf-8 characters in buffer ((maybe some fups))
			cptr e = UTF8LastChar ( bu_end );		// find start of last utf-8 character

			if ( e+UTF8CharNominalSize(e) == bu_end )
			{
				e = bu_end;							// last character is not truncated => include it in loop
			}

			while ( a<e )
			{
				MyCharFoo ( a );					// do foo with the current char
				UTF8SkipChar ( a );					// skip to next
			}

			if ( e<bu_end )							// don't forget the truncated last char: remember the 'golden rule'!
			{										// running foo on the truncated char directly will run beyond string end!
				MyCharFoo ( UTF8ReplacementChar );	// do foo with a replacement for truncated last char
			}
		}
*/


/* ********************************************************************************
					UCS4 <-> UTF8 CHARACTER CONVERSION
******************************************************************************** */


extern UCS4Char		UCS4CharFromUTF8		( UTF8Char s );
extern UTF8Char		UTF8CharFromUCS4		( UCS4Char n );					// returns UTF8Char in cstring pool
extern void			UCS4CharToUTF8			( UCS4Char n, UTF8CharPtr& dest );
inline UCS4Char		UCS4CharFromUCS4		( UCS4Char n )					{ return long(n)    >=0 ? n : UCS4ReplacementChar; }
inline UCS2Char		UCS2CharFromUCS4		( UCS4Char n )					{ return UCS2Char(n)==n ? n : UCS2ReplacementChar; }
inline UCS1Char		UCS1CharFromUCS4		( UCS4Char n )					{ return UCS1Char(n)==n ? n : UCS1ReplacementChar; }



/* ****************************************************************
					CONVERSION 8-bit CHAR <-> UCS4
**************************************************************** */

typedef UCS2Char	UCS2Table[256];

extern const UCS2Table ucs2_from_mac_roman;
extern const UCS2Table ucs2_from_ascii_ger;
extern const UCS2Table ucs2_from_rtos;
extern const UCS2Table ucs2_from_cp_437;
extern const UCS2Table ucs2_from_atari_st;

inline	UCS2Char	UCS2CharFrom8Bit		( char c, const UCS2Table t){ return t[(uchar)c]; }
inline	UCS2Char	UCS2CharFromLatin1		( char c )					{ return (uchar)c; }
inline	UCS2Char	UCS2CharFromAsciiUS		( char c )					{ return c&0x7f; }
inline	UCS2Char	UCS2CharFromMacRoman	( char c )					{ return ucs2_from_mac_roman[(uchar)c]; }
inline	UCS2Char	UCS2CharFromAsciiGer	( char c )					{ return ucs2_from_ascii_ger[(uchar)c]; }
inline	UCS2Char	UCS2CharFromRtos		( char c )					{ return ucs2_from_rtos[(uchar)c]; }
inline	UCS2Char	UCS2CharFromCp437		( char c )					{ return ucs2_from_cp_437[(uchar)c]; }
inline	UCS2Char	UCS2CharFromAtariST		( char c )					{ return ucs2_from_atari_st[(uchar)c]; }

extern	char		UCS4CharTo8Bit			( UCS4Char c, const UCS2Table t );
inline	char		UCS4CharToLatin1		( UCS4Char c )				{ return c>>8 ? '?' : c; }
inline	char		UCS4CharToAsciiUS		( UCS4Char c )				{ return c>>7 ? '?' : c; }
inline	char		UCS4CharToAsciiGer		( UCS4Char c )				{ return (c|0x20)<='z' && c!='@' ? c : UCS4CharTo8Bit(c,ucs2_from_ascii_ger); }
inline	char		UCS4CharToRtos			( UCS4Char c )				{ return (c|0x20)<='z' && c!='@' ? c : UCS4CharTo8Bit(c,ucs2_from_rtos); }
inline	char		UCS4CharToMacRoman		( UCS4Char c )				{ return c>>7 ? UCS4CharTo8Bit(c,ucs2_from_mac_roman) : c; }
inline	char		UCS4CharToCp437			( UCS4Char c )				{ return c>>7 ? UCS4CharTo8Bit(c,ucs2_from_cp_437)    : c; }
inline	char		UCS4CharToAtariST		( UCS4Char c )				{ return c>>7 ? UCS4CharTo8Bit(c,ucs2_from_atari_st)  : c; }




/* ****************************************************************
					UPPERCASE / LOWERCASE
**************************************************************** */

extern UCS4Char		ucs4_char_to_lower	( UCS4Char );
extern UCS4Char		ucs4_char_to_upper	( UCS4Char );
inline UCS4Char		UCS4CharToLower		( UCS4Char c )		{ return c<'A' ? c : c<='Z' ? c^0x20 : c<0xc0 ? c : ucs4_char_to_lower(c); }
inline UCS4Char		UCS4CharToUpper		( UCS4Char c )		{ return c<'a' ? c : c<='z' ? c^0x20 : c<0xc0 ? c : ucs4_char_to_upper(c); }

extern void			UTF8CharToUpper 	( UTF8Char q, UTF8CharPtr& dest );
extern void			UTF8CharToLower 	( UTF8Char q, UTF8CharPtr& dest );
extern UTF8Char		UTF8CharToLower		( UTF8Char );						// returns UTF8Char in cstring pool
extern UTF8Char		UTF8CharToUpper		( UTF8Char );						// returns UTF8Char in cstring pool




/* ****************************************************************
					UNICODE INFO METHODS
**************************************************************** */

// official name
#if defined(_INCLUDE_UNICODE_CHARACTER_NAMES)
extern	str			UCS4CharName			( UCS4Char );	// returns 7-bit-clean name in cstring pool
extern	str			UTF8CharName			( UTF8Char );	// returns 7-bit-clean name in cstring pool
#endif

// classification
extern	CharClass	UCS4CharGeneralClass	( UCS4Char );	// see enumeration at top
extern	CharClass	UTF8CharGeneralClass	( UTF8Char );	// see enumeration at top

extern	bool		UCS4CharIsControl		( UCS4Char );	// $00..$1F, $7F, $80..$9F
extern	bool		UTF8CharIsControl		( UTF8Char );	// $00..$1F, $7F, $80..$9F

extern	bool		UCS4CharIsLetter		( UCS4Char );	// also non-latin
extern	bool		UTF8CharIsLetter		( UTF8Char );	// also non-latin

extern	bool		UCS4CharIsPrintable		( UCS4Char );	// also non-latin
extern	bool		UTF8CharIsPrintable		( UTF8Char );	// also non-latin

extern	bool		UCS4CharIsDecimalDigit	( UCS4Char );	// binary .. decimal digit, also non-latin
extern	bool		UTF8CharIsDecimalDigit	( UTF8Char );	// binary .. decimal digit, also non-latin

extern	bool		UCS4CharIsNumber		( UCS4Char );	// digits, numbers & decorated numbers
extern	bool		UTF8CharIsNumber		( UTF8Char );	// digits, numbers & decorated numbers

extern	bool 		UTF8CharIsUppercase 	( UTF8Char );	// ''will change if converted to lowercase''
inline	bool 		UCS4CharIsUppercase 	( UCS4Char n )	{ return n<0xc0 ? _is_uppercase(n) : ucs4_char_to_lower(n) != n; }

extern	bool 		UTF8CharIsLowercase 	( UTF8Char );	// ''will change if converted to uppercase''
inline	bool 		UCS4CharIsLowercase 	( UCS4Char n )	{ return n<0xc0 ? _is_lowercase(n) : ucs4_char_to_upper(n) != n; }

// numeric value
extern	long		UCS4CharDigitValue		( UCS4Char );	// binary .. decimal digit value, also non-latin.  -1 if NaN
extern	long		UTF8CharDigitValue		( UTF8Char );	// binary .. decimal digit value, also non-latin.  -1 if NaN

extern	float		UCS4CharNumberValue		( UCS4Char );	// digit, number & decorated number value. some fractionals.
extern	float		UTF8CharNumberValue		( UTF8Char );	// digit, number & decorated number value. some fractionals.

// quick variants
inline	bool		UCS4CharIsDec			( UCS4Char n )	{ return n-'0' <= '9'-'0';  }	// '0'..'9'
inline	bool		UCS4CharIsOct			( UCS4Char n )	{ return n-'0' <= '7'-'0';  }	// '0'..'7'
inline	bool		UCS4CharIsBin			( UCS4Char n )	{ return n-'0' <= '1'-'0';  }	// '0'..'1'
inline	bool		UCS4CharIsHex			( UCS4Char n )	{ return n-'0' <= '9'-'0' || (n|0x20)-'a' <= 'f'-'a'; }
inline	ulong		UCS4CharVal				( UCS4Char n )	{ return n-'0'; }				// 0..9; NaN>9
inline	ulong		UCS4CharXVal			( UCS4Char n )	{ return n<='9' ? n-'0' : (n|=0x20)>='a' ? n-'a'+10 : 99; }	// 0..36; NaN>36

inline	bool		UTF8CharIsDec			( UTF8Char p )	{ return _is_dec_digit(*p);  }	// '0'..'9'
inline	bool		UTF8CharIsOct			( UTF8Char p )	{ return _is_oct_digit(*p);  }	// '0'..'7'
inline	bool		UTF8CharIsBin			( UTF8Char p )	{ return _is_bin_digit(*p);  }	// '0'..'1'
inline	bool		UTF8CharIsHex			( UTF8Char p )	{ return _is_hex_digit(*p);  }	// 0-9,a-f,A-F
inline	ulong		UTF8CharVal				( UTF8Char p )	{ return _digit_val(*p);     }	// 0..9; NaN>9
inline	ulong		UTF8CharXVal			( UTF8Char p )	{ return _digit_value(*p);   }	// 0..36; NaN>36

// print width
inline	uint		UCS4CharPrintWidth		( UCS4Char /*n*/ )	{ return 1; }	// ***TODO*** preliminary // width of char in monospaced font  ->  0,1, or 2
inline	uint		UTF8CharPrintWidth		( UTF8Char /*s*/ )	{ return 1; }	// ***TODO*** preliminary // width of char in monospaced font  ->  0,1, or 2




#endif





