Feature/utf8 patch (#26)

* Patched, ignoring whitespace, from Kage branch

* Re-added .gitignore file

* Cleaned up some remaining merge conflicts

* Removed unused variable left over from merge

* Continued code merge cleanup...

* Cleaning up the merge

* Removing some whitespace changes

* Whitespace cleanup continues..

.gitignore

@@ -8,6 +8,7 @@ Server/src/Makefile.*
 Server/src/intsize
 Server/src/netrhost~
 Server/src/date.txt
+Server/src/buildnum.data
 Server/game/old*/*.gz
 Server/game/prev*/*.flat
 Server/game/prev*/*dump*

Mushcode/ColoredSpeech

@@ -33,11 +33,11 @@
 @@ Create hook object with default hooks and trigger attribute to add @hooks.
 @@
 @create Hook Handler=10
-&I_SAY Hook Handler=0
-&I_POSE Hook Handler=0
-&I_@EMIT Hook Handler=0
+&I_SAY Hook Handler=[not(hastype(%#,PLAYER))]
+&I_POSE Hook Handler=[not(hastype(%#,PLAYER))]
+&I_@EMIT Hook Handler=[not(hastype(%#,PLAYER))]
 &TRIG_HOOKS Hook Handler=@hook/ignore say;@hook/ignore S;@hook/ignore pose;@hook/ignore P;@hook/ignore @emit;@hook/ignore E
 
 @@ Trigger the hooks and add the hook object, or provide directions needed to setup on existing hook object.
 @@
-@ifelse [match(config(hook_obj),-1)]={@admin hook_obj=[setr(0,num(Hook Handler))];@trig %q0/TRIG_HOOKS},{think You already have a hook handler setup. You will need to copy the I_SAY, I_POSE, and I_@EMIT attributes from [setr(1,num(Hook Handler))] to your hook object and trigger the TRIG_HOOKS attribute on %q1.}
+@ifelse [match(config(hook_obj),-1)]={@admin hook_obj=[mid(setr(0,num(Hook Handler)),1,strlen(%q0))];@trig %q0/TRIG_HOOKS},{think You already have a hook handler setup. You will need to copy the I_SAY, I_POSE, and I_@EMIT attributes from [setr(1,num(Hook Handler))] to your hook object and trigger the TRIG_HOOKS attribute on %q1.}

Server/hdrs/externs.h

@@ -39,7 +39,6 @@
 #define atrpGod(s) (s == 7)
 #define atrpPreReg(s) (s == 0)
 
-
 #define SPLIT_NORMAL		0x00
 #define SPLIT_HILITE		0x01
 #define SPLIT_FLASH		0x02
@@ -57,6 +56,7 @@ typedef struct ansisplit {
 	int	i_special;	/* Special ansi characters */
 	char	c_accent;	/* Various accent characters */
         int	i_ascii8;	/* ASCII-8 encoding */
+    int i_utf8; /* UTF-8 encoding */
 } ANSISPLIT;
 
 typedef struct atrp {
@@ -144,7 +144,7 @@ extern char *	FDECL(parse_arglist, (dbref, dbref, dbref, char *, char, int,
 			char *[], int, char*[], int, int, char *[], int));
 extern int	FDECL(get_gender, (dbref));
 #ifdef ZENTY_ANSI
-extern void     FDECL(parse_ansi, (char *, char *, char **, char *, char **));
+extern void     FDECL(parse_ansi, (char *, char *, char **, char *, char **, char*, char **));
 extern int      FDECL(parse_comments, (char *, char *, char **));
 #endif
 extern char *	FDECL(mushexec, (dbref, dbref, dbref, int, char *, char *[], int, char *[], int, int, char *));
@@ -421,6 +421,10 @@ extern int FDECL(tboolchk,(char *));
 extern char *	FDECL(find_cluster, (dbref, dbref, int));
 extern void  	FDECL(trigger_cluster_action, (dbref, dbref));
 
+extern char *   FDECL(encode_utf8, (char *));
+extern char * 	FDECL(utf8toucp, (char *));
+extern char * 	FDECL(ucptoutf8, (char *));
+
 /* From boolexp.c */
 extern int	FDECL(eval_boolexp, (dbref, dbref, dbref, BOOLEXP *, int));
 extern BOOLEXP *FDECL(parse_boolexp, (dbref,const char *, int));

Server/hdrs/flags.h

@@ -245,7 +245,7 @@
 #define TOG_ACCENTS		0x00400000	/* Accents being displayed */
 #define TOG_PREMAILVALIDATE	0x00800000	/* Pre-Validate the mail send list before sending mail */
 #define TOG_SAFELOG             0x01000000	/* Allow 'clean logging' by the player */
-/* 0x02000000 free */
+#define TOG_UTF8			0x02000000	/* UTF8 being displayed */
 /* 0x04000000 free */
 #define TOG_NODEFAULT		0x08000000	/* Allow target to inherit default attribs */
 #define TOG_EXFULLWIZATTR	0x10000000	/* Examine Wiz attribs */
@@ -668,6 +668,7 @@ extern int	FDECL(has_aflag, (dbref, dbref, int, char *));
 #define TogHideIdle(x)	((Toggles2(x) & TOG_HIDEIDLE) != 0)
 #define TogMortReal(x)	((Toggles2(x) & TOG_MORTALREALITY) != 0)
 #define Accents(x)	((Toggles2(x) & TOG_ACCENTS) != 0)
+#define UTF8(x)		((Toggles2(x) & TOG_UTF8) != 0)
 #define MailValid(x)	((Toggles2(x) & TOG_PREMAILVALIDATE) != 0)
 #define KeepAlive(x)	((Toggles2(x) & TOG_KEEPALIVE) != 0)
 #define ChkReality(x)	((Toggles2(x) & TOG_CHKREALITY) != 0)

Server/hdrs/mudconf.h

@@ -453,6 +453,8 @@ struct confdata {
 	int	mysql_port;
 #endif
 	int	name_with_desc;	/* Toggle to enable names with descs when looking (if not-examinable) */
+    int allow_fancy_quotes; /* Allow Unicode 'fancy' quotes or replace them with standard ascii quotes */
+    int allow_fullwidth_colon; /* Allow unicode fullwidth colon or replace it with ascii colon */
 #else
 	int	paylimit;	/* getting money gets hard over this much */
 	int	digcost;	/* cost of @dig command */

Server/src/bsd.c

@@ -48,6 +48,7 @@ void bzero(void *, int);
 #include "attrs.h"
 #include "mail.h"
 #include "rhost_ansi.h"
+#include "rhost_utf8.h"
 #include "local.h"
 #include "door.h"
 
@@ -367,7 +368,7 @@ shovechars(int port,char* address)
          s_cutter2[8], *progatr_str, *progatr_strptr, *s_progatr, *b_progatr, 
          *t_progatr, *b_progatrptr, *tstrtokr;
 #ifdef ZENTY_ANSI
-    char *s_buff, *s_buffptr, *s_buff2, *s_buff2ptr;
+    char *s_buff, *s_buffptr, *s_buff2, *s_buff2ptr, *s_buff3, *s_buff3ptr;
 #endif
     FILE *f;
     int silent, i_progatr, anum;
@@ -455,10 +456,12 @@ shovechars(int port,char* address)
 #ifdef ZENTY_ANSI
                   s_buffptr = s_buff = alloc_lbuf("parse_ansi_prompt");
                   s_buff2ptr = s_buff2 = alloc_lbuf("parse_ansi_prompt2");
-                  parse_ansi((char *) progatr, s_buff, &s_buffptr, s_buff2, &s_buff2ptr);
+                  s_buff3ptr = s_buff3 = alloc_lbuf("parse_ansi_prompt3");
+                  parse_ansi((char *) progatr, s_buff, &s_buffptr, s_buff2, &s_buff2ptr, s_buff3, &s_buff3ptr);
                   queue_string(d, unsafe_tprintf("%s%s%s \377\371", ANSI_HILITE, s_buff, ANSI_NORMAL));
                   free_lbuf(s_buff);
                   free_lbuf(s_buff2);
+                  free_lbuf(s_buff3);
 #else
                   queue_string(d, unsafe_tprintf("%s%s%s \377\371", ANSI_HILITE, progatr, ANSI_NORMAL));
 #endif
@@ -753,6 +756,7 @@ shovechars(int port,char* address)
 		/* Process received data */
 
                 i_oldlastcnt = d->input_tot;
+
 		if (!process_input(d)) {
 		    shutdownsock(d, R_SOCKDIED);
 		    continue;
@@ -2027,14 +2031,15 @@ process_input(DESC * d)
 {
     static char buf[LBUF_SIZE];
     int got, in, lost;
-    char *p, *pend, *q, *qend, qfind[8], *qf;
+    char *p, *pend, *q, *qend, qfind[12], *qf, *tmpptr = NULL, tmpbuf[15];
     char *cmdsave;
 
     DPUSH; /* #16 */
     cmdsave = mudstate.debug_cmd;
     mudstate.debug_cmd = (char *) "< process_input >";
 
     memset(qfind, '\0', sizeof(qfind));
+    memset(tmpbuf, '\0', sizeof(tmpbuf));
     got = in = READ(d->descriptor, buf, sizeof buf);
     if (got <= 0) {
 	mudstate.debug_cmd = cmdsave;
@@ -2071,20 +2076,45 @@ process_input(DESC * d)
 		(d->raw_input_at)--;
   	} else if (p < pend && isascii((int)*q) && isprint((int)*q)) {
 	    *p++ = *q;
-        } else if ( (((int)(unsigned char)*q) > 160) && 
-                    ((!mudconf.accent_extend && ((int)(unsigned char)*q) < 250) || (mudconf.accent_extend && ((int)(unsigned char)*q) < 256)) && 
-                    ((p+10) < pend) ) {
-            if ( (((int)(unsigned char)*q == 255) && *(q++) != '\0') || ((int)(unsigned char)*q != 255) ) {
-               sprintf(qfind, "%c<%3d>", '%', (int)(unsigned char)*q);
-               in+=5;
-               got+=5;
+        } else if ((p+13) < pend && IS_4BYTE((int)(unsigned char)*q) && IS_CBYTE(*(q+1)) && IS_CBYTE(*(q+2)) && IS_CBYTE(*(q+3))) {
+            sprintf(tmpbuf, "%02x%02x%02x%02x", (int)(unsigned char)*q, (int)(unsigned char)*(q+1), (int)(unsigned char)*(q+2), (int)(unsigned char)*(q+3));
+            tmpptr = encode_utf8(tmpbuf);
+            sprintf(qfind, "%s", tmpptr);
+
+            q+=3;
+            in+=12;
+            got+=12;
                qf = qfind;
                while ( *qf ) {
                   *p++ = *qf++;
                }
+        } else if ((p+13) < pend && IS_3BYTE((int)(unsigned char)*q) && IS_CBYTE(*(q+1)) && IS_CBYTE(*(q+2))) {
+            sprintf(tmpbuf, "%02x%02x%02x", (int)(unsigned char)*q, (int)(unsigned char)*(q+1), (int)(unsigned char)*(q+2));
+            tmpptr = encode_utf8(tmpbuf);
+            sprintf(qfind, "%s", tmpptr);
+
+            q+=2;
+            in+=10;
+            got+=10;
+            qf = qfind;
+            while (*qf) {
+                *p++ = *qf++;
+            }   
+        } else if ((p+13) < pend && IS_2BYTE((int)(unsigned char)*q) && IS_CBYTE(*(q+1))) {
+            sprintf(tmpbuf, "%02x%02x", (int)(unsigned char)*q, (int)(unsigned char)*(q+1));
+            tmpptr = encode_utf8(tmpbuf);
+            sprintf(qfind, "%s", tmpptr);
+
+            q+=1;
+            in+=8;
+            got+=8;
+            qf = qfind;
+            while (*qf) {
+                *p++ = *qf++;
             }
         } else if ( (((int)(unsigned char)*q) == 255) && (((int)(unsigned char)*(q+1)) != '\0') ) {
             q++;
+
 	} else {
 	    in--;
 	    if (p >= pend)
@@ -2104,6 +2134,11 @@ process_input(DESC * d)
        d->input_size += in;
     if ( lost > 0 )
        d->input_lost += lost;
+
+    if (tmpptr) {
+        free(tmpptr);
+    }
+
     mudstate.debug_cmd = cmdsave;
     RETURN(1); /* #16 */
 }

Server/src/command.c

@@ -9436,7 +9436,7 @@ void do_program(dbref player, dbref cause, int key, char *name, char *command)
    char *buf, *attrib, *tmplbuf, *tmplbufptr, *progatr, strprompt[LBUF_SIZE], *tpr_buff, *tprp_buff;
    DESC *d;
 #ifdef ZENTY_ANSI
-   char *s_buff, *s_buff2, *s_buffptr, *s_buff2ptr;
+   char *s_buff, *s_buff2, *s_buff3, *s_buffptr, *s_buff2ptr, *s_buff3ptr;
 #endif
 
    if (!*name || !name) {
@@ -9508,10 +9508,12 @@ void do_program(dbref player, dbref cause, int key, char *name, char *command)
 #ifdef ZENTY_ANSI
               s_buffptr = s_buff = alloc_lbuf("parse_ansi_prompt");
               s_buff2ptr = s_buff2 = alloc_lbuf("parse_ansi_prompt2");
-              parse_ansi((char *) strprompt, s_buff, &s_buffptr, s_buff2, &s_buff2ptr);
+              s_buff3ptr = s_buff3 = alloc_lbuf("parse_ansi_prompt3");
+              parse_ansi((char *) strprompt, s_buff, &s_buffptr, s_buff2, &s_buff2ptr, s_buff3, &s_buff3ptr);
               queue_string(d, safe_tprintf(tpr_buff, &tprp_buff, "%s%s%s \377\371", ANSI_HILITE, s_buff, ANSI_NORMAL));
               free_lbuf(s_buff);
               free_lbuf(s_buff2);
+              free_lbuf(s_buff3);
 #else
               queue_string(d, safe_tprintf(tpr_buff, &tprp_buff, "%s%s%s \377\371", ANSI_HILITE, strprompt, ANSI_NORMAL));
 #endif
@@ -11818,7 +11820,7 @@ do_progreset(dbref player, dbref cause, int key, char *name)
    char *buff = NULL, *tpr_buff, *tprp_buff;
    int i_buff = 0;
 #ifdef ZENTY_ANSI
-   char *s_buff, *s_buff2, *s_buffptr, *s_buff2ptr;
+   char *s_buff, *s_buff2, *s_buff3, *s_buffptr, *s_buff2ptr, *s_buff3ptr;
 #endif
 
    if ( !name && !*name ) {
@@ -11866,10 +11868,12 @@ do_progreset(dbref player, dbref cause, int key, char *name)
 #ifdef ZENTY_ANSI
               s_buffptr = s_buff = alloc_lbuf("parse_ansi_prompt");
               s_buff2ptr = s_buff2 = alloc_lbuf("parse_ansi_prompt2");
-              parse_ansi((char *) buff, s_buff, &s_buffptr, s_buff2, &s_buff2ptr);
+              s_buff3ptr = s_buff3 = alloc_lbuf("parse_ansi_prompt3");
+              parse_ansi((char *) buff, s_buff, &s_buffptr, s_buff2, &s_buff2ptr, s_buff3, &s_buff3ptr);
               queue_string(d, safe_tprintf(tpr_buff, &tprp_buff, "%s%s%s \377\371", ANSI_HILITE, s_buff, ANSI_NORMAL));
               free_lbuf(s_buff);
               free_lbuf(s_buff2);
+              free_lbuf(s_buff3);
 #else
               queue_string(d, safe_tprintf(tpr_buff, &tprp_buff, "%s%s%s \377\371", ANSI_HILITE, buff, ANSI_NORMAL));
 #endif

Server/src/conf.c

@@ -767,6 +767,8 @@ NDECL(cf_init)
     mudstate.nameofqreg[10]='\0';
 #endif
     mudstate.emit_substitute = 0; /* Toggle @emit/substitute */
+    mudconf.allow_fancy_quotes = 0;  /* Allow UTF-8 double quote characters */
+    mudconf.allow_fullwidth_colon = 0; /* Allow UTF-8 fullwidth colon character */
 #else
     mudconf.paylimit = 10000;
     mudconf.digcost = 10;
@@ -5051,6 +5053,14 @@ CONF conftable[] =
      (char *) "Default object TX level.\r\n"\
               "                             Default: 1   Value: %d"},
 #endif /* REALITY_LEVELS */
+    {(char *)"allow_fancy_quotes",
+     cf_int, CA_GOD | CA_IMMORTAL, &mudconf.allow_fancy_quotes, 0, 0, CA_WIZARD,
+     (char *) "Allow UTF-8 encoded double quotes.\r\n"\
+              "                             Default: 0   Value: %d"},
+    {(char *)"allow_fullwidth_colon",
+     cf_int, CA_GOD | CA_IMMORTAL, &mudconf.allow_fullwidth_colon, 0, 0, CA_WIZARD,
+              "Allow UTF-8 encoded full width colon character.\r\n"\
+              "                             Default: 0   Value: %d"},
     {NULL,
      NULL, 0, NULL, 0}};