Commit: b7a9785f64e65c62671dcba922e77872d0324bae - mingw-catgets (git)

POSIX.1 National Language Support API for MinGW

Commit MetaInfo

• Revisión: b7a9785f64e65c62671dcba922e77872d0324bae (tree)
• Tiempo: 2007-06-19 07:36:08
• Autor: Keith Marshall <keithmarshall@user...>
• Commiter: Keith Marshall

Log Message

Handle unicode input streams.

Cambiar Resumen

• modified: ChangeLog
• modified: Makefile.in
• modified: aclocal.m4
• modified: configure
• modified: configure.ac
• modified: include/gcmsgs.h
• add: include/mcutfsig.h
• modified: mcsource.c
• add: mcutfsig.c

Diferencia incremental

--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,37 @@
[phone]  Keith Marshall  <[e-mail]>
+
+	Handle unicode input streams.
+
+	* mcutfsig.c, include/mcutfsig.h: New files; they define the
+	interface and implementation for...
+	(mc_utf_signature): ...this new function.
+
+	* mcsource.c (mc_discard): New static function.
+	(mc_source): Use it to clean up heap on abnormal termination.
+	Include `mcutfsig.h'; use `mc_utf_signature' to identify unicode
+	streams; map codeset using new local variables `input_encoding'
+	and `input_code_size'; use them with...
+	(UTF_TYPE): ...this new macro, to parse input accordingly.
+
+	* include/gcmsgs.h (MSG_HAD_CODESET): New wording.
+	(MSG_UTF_CODESET, NMSG_UTF_UNKNOWN): New message definitions.
+	(MSG_UTF_SIZE_ERROR, MSG_UTF_FRAME_ERROR): Likewise.
+
+	* Makefile.in (canonical_prefix): New macro.
+	(HEADER_TRANSFORM_SCRIPT): Use it.
+	(GENCAT_HEADERS): Add `mcutfsig.h'.
+	(GENCAT_SOURCES): Add `mcutfsig.c'.
+	(target): Macro renamed as...
+	(tool_prefix): ...this.
+	(install-progs): Updated to use it.
+	(clean): Remove all locally generated headers.
+
+	* aclocal.m4 (MINGW_AC_HOST_CANONICAL_PREFIX): New macro.
+	(MSYS_AC_CANONICAL_PATH): New macro; copied from `man' sources.
+	* configure.ac: Use them.
+	(GENCAT_AC_OBJECTS_ADD): Add `mcutfsig'.
+	* configure: Regenerated.
+
 2007-06-01  Keith Marshall  <[e-mail]>
 
 	* repl/include/langinfo.h (LC_CTYPE): Include locale.h, to define.

--- a/Makefile.in
+++ b/Makefile.in
@@ -32,6 +32,8 @@ VPATH = ${srcdir}:${srcdir}/include:${srcdir}/repl
 prefix = @prefix@
 exec_prefix = @exec_prefix@
 
+canonical_prefix = @canonical_prefix@
+
 bindir = @bindir@
 libdir = @libdir@
 includedir = @includedir@
@@ -85,8 +87,8 @@ CATGETS_HEADERS = mctab.h msgcat.h
 CATGETS_SOURCES = catopen.c catgets.c catclose.c mctab.c mcref.c mcfree.c
 CATGETS_OBJECTS = $(subst .c,.$(OBJEXT),$(CATGETS_SOURCES))
 
-GENCAT_HEADERS  = gencat.h gcmsgs.h
-GENCAT_SOURCES  = gencat.c mcload.c mcsource.c mciconv.c mcmerge.c
+GENCAT_HEADERS  = gencat.h gcmsgs.h mcutfsig.h
+GENCAT_SOURCES  = gencat.c mcload.c mcsource.c mciconv.c mcmerge.c mcutfsig.c
 GENCAT_OBJECTS  = @GENCAT_OBJECTS@
 
 OTHER_SOURCES   = COPYING ChangeLog mkinstalldirs install-sh $(MISC_SOURCES)
@@ -110,7 +112,7 @@ INSTALL_PROGS	= gendefs.awk
 	$(LN_S) $< $@
 
 HEADER_TRANSFORM_SCRIPT = \
-  -e 's,@HOST_PREFIX@,${prefix},g' \
+  -e 's,@HOST_PREFIX@,${canonical_prefix},g' \
   -e 's,@PATH_SEPARATOR_INTERNAL@,$(PATH_SEPARATOR_INTERNAL),g' \
   -e 's,$(subst .,\.,$@)\.in$$,$@ -- $(AUTOGENERATED),'
 
@@ -146,7 +148,7 @@ $(GENCAT_OBJECTS): $(GENCAT_HEADERS) $(CATGETS_OBJECTS)
 ## ============== ##
 
 mkinstalldirs = ${srcdir}/mkinstalldirs
-target = @target_alias@`test -n "@target_alias@" && echo "-"`
+tool_prefix = @target_alias@`test -n "@target_alias@" && echo "-"`
 
 INSTALL = @INSTALL@
 INSTALL_PROGRAM = @INSTALL_PROGRAM@
@@ -159,10 +161,10 @@ install: install-headers install-progs install-libs
 install-progs: all
 	$(mkinstalldirs) --require ${exec_prefix} ${bindir} ${libdir}
 	for prog in $(BUILD_PROGS); do \
-	  $(INSTALL_PROGRAM) $$prog ${bindir}/$(target)$$prog; \
+	  $(INSTALL_PROGRAM) $$prog ${bindir}/$(tool_prefix)$$prog; \
 	done
 	for prog in $(INSTALL_PROGS); do \
-	  $(INSTALL_PROGRAM) ${srcdir}/$$prog ${bindir}/$(target)$$prog; \
+	  $(INSTALL_PROGRAM) ${srcdir}/$$prog ${bindir}/$(tool_prefix)$$prog; \
 	done
 
 install-libs: all-libs $(all-dll)
@@ -230,9 +232,10 @@ bindist: all
 ## ========== ##
 
 clean:
+	rm -f nlspath.h $(LOCAL_HEADERS) $(REPLACEMENT_HEADERS)
 	rm -f *.$(OBJEXT) *~ gencat$(EXEEXT)
 
 distclean: clean
 	rm -rf Makefile config.h config.[ls]* autom4te.cache
 
-# $RCSfile$Revision$: end of file
+# $RCSfile$Revision: 1.1.1.1 $: end of file

--- a/aclocal.m4
+++ b/aclocal.m4
@@ -15,6 +15,42 @@ AC_DEFUN([MINGW_AC_WIN32_NATIVE_HOST],
 #endif]], [mingw_cv_win32_host=no], [mingw_cv_win32_host=yes]))dnl
 ])# MINGW_AC_WIN32_NATIVE_HOST
 
+# MINGW_AC_HOST_CANONICAL_PREFIX
+# ------------------------------
+# Set the AC_SUBST variable `canonical_prefix' to the canonical form
+# of `prefix', as applicable for a mingw32 host.
+#
+AC_DEFUN([MINGW_AC_HOST_CANONICAL_PREFIX],
+[AC_SUBST([canonical_prefix])dnl
+ ac_val=$prefix; test "x$ac_val" = xNONE && ac_val=$ac_default_prefix
+ MSYS_AC_CANONICAL_PATH([canonical_prefix],[$ac_val])dnl
+])# MINGW_AC_HOST_CANONICAL_PREFIX
+
+# MSYS_AC_CANONICAL_PATH( VAR, PATHNAME )
+# ---------------------------------------
+# Set VAR to the canonically resolved absolute equivalent of PATHNAME,
+# (which may be a relative path, and need not refer to any existing entity).
+#
+# On Win32-MSYS build hosts, the returned path is resolved to its true
+# native Win32 path name, (but with slashes, not backslashes).
+#
+# On any other system, it is simply the result which would be obtained
+# if PATHNAME represented an existing directory, and the pwd command was
+# executed in that directory.
+#
+AC_DEFUN([MSYS_AC_CANONICAL_PATH],
+[ac_dir="$2"
+ pwd -W >/dev/null 2>&1 && ac_pwd_w="pwd -W" || ac_pwd_w=pwd
+ until ac_val=`exec 2>/dev/null; cd "$ac_dir" && $ac_pwd_w`
+ do
+   ac_dir=`AS_DIRNAME(["$ac_dir"])`
+ done
+ ac_dir=`echo "$ac_dir" | sed 's?^[[./]]*??'`
+ ac_val=`echo "$ac_val" | sed 's?/*$[]??'`
+ $1=`echo "$2" | sed "s?^[[./]]*$ac_dir/*?$ac_val/?"'
+   s?/*$[]??'`dnl
+])# MSYS_AC_CANONICAL_PATH
+
 # MINGW_AC_CHECK_HEADER( LISTVAR, HEADER )
 # ----------------------------------------
 # Invoke AC_CHECK_HEADER, to check availability of HEADER;
@@ -152,3 +188,5 @@ AC_DEFUN([CATGETS_AC_CONFIG_VERSION_DEFINE],
 [AC_DEFINE_UNQUOTED([$1],[`IFS=.;set x $PACKAGE_VERSION;echo ${$3}`],
  [Define numerically to the catgets $2 version number])dnl
 ])# CATGETS_AC_CONFIG_VERSION_DEFINE
+
+# $RCSfile$Revision$: end of file

--- a/configure
+++ b/configure
@@ -310,7 +310,7 @@ ac_includes_default="\
 # include <unistd.h>
 #endif"
 
-ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS CC CFLAGS LDFLAGS CPPFLAGS ac_ct_CC EXEEXT OBJEXT LN_S AR ac_ct_AR INSTALL_PROGRAM INSTALL_SCRIPT INSTALL_DATA MAKE_DLL DLLVERSION HOST_PATH_SEPARATOR REPLACEMENT_HEADERS CPP EGREP LOCAL_HEADERS GENCAT_OBJECTS INCICONV LIBICONV LIBOBJS LTLIBOBJS'
+ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS CC CFLAGS LDFLAGS CPPFLAGS ac_ct_CC EXEEXT OBJEXT LN_S AR ac_ct_AR INSTALL_PROGRAM INSTALL_SCRIPT INSTALL_DATA MAKE_DLL DLLVERSION canonical_prefix HOST_PATH_SEPARATOR REPLACEMENT_HEADERS CPP EGREP LOCAL_HEADERS GENCAT_OBJECTS INCICONV LIBICONV LIBOBJS LTLIBOBJS'
 ac_subst_files=''
 
 # Initialize some variables set by options.
@@ -2508,9 +2508,31 @@ echo $ECHO_N "checking release version for mingw32 DLLs... $ECHO_C" >&6
  echo "$as_me:$LINENO: result: ${DLLVERSION}" >&5
 echo "${ECHO_T}${DLLVERSION}" >&6
 
-# We need to identify the appropriate PATH separator character,
-# to be used in the default NLSPATH definition.
+# We need to identify the canonical path prefix, and the appropriate
+# PATH separator character, to be used in the default NLSPATH definition.
 #
+   ac_val=$prefix; test "x$ac_val" = xNONE && ac_val=$ac_default_prefix
+ ac_dir="$ac_val"
+ pwd -W >/dev/null 2>&1 && ac_pwd_w="pwd -W" || ac_pwd_w=pwd
+ until ac_val=`exec 2>/dev/null; cd "$ac_dir" && $ac_pwd_w`
+ do
+   ac_dir=`(dirname "$ac_dir") 2>/dev/null ||
+$as_expr X"$ac_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_dir" : 'X\(//\)[^/]' \| \
+	 X"$ac_dir" : 'X\(//\)$' \| \
+	 X"$ac_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+ done
+ ac_dir=`echo "$ac_dir" | sed 's?^[./]*??'`
+ ac_val=`echo "$ac_val" | sed 's?/*$??'`
+ canonical_prefix=`echo "$ac_val" | sed "s?^[./]*$ac_dir/*?$ac_val/?"'
+   s?/*$??'`
   echo "$as_me:$LINENO: checking NLSPATH separator character used at runtime" >&5
 echo $ECHO_N "checking NLSPATH separator character used at runtime... $ECHO_C" >&6
 if test "${mingw_cv_host_path_separator+set}" = set; then
@@ -4332,6 +4354,7 @@ done
    GENCAT_OBJECTS=${GENCAT_OBJECTS}' mciconv.$(OBJEXT)'
    GENCAT_OBJECTS=${GENCAT_OBJECTS}' mcsource.$(OBJEXT)'
    GENCAT_OBJECTS=${GENCAT_OBJECTS}' mcmerge.$(OBJEXT)'
+   GENCAT_OBJECTS=${GENCAT_OBJECTS}' mcutfsig.$(OBJEXT)'
 
 # We also require `iconv', always expecting it to be pre-installed; we
 # need this to check if we are using a standard `libc' implementation,
@@ -5468,6 +5491,7 @@ s,@INSTALL_SCRIPT@,$INSTALL_SCRIPT,;t t
 s,@INSTALL_DATA@,$INSTALL_DATA,;t t
 s,@MAKE_DLL@,$MAKE_DLL,;t t
 s,@DLLVERSION@,$DLLVERSION,;t t
+s,@canonical_prefix@,$canonical_prefix,;t t
 s,@HOST_PATH_SEPARATOR@,$HOST_PATH_SEPARATOR,;t t
 s,@REPLACEMENT_HEADERS@,$REPLACEMENT_HEADERS,;t t
 s,@CPP@,$CPP,;t t
@@ -5970,4 +5994,4 @@ if test "$no_create" != yes; then
 fi
 
 #
-# $RCSfile$Revision$: end of file
+# $RCSfile$Revision: 1.1.1.1 $: end of file

--- a/configure.ac
+++ b/configure.ac
@@ -65,9 +65,10 @@
 #
   MINGW_AC_HOST_CONFIG_DLL([__MINGW_AC_PACKAGE_DLL_VERSION__])
 
-# We need to identify the appropriate PATH separator character,
-# to be used in the default NLSPATH definition.
+# We need to identify the canonical path prefix, and the appropriate
+# PATH separator character, to be used in the default NLSPATH definition.
 #
+  MINGW_AC_HOST_CANONICAL_PREFIX
   MINGW_AC_HOST_PATH_SEPARATOR([NLSPATH])
 
 # Schedule an automatic header update, if we find a `unistd.h',
@@ -108,6 +109,7 @@
   GENCAT_AC_OBJECTS_ADD([mciconv])
   GENCAT_AC_OBJECTS_ADD([mcsource])
   GENCAT_AC_OBJECTS_ADD([mcmerge])
+  GENCAT_AC_OBJECTS_ADD([mcutfsig])
 
 # We also require `iconv', always expecting it to be pre-installed; we
 # need this to check if we are using a standard `libc' implementation,
@@ -118,4 +120,4 @@
   AC_CONFIG_FILES([Makefile])
   AC_OUTPUT
 #
-# $RCSfile$Revision$: end of file
+# $RCSfile$Revision: 1.1.1.1 $: end of file

--- a/include/gcmsgs.h
+++ b/include/gcmsgs.h
@@ -44,7 +44,7 @@
 #define MSG_BAD_CATALOGUE    2, 1, "%s: %s: file is not a valid message catalogue\n"
 #define MSG_UNKNOWN_CODESET  2, 2, "%s: %s: unknown codeset descriptor\n"
 #define MSG_CODESET_CLASH    2, 3, "%s:%u: codeset `%s' conflicts with prior declaration\n"
-#define MSG_HAD_CODESET      2, 4, "%s:%u: codeset `%s' was previously declared here\n"
+#define MSG_HAD_CODESET      2, 4, "%s:%u: codeset `%s' previously declared here\n"
 #define MSG_SETNUM_NOT_INCR  2, 5, "invalid set number: expecting > %d; got %d\n"
 #define MSG_MSGNUM_NOT_INCR  2, 6, "invalid message number: expecting > %d; got %d\n"
 #define MSG_REDEFINED        2, 7, "%s: %s:%u: redefinition of message %u in set %u\n"
@@ -53,7 +53,11 @@
 #define MSG_TEXT_DISCARDED   3, 3, "%s:%u: incomplete message marked for deletion\n"
 #define MSG_MISSING_NEWLINE  3, 4, "%s:%u: missing newline at end of file\n"
 #define MSG_BAD_INDEX        3, 5, "invalid reference in message index"
+#define MSG_UTF_UNKNOWN      4, 1, "%s:unrecognisable encoding format\n"
+#define MSG_UTF_SIZE_ERROR   4, 2, "%s:invalid byte count per code point; value was %d\n"
+#define MSG_UTF_FRAME_ERROR  4, 3, "%s:%u:UTF-%u%cE input framing error\n"
+#define MSG_UTF_CODESET      4, 4, "%s:input codeset identified as %s; conflicts with ...\n"
 /* !
  * !$ end of file
  */
-#endif /* !defined( GCMSGS_H ): $RCSfile$Revision: 1.2 $: end of file */
+#endif /* !defined( GCMSGS_H ): $RCSfile$Revision: 1.3 $: end of file */

--- /dev/null
+++ b/include/mcutfsig.h
@@ -0,0 +1,65 @@
+#ifndef _MCUTFSIG_H_
+/*
+ * mcutfsig.h
+ *
+ * $Id$
+ *
+ * Copyright (C) 2007, Keith Marshall
+ *
+ * Header file defining the `mc_utf_signature' function API, and a set
+ * of supporting macros, used for obtaining and manipulating an encoding
+ * `signature' for UTF-8, UTF-16 and UTF-32 encoded input files.
+ *
+ * Written by Keith Marshall  <[e-mail]>
+ * Last Revision: 30-May-2007
+ *
+ *
+ * This is free software.  It is provided AS IS, in the hope that it may
+ * be useful, but WITHOUT WARRANTY OF ANY KIND, not even an IMPLIED WARRANTY
+ * of MERCHANTABILITY, nor of FITNESS FOR ANY PARTICULAR PURPOSE.
+ *
+ * Permission is granted to redistribute this software, either "as is" or
+ * in modified form, under the terms of the GNU General Public License, as
+ * published by the Free Software Foundation; either version 2, or (at your
+ * option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; see the file COPYING.  If not, write to the
+ * Free Software Foundation, 51 Franklin St - Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ *
+ */
+#define _MCUTFSIG_H_
+
+/*
+ * Flags used to designate the `endianness' of the input encoding,
+ * and to record presence or absence of a byte order mark.
+ *
+ */
+#define UTF_BIG_ENDIAN            0100
+#define UTF_LITTLE_ENDIAN         0200
+#define UTF_WITH_BYTE_ORDER_MARK  0400
+
+/*
+ * Mask used to isolate a bit-field representing the number of bytes
+ * per encoding unit in the input stream.
+ *
+ */
+#define UTF_CODE_UNIT_SIZE_MASK   0007
+
+/*
+ * Macros used to disambiguate the codeset name, wrt byte order.
+ *
+ */
+#define UTF_IS_MB(FLAGS)          (UTF_CODE_SIZE(FLAGS) > 1)
+#define UTF_CODE_SIZE(FLAGS)      ((FLAGS) & UTF_CODE_UNIT_SIZE_MASK)
+#define UTF_BYTE_ORDER(FLAGS)     (UTF_IS_MB(FLAGS) ? UTF_SUFFIX(FLAGS) : '\0')
+#define UTF_SUFFIX(FLAGS)         (((FLAGS) & UTF_LITTLE_ENDIAN) ? 'L' : 'B')
+
+/*
+ * Function prototypes.
+ *
+ */
+unsigned short mc_ucs_signature( unsigned char *stream );
+
+#endif /* !defined(_MCUTFSIG_H_): $RCSfile$Revision$: end of file */

--- a/mcsource.c
+++ b/mcsource.c
@@ -57,6 +57,7 @@
 #include <debug.h>
 
 #include <platform.h>
+#include <mcutfsig.h>
 
 #ifdef DEBUG_BUFSIZ
 # undef  BUFSIZ
@@ -205,9 +206,35 @@ char *mc_update_workspace( char *buf, char *cache, unsigned int count )
   return buf;
 }
 
+static inline
+struct msgdict *mc_discard( struct msgdict *index, char *messages )
+{
+  /* A helper function, to reclaim all memory allocated to a local
+   * message dictionary, prior to aborting compilation of the current
+   * message catalogue source file.
+   */
+  while( index )
+  {
+    /* Walk the linked list of dictionary index entries, (if any),
+     * releasing the memory block alloted to each individually.
+     */
+    struct msgdict *next = index->link;
+    free( index );
+    index = next;
+  }
+  if( messages )
+    /*
+     * All of the indexed messages are collected into a single block,
+     * which is allocated, and so must be released, separately.
+     */
+    free( messages );
+  return index;
+}
+
 struct msgdict *mc_source( const char *input )
 {
 # define CODESET_DECLARED  codeset_decl_src, codeset_decl_lineno
+# define UTF_TYPE(ORDER)   8 * input_code_size, (ORDER)
 
   dinvoke( int dtrace = 0; )
 
@@ -231,6 +258,9 @@ struct msgdict *mc_source( const char *input )
   static char *codeset = NULL;
   static const char *codeset_decl_src = NULL;
   static unsigned int codeset_decl_lineno = 0;
+
+  unsigned short input_encoding = 0, input_code_size;
+
   static iconv_t iconv_map[2] = {(iconv_t)(-1), (iconv_t)(-1)};
   char *messages; off_t msgloc, headroom;
   /*
@@ -297,6 +327,101 @@ struct msgdict *mc_source( const char *input )
     char *p = buf;
     int high_water_mark = count - ( count >> 2 );
     dfprintf(( stderr, "\n%s:%u:read %u byte%s", input, linenum, count, count == 1 ? "" : "s" ));
+
+    if( input_encoding == 0 )
+    {
+      input_encoding = mc_utf_signature( buf );
+      switch( input_code_size = input_encoding & UTF_CODE_UNIT_SIZE_MASK )
+      {
+	case 1:
+	  if( (input_encoding & UTF_WITH_BYTE_ORDER_MARK) != 0 )
+	  {
+	    /*
+	     * This is UTF-8 input encoding, affirmed by the presence of
+	     * the byte order mark, (three bytes), which we must skip.
+	     */
+	    p += 3;
+	    count -= 3;
+	  }
+	  break;
+
+	case 2:
+	case 4:
+	  if( (input_encoding & UTF_WITH_BYTE_ORDER_MARK) != 0 )
+	  {
+	    /* This is either UTF-16, or UTF-32, also affirmed by the BOM,
+	     * which occupies the first code unit, so skip it.
+	     */
+	    p += input_code_size;
+	    count -= input_code_size;
+	  }
+	  break;
+
+	default:
+	  /*
+	   * This isn't valid, for any recognisable codeset in the required
+	   * POSIX Portable Character Set input context; diagnose, clean up,
+	   * and bail out.
+	   */
+	  dfputc(( '\n', stderr ));
+	  fprintf( errmsg( MSG_UTF_UNKNOWN ), input );
+	  fprintf( errmsg( MSG_UTF_SIZE_ERROR ), input, input_code_size );
+	  free( messages );
+	  close( input_fd );
+	  return NULL;
+      }
+
+      if( input_encoding > 1 )
+      {
+	/* We've detected a UTF input encoding, which implicitly specifies
+	 * the codeset of the messages defined within this source file.
+	 */
+	char utf_byte_order = UTF_BYTE_ORDER( input_encoding );
+	sprintf( keyword, "UTF-%d%cE", 8 * input_code_size, utf_byte_order );
+
+	dfprintf(( stderr, "\n%s:", input ));
+	dinvoke( if( (input_encoding & UTF_WITH_BYTE_ORDER_MARK) != 0 ) )
+	  dfprintf(( stderr, "unicode byte order mark detected; " ));
+	dfprintf(( stderr, "encoding identified as %s", keyword ));
+
+	if( codeset != NULL ) 
+	{
+	  /* We could coalesce these two conditions into a single test,
+	   * but we choose to nest them thus, to facilitate a possible
+	   * future change, to support codeset alternation.
+	   */
+	  if( strcmp( keyword, codeset ) != 0 )
+	  {
+	    /* The detected UTF input encoding is not compatible with the
+	     * previously declared codeset of the messages in the catalogue;
+	     * diagnose, and skip this source file.
+	     */
+	    dfputc(( '\n', stderr ));
+	    fprintf( errmsg( MSG_UTF_CODESET ), input, keyword );
+	    fprintf( errmsg( MSG_HAD_CODESET ), CODESET_DECLARED, codeset );
+	    free( messages );
+	    close( input_fd );
+	    return NULL;
+	  }
+	}
+
+	else
+	{
+	  /* We don't yet have a codeset declaration; establish one implicitly,
+	   * based on the identified input encoding.
+	   */
+	  id = strdup( keyword );
+	  if( (codeset = map_codeset( iconv_map, id, "wchar_t" )) == NULL )
+	  {
+	    free( id );
+	  }
+
+	  else
+	    codeset_decl_src = input;
+	}
+      }	
+    }
+
     while( count > 0 )
     {
       /* ... scanning character by character,
@@ -325,14 +450,64 @@ struct msgdict *mc_source( const char *input )
          * transforming to the wide character domain, for local processing.
          */
         p += ((skip = iconv_mbtowc( &c, p, count )) > 0) ? skip : 0;
+
+	/* For UTF-16 or UTF-32 input encodings, the `skip' count *must*
+	 * match the codeset size, ...
+	 */
+	if( (input_code_size > 1) && (skip != input_code_size) )
+	{
+	  /* ... or we have a framing error; diagnose,
+	   * and discard this input stream.
+	   */
+	  dfputc(( '\n', stderr ));
+	  fprintf( errmsg( MSG_UTF_FRAME_ERROR ), input, linenum, codeset );
+	  return mc_discard( head, messages );
+	}
       }
 
       else
       {
         /* We are parsing context which is defined in the POSIX,
-         * or "C" locale, so read single byte character sequences.
+         * or "C" locale, so read single byte character sequences,
+	 * but stripping out any padding NULs required to fill the
+	 * input stream to a UTF-16 or UTF-32 framing boundary.
          */
+	int utf_skip = input_code_size - 1;
+	if( (utf_skip > 0) && ((input_encoding & UTF_BIG_ENDIAN) != 0) )
+	{
+	  /* Big-Endian Unicode should have padding NULs before the
+	   * POSIX `C' locale byte required.
+	   */
+	  while( (*p == '\0') && utf_skip-- && count-- )
+	    ++p;
+	  if( (utf_skip > 0) || (*p == '\0') )
+	  {
+	    /* Diagnose and bail out, if the number of padding NULs
+	     * wasn't what we expected.
+	     */
+	    dfputc(( '\n', stderr ));
+	    fprintf( errmsg( MSG_UTF_FRAME_ERROR ), input, linenum, UTF_TYPE( 'B' ));
+	    return mc_discard( head, messages );
+	  }
+	}
         c = (wchar_t)(*p++);
+	if( (utf_skip > 0) && ((input_encoding & UTF_LITTLE_ENDIAN) != 0) )
+	{
+	  /* Little-Endian Unicode should have the padding NULs after
+	   * this significant byte.
+	   */
+	  while( (*p == '\0') && utf_skip-- && count-- )
+	    ++p;
+	  if( (utf_skip > 0) || (*p == '\0') )
+	  {
+	    /* Diagnose and bail out, if the number of padding NULs
+	     * wasn't what we expected.
+	     */
+	    dfputc(( '\n', stderr ));
+	    fprintf( errmsg( MSG_UTF_FRAME_ERROR ), input, linenum, UTF_TYPE( 'L' ));
+	    return mc_discard( head, messages );
+	  }
+	}
       }
 
       if( skip > 0 )
@@ -460,12 +635,13 @@ struct msgdict *mc_source( const char *input )
 		{
 		  if( strcmp( codeset, id ) != 0 )
 		  {
+		    dfputc(( '\n', stderr ));
 		    fprintf( errmsg( MSG_CODESET_CLASH ), input, linenum, id );
 		    fprintf( errmsg( MSG_HAD_CODESET ), CODESET_DECLARED, codeset );
 		  }
 		  free( id );
 		}
-		dfprintf(( stderr, "; declare %s", keyword ));
+		dfprintf(( stderr, "\n%s:%u:declare %s", input, linenum, keyword ));
 	      }
 	    }
 
@@ -1087,4 +1263,4 @@ struct msgdict *mc_source( const char *input )
   return head;
 }
 
-/* $RCSfile$Revision: 1.9 $: end of file */
+/* $RCSfile$Revision: 1.10 $: end of file */

--- /dev/null
+++ b/mcutfsig.c
@@ -0,0 +1,118 @@
+/*
+ * mcutfsig.c
+ *
+ * $Id$
+ *
+ * Copyright (C) 2007, Keith Marshall
+ *
+ * This file implements the `mc_utf_signature' function, which is used
+ * by `gencat', to identify message definition source files which appear
+ * to exhibit any recognisable standard of Unicode encoding.
+ *
+ * Written by Keith Marshall  <[e-mail]>
+ * Last Revision: 22-May-2007
+ *
+ *
+ * This is free software.  It is provided AS IS, in the hope that it may
+ * be useful, but WITHOUT WARRANTY OF ANY KIND, not even an IMPLIED WARRANTY
+ * of MERCHANTABILITY, nor of FITNESS FOR ANY PARTICULAR PURPOSE.
+ *
+ * Permission is granted to redistribute this software, either "as is" or
+ * in modified form, under the terms of the GNU General Public License, as
+ * published by the Free Software Foundation; either version 2, or (at your
+ * option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this software; see the file COPYING.  If not, write to the
+ * Free Software Foundation, 51 Franklin St - Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ *
+ */
+#include <mcutfsig.h>
+
+unsigned short mc_utf_signature( unsigned char *stream )
+{
+  /* Inspect the first few bytes of the specified data stream;
+   * attempt to identify a potential Unicode encoding signature,
+   * defaulting to non-specific single byte encoding units.
+   */
+  unsigned short signature = 1;
+  /*
+   * The first character in the input stream must not be NUL,
+   * and must be a member of the POSIX Portable Character Set;
+   * if it isn't, then it may indicate a Unicode stream.
+   */
+  if( *stream == 0 )
+  {
+    /* An initial NUL byte anticipates a big-endian Unicode stream;
+     * one such byte implies UTF-16, without a Byte Order Mark, while
+     * two such followed by the big-endian form of the BOM, or three
+     * without a BOM, indicates UTF-32.
+     */
+    int count = 4;
+    while( count-- && (*stream++ == '\0') )
+      ++signature;
+    signature += UTF_BIG_ENDIAN;
+  }
+  if( (*stream & 0xfe) == 0xfe )
+  {
+    /* This looks like it might be a Unicode Byte Order Mark;
+     * identify the UTF encoding standard, if any, which it represents.
+     */
+    unsigned bom = *stream++ << 8; bom |= *stream++;
+    switch( bom )
+    {
+      case 0xfffe:
+	/*
+	 * This is the BOM signature for a little-endian Unicode stream;
+	 * the first byte has already been included in the initial size
+	 * assigned for the encoding unit; adjust this to accommodate the
+	 * second byte, and incorporate the little-endian flag.
+	 */
+	signature += UTF_WITH_BYTE_ORDER_MARK + UTF_LITTLE_ENDIAN + 1;
+	if( *stream == '\0' )
+	{
+	  int count = 4;
+	  while( count-- && (*stream++ == '\0') )
+	    ++signature;
+	}
+	break;
+
+      case 0xfeff:
+	/*
+	 * This is the BOM signature for a big-endian Unicode stream;
+	 * if preceded by two NULs, (already counted), then it is UTF-32,
+	 * else it is UTF-16.  In either case, adding an additional one
+	 * to the accumulated size of the encoding unit yields the
+	 * desired result, since the first byte of the BOM, and
+	 * any leading NULs, have already been counted.
+	 */
+	signature += UTF_WITH_BYTE_ORDER_MARK + UTF_BIG_ENDIAN + 1;
+	break;
+
+      case 0xffbb:
+	/*
+	 * Provided it's followed by one further `0xbf' byte, this is the
+	 * BOM used as a signature for a UTF-8 encoded stream; it becomes
+	 * invalid, if there were any leading NUL bytes.
+	 */
+	if( (signature == 1) && (*stream++ == (unsigned char)('\xbf')) )
+	  signature |= UTF_WITH_BYTE_ORDER_MARK;
+    }
+  }
+  else if( (signature == 1) && (*++stream == 0) )
+  {
+    /* NUL as the second byte in the input stream indicates a probable
+     * little-endian Unicode input stream, although this is not indicated
+     * by a Byte Order Mark; count the trailing NULs, to determine if we
+     * should interpret it as UTF-16LE, or as UTF-32LE.
+     */
+    int count = 4;
+    while( count-- && (*stream++ == '\0') )
+      ++signature;
+    signature += UTF_LITTLE_ENDIAN;
+  }
+  return signature;
+}
+
+/* $RCSfile$Revision$: end of file */