Massive Python/Pyrex wrapper cleanup. We now use Pyxelator to generate

the C function and data structure wrappers for Pyrex.  We now require
Pyrex 0.9.5 to build the Python wrappers, as only >=0.9.5 has working
weakref support.  We actually need 0.9.5.1, but it's not yet released
(0.9.5 has a crash bug in enum wrapping that we trigger).

Pyxelator works a lot better with non-anonymous enums/structs, so libyasm
has been scrubbed for this.

Next step: full Yasm data structure inspection.

svn path=/trunk/yasm/; revision=1745

configure.ac

@@ -257,8 +257,8 @@ else
     if test -z "$PYTHON" || test "$PYTHON" = : ; then
         have_python=no
     else
-        AC_MSG_CHECKING([for Pyrex >= 0.9.3])
-        PYREX_CHECK_VERSION(0.9.3, [AC_MSG_RESULT(yes)
+        AC_MSG_CHECKING([for Pyrex >= 0.9.5])
+        PYREX_CHECK_VERSION(0.9.5, [AC_MSG_RESULT(yes)
                                     have_pyrex=yes],
                                    [AC_MSG_RESULT(no)
                                     have_pyrex=no])

libyasm.h

@@ -44,8 +44,14 @@
 #ifndef YASM_LIB_H
 #define YASM_LIB_H
 
+#ifdef YASM_PYXELATOR
+typedef struct __FILE FILE;
+typedef struct __va_list va_list;
+typedef unsigned long size_t;
+#else
 #include <stdio.h>
 #include <stdarg.h>
+#endif
 
 #ifdef YASM_LIB_INTERNAL
 # include <libyasm/compat-queue.h>

libyasm/arch.h

@@ -35,14 +35,14 @@
 #define YASM_ARCH_H
 
 /** Errors that may be returned by yasm_arch_module::create(). */
-typedef enum {
+typedef enum yasm_arch_create_error {
     YASM_ARCH_CREATE_OK = 0,		/**< No error. */
     YASM_ARCH_CREATE_BAD_MACHINE,	/**< Unrecognized machine name. */
     YASM_ARCH_CREATE_BAD_PARSER		/**< Unrecognized parser name. */
 } yasm_arch_create_error;
 
 /** Return values for yasm_arch_module::parse_check_insnprefix(). */
-typedef enum {
+typedef enum yasm_arch_insnprefix {
     YASM_ARCH_NOTINSNPREFIX = 0,	/**< Unrecognized */
     YASM_ARCH_INSN,			/**< An instruction */
     YASM_ARCH_PREFIX			/**< An instruction prefix */
@@ -51,7 +51,7 @@ typedef enum {
 /** Types of registers / target modifiers that may be returned by
  * yasm_arch_module::parse_check_regtmod().
  */
-typedef enum {
+typedef enum yasm_arch_regtmod {
     YASM_ARCH_NOTREGTMOD = 0,		/**< Unrecognized */
     YASM_ARCH_REG,			/**< A "normal" register */
     YASM_ARCH_REGGROUP,			/**< A group of indexable registers */
@@ -248,7 +248,7 @@ struct yasm_insn_operand {
     /*@reldef@*/ STAILQ_ENTRY(yasm_insn_operand) link;
 
     /** Operand type. */
-    enum {
+    enum yasm_insn_operand_type {
 	YASM_INSN__OPERAND_REG = 1,	/**< A register. */
 	YASM_INSN__OPERAND_SEGREG,	/**< A segment register. */
 	YASM_INSN__OPERAND_MEMORY,	/**< An effective address

libyasm/bc-int.h

@@ -38,7 +38,7 @@ typedef struct yasm_bytecode_callback {
     int (*tobytes) (yasm_bytecode *bc, unsigned char **bufp, void *d,
 		    yasm_output_value_func output_value,
 		    /*@null@*/ yasm_output_reloc_func output_reloc);
-    enum {
+    enum yasm_bytecode_special_type {
 	YASM_BC_SPECIAL_NONE = 0,
 	YASM_BC_SPECIAL_RESERVE,/* Reserves space instead of outputting data */
 	YASM_BC_SPECIAL_OFFSET	/* Adjusts offset instead of calculating len */

libyasm/bitvect.h

@@ -82,15 +82,15 @@ typedef  Z_longword         *Z_longwordptr;
     #ifdef MACOS_TRADITIONAL
         #define boolean Boolean
     #else
-        typedef enum { false = FALSE, true = TRUE } boolean;
+        typedef enum boolean { false = FALSE, true = TRUE } boolean;
     #endif
 #endif
 
 /*****************************************************************************/
 /*  MODULE INTERFACE:                                                        */
 /*****************************************************************************/
 
-typedef enum
+typedef enum ErrCode
     {
         ErrCode_Ok = 0,    /* everything went allright                       */
 

libyasm/coretype.h

@@ -184,7 +184,7 @@ typedef struct yasm_valparamhead yasm_valparamhead;
 typedef struct yasm_insn_operands yasm_insn_operands;
 
 /** Expression operators usable in #yasm_expr expressions. */
-typedef enum {
+typedef enum yasm_expr_op {
     YASM_EXPR_IDENT,	/**< No operation, just a value. */
     YASM_EXPR_ADD,	/**< Arithmetic addition (+). */
     YASM_EXPR_SUB,	/**< Arithmetic subtraction (-). */
@@ -225,7 +225,7 @@ typedef enum {
  * \see symrec.h for related functions.
  * \note YASM_SYM_EXTERN and YASM_SYM_COMMON are mutually exclusive.
  */
-typedef enum {
+typedef enum yasm_sym_vis {
     YASM_SYM_LOCAL = 0,		/**< Default, local only */
     YASM_SYM_GLOBAL = 1 << 0,	/**< If symbol is declared GLOBAL */
     YASM_SYM_COMMON = 1 << 1,	/**< If symbol is declared COMMON */

libyasm/errwarn.h

@@ -35,7 +35,7 @@
 #define YASM_ERRWARN_H
 
 /** Warning classes (that may be enabled/disabled). */
-typedef enum {
+typedef enum yasm_warn_class {
     YASM_WARN_NONE = 0,	    /**< No warning */
     YASM_WARN_GENERAL,	    /**< Non-specific warnings */
     YASM_WARN_UNREC_CHAR,   /**< Unrecognized characters (while tokenizing) */
@@ -45,7 +45,7 @@ typedef enum {
 } yasm_warn_class;
 
 /** Error classes.  Bitmask-based to support limited subclassing. */
-typedef enum {
+typedef enum yasm_error_class {
     YASM_ERROR_NONE		= 0x0000, /**< No error */
     YASM_ERROR_GENERAL		= 0xFFFF, /**< Non-specific */
     YASM_ERROR_ARITHMETIC	= 0x0001, /**< Arithmetic error (general) */

libyasm/expr-int.h

@@ -31,7 +31,7 @@
  * Note precbc must be used carefully (in a-b pairs), as only symrecs can
  * become the relative term in a #yasm_value.
  */
-typedef enum {
+typedef enum yasm_expr__type {
     YASM_EXPR_NONE = 0,
     YASM_EXPR_REG = 1<<0,
     YASM_EXPR_INT = 1<<1,

libyasm/symrec.c

@@ -47,15 +47,6 @@
 #include "objfmt.h"
 
 
-/* DEFINED is set with EXTERN and COMMON below */
-typedef enum {
-    SYM_NOSTATUS = 0,
-    SYM_USED = 1 << 0,		/* for using variables before definition */
-    SYM_DEFINED = 1 << 1,	/* once it's been defined in the file */
-    SYM_VALUED = 1 << 2,	/* once its value has been determined */
-    SYM_NOTINTABLE = 1 << 3	/* if it's not in sym_table (ex. '$') */
-} sym_status;
-
 typedef enum {
     SYM_UNKNOWN,		/* for unknown type (COMMON/EXTERN) */
     SYM_EQU,			/* for EQU defined symbols (expressions) */
@@ -70,7 +61,7 @@ typedef enum {
 struct yasm_symrec {
     char *name;
     sym_type type;
-    sym_status status;
+    yasm_sym_status status;
     yasm_sym_vis visibility;
     unsigned long line;		/*  symbol was first declared or used on */
     union {
@@ -136,7 +127,7 @@ symtab_get_or_new_in_table(yasm_symtab *symtab, /*@only@*/ char *name)
     yasm_symrec *rec = symrec_new_common(name);
     int replace = 0;
 
-    rec->status = SYM_NOSTATUS;
+    rec->status = YASM_SYM_NOSTATUS;
 
     return HAMT_insert(symtab->sym_table, name, rec, &replace,
 		       symrec_destroy_one);
@@ -148,7 +139,7 @@ symtab_get_or_new_not_in_table(yasm_symtab *symtab, /*@only@*/ char *name)
     non_table_symrec *sym = yasm_xmalloc(sizeof(non_table_symrec));
     sym->rec = symrec_new_common(name);
 
-    sym->rec->status = SYM_NOTINTABLE;
+    sym->rec->status = YASM_SYM_NOTINTABLE;
 
     SLIST_INSERT_HEAD(&symtab->non_table_syms, sym, link);
 
@@ -202,7 +193,7 @@ yasm_symtab_abs_sym(yasm_symtab *symtab)
     rec->type = SYM_EQU;
     rec->value.expn =
 	yasm_expr_create_ident(yasm_expr_int(yasm_intnum_create_uint(0)), 0);
-    rec->status |= SYM_DEFINED|SYM_VALUED|SYM_USED;
+    rec->status |= YASM_SYM_DEFINED|YASM_SYM_VALUED|YASM_SYM_USED;
     return rec;
 }
 
@@ -212,7 +203,7 @@ yasm_symtab_use(yasm_symtab *symtab, const char *name, unsigned long line)
     yasm_symrec *rec = symtab_get_or_new(symtab, name, 1);
     if (rec->line == 0)
 	rec->line = line;	/* set line number of first use */
-    rec->status |= SYM_USED;
+    rec->status |= YASM_SYM_USED;
     return rec;
 }
 
@@ -229,7 +220,7 @@ symtab_define(yasm_symtab *symtab, const char *name, sym_type type,
     yasm_symrec *rec = symtab_get_or_new(symtab, name, in_table);
 
     /* Has it been defined before (either by DEFINED or COMMON/EXTERN)? */
-    if (rec->status & SYM_DEFINED) {
+    if (rec->status & YASM_SYM_DEFINED) {
 	yasm_error_set_xref(rec->line, N_("`%s' previously defined here"),
 			    name);
 	yasm_error_set(YASM_ERROR_GENERAL, N_("redefinition of `%s'"),
@@ -240,7 +231,7 @@ symtab_define(yasm_symtab *symtab, const char *name, sym_type type,
 			  N_("`%s' both defined and declared extern"), name);
 	rec->line = line;	/* set line number of definition */
 	rec->type = type;
-	rec->status |= SYM_DEFINED;
+	rec->status |= YASM_SYM_DEFINED;
     }
     return rec;
 }
@@ -251,7 +242,7 @@ yasm_symtab_define_equ(yasm_symtab *symtab, const char *name, yasm_expr *e,
 {
     yasm_symrec *rec = symtab_define(symtab, name, SYM_EQU, 1, line);
     rec->value.expn = e;
-    rec->status |= SYM_VALUED;
+    rec->status |= YASM_SYM_VALUED;
     return rec;
 }
 
@@ -283,7 +274,7 @@ yasm_symtab_define_special(yasm_symtab *symtab, const char *name,
 			   yasm_sym_vis vis)
 {
     yasm_symrec *rec = symtab_define(symtab, name, SYM_SPECIAL, 1, 0);
-    rec->status |= SYM_VALUED;
+    rec->status |= YASM_SYM_VALUED;
     rec->visibility = vis;
     return rec;
 }
@@ -312,7 +303,7 @@ yasm_symrec_declare(yasm_symrec *rec, yasm_sym_vis vis, unsigned long line)
      * X   1      -      1      -
      */
     if ((vis == YASM_SYM_GLOBAL) ||
-	(!(rec->status & SYM_DEFINED) &&
+	(!(rec->status & YASM_SYM_DEFINED) &&
 	 (!(rec->visibility & (YASM_SYM_COMMON | YASM_SYM_EXTERN)) ||
 	  ((rec->visibility & YASM_SYM_COMMON) && (vis == YASM_SYM_COMMON)) ||
 	  ((rec->visibility & YASM_SYM_EXTERN) && (vis == YASM_SYM_EXTERN)))))
@@ -337,7 +328,7 @@ symtab_parser_finalize_checksym(yasm_symrec *sym, /*@null@*/ void *d)
     yasm_section *sect;
 
     /* error if a symbol is used but never defined or extern/common declared */
-    if ((sym->status & SYM_USED) && !(sym->status & SYM_DEFINED) &&
+    if ((sym->status & YASM_SYM_USED) && !(sym->status & YASM_SYM_DEFINED) &&
 	!(sym->visibility & (YASM_SYM_EXTERN | YASM_SYM_COMMON))) {
 	if (info->undef_extern && info->objfmt)
 	    yasm_objfmt_extern_declare(info->objfmt, sym->name, NULL, 1);
@@ -442,6 +433,12 @@ yasm_symrec_get_visibility(const yasm_symrec *sym)
     return sym->visibility;
 }
 
+yasm_sym_status
+yasm_symrec_get_status(const yasm_symrec *sym)
+{
+    return sym->status;
+}
+
 unsigned long
 yasm_symrec_get_line(const yasm_symrec *sym)
 {
@@ -530,16 +527,16 @@ yasm_symrec_print(const yasm_symrec *sym, FILE *f, int indent_level)
     }
 
     fprintf(f, "%*sStatus=", indent_level, "");
-    if (sym->status == SYM_NOSTATUS)
+    if (sym->status == YASM_SYM_NOSTATUS)
 	fprintf(f, "None\n");
     else {
-	if (sym->status & SYM_USED)
+	if (sym->status & YASM_SYM_USED)
 	    fprintf(f, "Used,");
-	if (sym->status & SYM_DEFINED)
+	if (sym->status & YASM_SYM_DEFINED)
 	    fprintf(f, "Defined,");
-	if (sym->status & SYM_VALUED)
+	if (sym->status & YASM_SYM_VALUED)
 	    fprintf(f, "Valued,");
-	if (sym->status & SYM_NOTINTABLE)
+	if (sym->status & YASM_SYM_NOTINTABLE)
 	    fprintf(f, "Not in Table,");
 	fprintf(f, "\n");
     }

libyasm/symrec.h

@@ -34,6 +34,19 @@
 #ifndef YASM_SYMREC_H
 #define YASM_SYMREC_H
 
+/** Symbol status.  YASM_SYM_DEFINED is set by yasm_symtab_define_label(),
+ * yasm_symtab_define_equ(), or yasm_symtab_declare()/yasm_symrec_declare()
+ * with a visibility of #YASM_SYM_EXTERN or #YASM_SYM_COMMON.
+ */
+typedef enum yasm_sym_status {
+    YASM_SYM_NOSTATUS = 0,	    /**< no status */
+    YASM_SYM_USED = 1 << 0,	    /**< for use before definition */
+    YASM_SYM_DEFINED = 1 << 1,	    /**< once it's been defined in the file */
+    YASM_SYM_VALUED = 1 << 2,	    /**< once its value has been determined */
+    YASM_SYM_NOTINTABLE = 1 << 3    /**< if it's not in sym_table (ex. '$') */
+} yasm_sym_status;
+
+
 /** Create a new symbol table. */
 yasm_symtab *yasm_symtab_create(void);
 
@@ -217,6 +230,12 @@ void yasm_symtab_print(yasm_symtab *symtab, FILE *f, int indent_level);
  */
 yasm_sym_vis yasm_symrec_get_visibility(const yasm_symrec *sym);
 
+/** Get the status of a symbol.
+ * \param sym	    symbol
+ * \return Symbol status.
+ */
+yasm_sym_status yasm_symrec_get_status(const yasm_symrec *sym);
+
 /** Get the virtual line of a symbol (where it was first declared or used).
  * \param sym	    symbol
  * \return line	    virtual line

tools/python-yasm/Makefile.inc

@@ -1,23 +1,50 @@
 # $Id$
 
 PYBINDING_DEPS  = tools/python-yasm/bytecode.pxi
-PYBINDING_DEPS += tools/python-yasm/coretype.pxi
 PYBINDING_DEPS += tools/python-yasm/errwarn.pxi
 PYBINDING_DEPS += tools/python-yasm/expr.pxi
 PYBINDING_DEPS += tools/python-yasm/floatnum.pxi
 PYBINDING_DEPS += tools/python-yasm/intnum.pxi
 PYBINDING_DEPS += tools/python-yasm/symrec.pxi
 PYBINDING_DEPS += tools/python-yasm/value.pxi
 
+EXTRA_DIST += tools/python-yasm/pyxelator/cparse.py
+EXTRA_DIST += tools/python-yasm/pyxelator/genpyx.py
+EXTRA_DIST += tools/python-yasm/pyxelator/ir.py
+EXTRA_DIST += tools/python-yasm/pyxelator/lexer.py
+EXTRA_DIST += tools/python-yasm/pyxelator/node.py
+EXTRA_DIST += tools/python-yasm/pyxelator/parse_core.py
+EXTRA_DIST += tools/python-yasm/pyxelator/work_unit.py
+EXTRA_DIST += tools/python-yasm/pyxelator/wrap_yasm.py
 EXTRA_DIST += tools/python-yasm/setup.py
 EXTRA_DIST += tools/python-yasm/yasm.pyx
 EXTRA_DIST += $(PYBINDING_DEPS)
 
 if HAVE_PYTHON
 
-yasm_python.c: $(srcdir)/tools/python-yasm/yasm.pyx $(PYBINDING_DEPS)
+# Use Pyxelator to generate Pyrex function headers.
+_yasm.pxi: ${libyasm_a_SOURCES}
+	@rm -rf .tmp
+	@mkdir .tmp
+	$(PYTHON) $(srcdir)/tools/python-yasm/pyxelator/wrap_yasm.py \
+		"YASM_DIR=${srcdir}" "CPP=${CPP}" "CPPFLAGS=${CPPFLAGS}"
+	@rm -rf .tmp
+
+CLEANFILES += _yasm.pxi
+
+# Need to build a local copy of the main Pyrex input file to include _yasm.pxi
+# from the build directory.  Also need to fixup the other .pxi include paths.
+yasm.pyx: $(srcdir)/tools/python-yasm/yasm.pyx
+	cat $(srcdir)/tools/python-yasm/yasm.pyx \
+		| sed -e 's,^include "\([^_]\),include "${srcdir}/tools/python-yasm/\1,' \
+		> $@
+
+CLEANFILES += yasm.pyx
+
+# Actually run Pyrex
+yasm_python.c: yasm.pyx _yasm.pxi $(PYBINDING_DEPS)
 	$(PYTHON) -c "from Pyrex.Compiler.Main import main; main(command_line=1)" \
-		-o $@ `test -f tools/python-yasm/yasm.pyx || echo '$(srcdir)/'`tools/python-yasm/yasm.pyx
+		-o $@ yasm.pyx
 
 CLEANFILES += yasm_python.c