Skip to content
Browse files
Initial work on preprocessor. Not yet complete!
  • Loading branch information
icculus committed Feb 9, 2009
1 parent 6fee7f9 commit 653472826333e223db585946ee83ce69e765195e
Showing with 2,601 additions and 0 deletions.
  1. +4 −0 CMakeLists.txt
  2. +192 −0 mojoshader.h
  3. +75 −0 mojoshader_internal.h
  4. +1,309 −0 mojoshader_lexer.c
  5. +174 −0
  6. +755 −0 mojoshader_preprocessor.c
  7. +92 −0 preprocess.c
@@ -42,6 +42,8 @@ ENDIF(MSVC)

@@ -66,6 +68,8 @@ ADD_EXECUTABLE(finderrors finderrors.c)
TARGET_LINK_LIBRARIES(finderrors mojoshader ${SDL_LIBRARY})
ADD_EXECUTABLE(assemble assemble.c)
TARGET_LINK_LIBRARIES(assemble mojoshader)
ADD_EXECUTABLE(preprocess preprocess.c)
TARGET_LINK_LIBRARIES(preprocess mojoshader)

# End of CMakeLists.txt ...

@@ -610,6 +610,198 @@ void MOJOSHADER_freeParseData(const MOJOSHADER_parseData *data);

/* Preprocessor interface... */

* Structure used to pass predefined macros. Maps to D3DXMACRO.
typedef struct MOJOSHADER_preprocessorDefine
const char *identifier;
const char *definition;
} MOJOSHADER_preprocessorDefine;

* Used with the MOJOSHADER_includeOpen callback. Maps to D3DXINCLUDE_TYPE.
typedef enum
MOJOSHADER_INCLUDETYPE_LOCAL, /* local header: #include "blah.h" */
MOJOSHADER_INCLUDETYPE_SYSTEM /* system header: #include <blah.h> */
} MOJOSHADER_includeType;

* Structure used to return data from preprocessing of a shader...
/* !!! FIXME: most of these ints should be unsigned. */
typedef struct MOJOSHADER_preprocessData
* The number of elements pointed to by (errors).
int error_count;

* (error_count) elements of data that specify errors that were generated
* by parsing this shader.
* This can be NULL if there were no errors or if (error_count) is zero.
MOJOSHADER_error *errors;

* Bytes of output from preprocessing. This is an ASCII string. We
* guarantee it to be NULL-terminated. Will be NULL on error.
const char *output;

* Byte count for output, not counting any null terminator.
* Will be 0 on error.
int output_len;

* This is the malloc implementation you passed to MOJOSHADER_parse().
MOJOSHADER_malloc malloc;

* This is the free implementation you passed to MOJOSHADER_parse().
MOJOSHADER_free free;

* This is the pointer you passed as opaque data for your allocator.
void *malloc_data;
} MOJOSHADER_preprocessData;

* This callback allows an app to handle #include statements for the
* preprocessor. When the preprocessor sees an #include, it will call this
* function to obtain the contents of the requested file. This is optional;
* the preprocessor will open files directly if no calback is supplied, but
* this allows an app to retrieve data from something other than the
* traditional filesystem (for example, headers packed in a .zip file or
* headers generated on-the-fly).
* This function maps to ID3DXInclude::Open()
* (inctype) specifies the type of header we wish to include.
* (fname) specifies the name of the file specified on the #include line.
* (parent) is a string of the entire source file containing the include, in
* its original, not-yet-preprocessed state. Note that this is just the
* contents of the specific file, not all source code that the preprocessor
* has seen through other includes, etc.
* (outdata) will be set by the callback to a pointer to the included file's
* contents. The callback is responsible for allocating this however they
* see fit (we provide allocator functions, but you may ignore them). This
* pointer must remain valid until the includeClose callback runs. This
* string does not need to be NULL-terminated.
* (outbytes) will be set by the callback to the number of bytes pointed to
* by (outdata).
* (m),(f), and (d) are the allocator details that the application passed to
* MojoShader. If these were NULL, MojoShader may have replaced them with its
* own internal allocators.
* The callback returns zero on error, non-zero on success.
* If you supply an includeOpen callback, you must supply includeClose, too.
typedef int (*MOJOSHADER_includeOpen)(MOJOSHADER_includeType inctype,
const char *fname, const char *parent,
const char **outdata, unsigned int *outbytes,
MOJOSHADER_malloc m, MOJOSHADER_free f, void *d);

* This callback allows an app to clean up the results of a previous
* includeOpen callback.
* This function maps to ID3DXInclude::Close()
* (data) is the data that was returned from a previous call to includeOpen.
* It is now safe to deallocate this data.
* (m),(f), and (d) are the same allocator details that were passed to your
* includeOpen callback.
* If you supply an includeClose callback, you must supply includeOpen, too.
typedef void (*MOJOSHADER_includeClose)(const char *data,
MOJOSHADER_malloc m, MOJOSHADER_free f, void *d);

* This function is optional. Even if you are dealing with shader source
* code, you don't need to explicitly use the preprocessor, as the compiler
* and assembler will use it behind the scenes. In fact, you probably never
* need this function unless you are debugging a custom tool (or debugging
* MojoShader itself).
* Preprocessing roughly follows the syntax of an ANSI C preprocessor, as
* Microsoft's Direct3D assembler and HLSL compiler use this syntax. Please
* note that we try to match the output you'd get from Direct3D's
* preprocessor, which has some quirks if you're expecting output that matches
* a generic C preprocessor.
* This function maps to D3DXPreprocessShader().
* (source) is an ASCII string of text to preprocess. It does not need to be
* NULL-terminated.
* (sourcelen) is the length of the string pointed to by (source), in bytes.
* (defines) points to (define_count) preprocessor definitions, and can be
* NULL. These are treated by the preprocessor as if the source code started
* with one #define for each entry you pass in here.
* (include_open) and (include_close) let the app control the preprocessor's
* behaviour for #include statements. Both are optional and can be NULL, but
* both must be specified if either is specified.
* This will return a MOJOSHADER_preprocessorData. You should pass this
* return value to MOJOSHADER_freePreprocessData() when you are done with
* it.
* This function will never return NULL, even if the system is completely
* out of memory upon entry (in which case, this function returns a static
* MOJOSHADER_preprocessData object, which is still safe to pass to
* MOJOSHADER_freePreprocessData()).
* As preprocessing requires some memory to be allocated, you may provide a
* custom allocator to this function, which will be used to allocate/free
* memory. They function just like malloc() and free(). We do not use
* realloc(). If you don't care, pass NULL in for the allocator functions.
* If your allocator needs instance-specific data, you may supply it with the
* (d) parameter. This pointer is passed as-is to your (m) and (f) functions.
* This function is thread safe, so long as the various callback functions
* are, too, and that the parameters remains intact for the duration of the
* call. This allows you to preprocess several shaders on separate CPU cores
* at the same time.
const MOJOSHADER_preprocessData *MOJOSHADER_preprocess(const char *source,
unsigned int sourcelen,
const MOJOSHADER_preprocessorDefine **defines,
unsigned int define_count,
MOJOSHADER_includeOpen include_open,
MOJOSHADER_includeClose include_close,
MOJOSHADER_malloc m, MOJOSHADER_free f, void *d);

* Call this to dispose of preprocessing results when you are done with them.
* This will call the MOJOSHADER_free function you provided to
* MOJOSHADER_preprocess() multiple times, if you provided one.
* Passing a NULL here is a safe no-op.
* This function is thread safe, so long as any allocator you passed into
* MOJOSHADER_preprocess() is, too.
void MOJOSHADER_freePreprocessData(const MOJOSHADER_preprocessData *data);

/* Assembler interface... */

@@ -273,6 +273,81 @@ typedef struct ErrorList
} ErrorList;

// preprocessor stuff.

typedef enum
TOKEN_UNKNOWN = 256, // start past ASCII character values.
} Token;

// This is opaque.
struct Preprocessor;
typedef struct Preprocessor Preprocessor;

typedef struct IncludeState
char *filename;
int included;
Token insert_token;
Token insert_token2;
char insert_tokchar;
const char *source_base;
const char *source;
const char *token;
const unsigned char *lexer_marker;
unsigned int bytes_left;
unsigned int line;
struct IncludeState *next;
} IncludeState;

Token preprocessor_internal_lexer(IncludeState *s);

// This will only fail if the allocator fails, so it doesn't return any
// error code...NULL on failure.
Preprocessor *preprocessor_start(const char *fname, const char *source,
unsigned int sourcelen,
MOJOSHADER_includeOpen open_callback,
MOJOSHADER_includeClose close_callback,
const MOJOSHADER_preprocessorDefine **defines,
unsigned int define_count,
MOJOSHADER_malloc m, MOJOSHADER_free f, void *d);

void preprocessor_end(Preprocessor *pp);
const char *preprocessor_error(Preprocessor *pp);
void preprocessor_clearerror(Preprocessor *pp);
int preprocessor_outofmemory(Preprocessor *pp);
const char *preprocessor_nexttoken(Preprocessor *_ctx,
unsigned int *_len, Token *_token);
const char *preprocessor_sourcepos(Preprocessor *pp, unsigned int *pos);


0 comments on commit 6534728

Please sign in to comment.