osolex.l

/* Copyright Contributors to the Open Shading Language project.
 * SPDX-License-Identifier: BSD-3-Clause
 * https://github.com/AcademySoftwareFoundation/OpenShadingLanguage
 */


/** Lexical scanner for OpenShadingLanguage 'object' files
 **/

%top{
/* Include this first to avoid warnings about re-definitions on windows */
#include <stdint.h>
}

/************************************************************
 * Definitions section
 ************************************************************/

%option reentrant bison-bridge
%option extra-type="OSL::pvt::OSOReader *"

/* Useful for debugging prototypes"
%option header-file="osolex.yy.h"
*/

/* Option 'noyywrap' indicates that when EOF is hit, yyin does not 
 * automatically reset to another file.
 */
%option noyywrap
%option nounput noinput

/* never-interactive fixes a Windows compatibility problem where the
 * lexer emits isatty calls that don't exist.
 */
%option never-interactive

 /* Option 'prefix' creates a C++ lexer with the given prefix, so that
  * we can link with other flex-generated lexers in the same application
  * without name conflicts.
  */
%option prefix="oso"

 /* %option perf-report */


 /* Define regular expression macros 
  ************************************************/

 /* white space, not counting newline */
WHITE           [ \t\v\f\r]+
 /* alpha character */
ALPHA           [A-Za-z]
 /* numerals */
DIGIT           [0-9]
 /* Integer literal */
INTEGER         [-+]?{DIGIT}+
 /* floating point literal (E, FLT1, FLT2, FLT3 are just helpers) */
E               [eE][-+]?{DIGIT}+
FLT1            [-+]?{DIGIT}+\.{DIGIT}*{E}?
FLT2            [-+]?{DIGIT}*\.{DIGIT}+{E}?
FLT3            [-+]?{DIGIT}+{E}
FLT             {FLT1}|{FLT2}|{FLT3}
 /* string literal */
STR     \"(\\.|[^\\"\n])*\"
 /* Identifier: alphanumeric, may contain digits after the first character.
  * Also '$' and '.' are allowed!
  */
IDENT           ({ALPHA}|[_$])({ALPHA}|{DIGIT}|[_$\.])*
 /* C preprocessor (cpp) directives */
COMMENT         \#[^\n]*\n
 /* Hints */
IDORLITERAL     {FLT}|{STR}|{INTEGER}|({IDENT}(\[{INTEGER}?\])?)
HINTPATTERN     \%{IDENT}(\{({IDORLITERAL}(\,{IDORLITERAL})*)?\})?


 /* Note for lex newbies: the following '%{ .. %}' section contains literal
  * C code that will be inserted at the top of code that flex generates.
  */
%{
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>
#include <string>

#include <OpenImageIO/ustring.h>
#include <OpenImageIO/strutil.h>
#include <OpenImageIO/filesystem.h>

#include "osoreader.h"
using namespace OSL;
using namespace OSL::pvt;

#include "osogram.hpp"   /* Generated by bison/yacc */

OSL_NAMESPACE_ENTER
namespace pvt {
    int osolex(YYSTYPE* yylval_param, YYLTYPE* yylloc_param, void* yyscanner, OSOReader* osoreader);
}
OSL_NAMESPACE_EXIT
#define YY_DECL int OSL::pvt::osolex(YYSTYPE* yylval_param, YYLTYPE* yylloc_param, void* yyscanner, OSOReader* osoreader)

#ifdef _WIN32
#define YY_NO_UNISTD_H
#endif

#ifdef _MSC_VER
#include <io.h>
#define isatty _isatty
#endif

// flex itself will generate fatal warnings about signed vs unsigned.
// Bypass that nonsense.
#if OSL_GNUC_VERSION >= 60000
#pragma GCC diagnostic ignored "-Wsign-compare"
#endif

// flex uses the 'register' keyword, warned because it's deprecated in C++17.
#if defined(__clang__)
#pragma GCC diagnostic ignored "-Wdeprecated-register"
#pragma GCC diagnostic ignored "-Wregister"
#endif
#if OSL_GNUC_VERSION >= 90000
#pragma GCC diagnostic ignored "-Wregister"
#endif

#if defined(__GNUC__) || defined(__clang__)
#pragma GCC visibility push(hidden)
#endif

%}

/* Declare modes */
%s DECLARATION


%%

 /************************************************
  * Lexical matching rules
  ************************************************/

 /* Comments */
{COMMENT}               {  /* skip it */ }

 /* keywords */
<DECLARATION>"closure"	{  return (yylval->i=CLOSURE); }
<DECLARATION>"color"	{  return (yylval->i=COLORTYPE); }
<DECLARATION>"float"	{  return (yylval->i=FLOATTYPE); }
<DECLARATION>"int"      {  return (yylval->i=INTTYPE); }
<DECLARATION>"matrix"	{  return (yylval->i=MATRIXTYPE); }
<DECLARATION>"normal"	{  return (yylval->i=NORMALTYPE); }
<DECLARATION>"point"	{  return (yylval->i=POINTTYPE); }
<DECLARATION>"string"	{  return (yylval->i=STRINGTYPE); }
<DECLARATION>"struct"	{  return (yylval->i=STRUCT); }
<DECLARATION>"vector"	{  return (yylval->i=VECTORTYPE); }

^local                  {
                           BEGIN (DECLARATION);
                           yylval->i = SymTypeLocal;
                           return SYMTYPE;
                        }

^temp                   {
                           BEGIN (DECLARATION);
                           yylval->i = SymTypeTemp;
                           return SYMTYPE;
                        }

^global                 {
                           BEGIN (DECLARATION);
                           yylval->i = SymTypeGlobal;
                           return SYMTYPE;
                        }

^param                  {
                           BEGIN (DECLARATION);
                           yylval->i = SymTypeParam;
                           return SYMTYPE;
                        }

^oparam                 {
                            BEGIN (DECLARATION);
                            yylval->i = SymTypeOutputParam;
                            return SYMTYPE;
                        }

^const                  {
                            BEGIN (DECLARATION);
                            yylval->i = SymTypeConst;
                            return SYMTYPE;
                        }

^code                   {
                            BEGIN (INITIAL);
                            return yylval->i = CODE;
                        }

 /* Identifiers */
{IDENT}	                {
                            yylval->s = ustring(yytext).c_str();
                            // std::cerr << "lex ident '" << yylval->s << "'\n";
                            return IDENTIFIER;
                        }

 /* Literal values */
{INTEGER}               {
                            yylval->i = OIIO::Strutil::from_string<int>(yytext);
                            // std::cerr << "lex int " << yylval->i << "\n";
                            return INT_LITERAL;
                        }

{FLT}                   {
                            yylval->f = OIIO::Strutil::from_string<float>(yytext);
                            // std::cerr << "lex float " << yylval->f << "\n";
                            return FLOAT_LITERAL;
                        }

{STR}                   {
                            // grab the material between the quotes
                            string_view s(yytext + 1, yyleng - 2);
                            std::string unescaped;
                            if (s.find('\\') != string_view::npos) {
                                // Only make a new string if we must unescape
                                unescaped = OIIO::Strutil::unescape_chars(s);
                                s = string_view(unescaped);
                            }
                            yylval->s = ustring(s).c_str();
                            // std::cerr << "osolex string '" << yylval->s << "'\n";
                            return STRING_LITERAL;
                        }

\%preprocessed_source\n(.*\n)* {
                            /* skip remainder of file */
                        }

{HINTPATTERN}           {
                            ustring s (yytext);
                            yylval->s = s.c_str();
                            return HINT;
                        }

 /* Ignore whitespace */
{WHITE} 		{  }

 /* Ignore a linefeed that ends with a trailing blackslash */
"\\\n"		        {  }

 /* End of line */
[\n]			{
                            osoreader->incr_lineno ();
                            return ENDOFLINE;
                        }

 /* catch-all rule for any other single characters */
.			{  return (yylval->i = *yytext); }

%%



OSL_NAMESPACE_ENTER

namespace pvt {   // OSL::pvt


static std::mutex osoread_mutex;


class OSOReader::Scope
{
    yyscan_t m_scanner;
    YY_BUFFER_STATE m_buffer;

    Scope() {
        yylex_init(&m_scanner);
    }
public:
    Scope(FILE* f) : Scope() {
        m_buffer = yy_create_buffer(f, YY_BUF_SIZE, m_scanner);
    }
    Scope(const std::string& str) : Scope() {
        m_buffer = yy_scan_string(str.c_str(), m_scanner);
    }
    ~Scope() {
        yy_delete_buffer(m_buffer, m_scanner);
        yylex_destroy(m_scanner);
    }

    // TODO: This is really the only part of parsing that needs the lock still.
    //       The calling code no longer sets the global locale, so it would
    //       likely be simple to clean this up.
    //
    bool parse(OSOReader* reader, const char* what) {
        yy_switch_to_buffer(m_buffer, m_scanner);
        int errcode = osoparse(m_scanner, reader); // osoparse returns nonzero if error
        if (errcode) {
            reader->errhandler().errorfmt("Failed parse of {} (error code {})",
                                          what, errcode);
        }
        return ! errcode;
    }
};



bool
OSOReader::parse_file (const std::string &filename)
{
    // The lexer/parser isn't thread-safe, so make sure Only one thread
    // can actually be reading a .oso file at a time.
    std::lock_guard<std::mutex> guard (osoread_mutex);

    FILE* osoin = OIIO::Filesystem::fopen (filename, "r");
    if (! osoin) {
        m_err.errorfmt("File {} not found", filename);
        return false;
    }

    Scope scope(osoin);
    bool ok = scope.parse(this, filename.c_str());

    fclose (osoin);

    return ok;
}


bool
OSOReader::parse_memory (const std::string &buffer)
{
    // The lexer/parser isn't thread-safe, so make sure Only one thread
    // can actually be reading a .oso file at a time.
    std::lock_guard<std::mutex> guard (osoread_mutex);

    Scope scope(buffer);
    bool ok = scope.parse(this, "preloaded OSO code");

    return ok;
}



}; // namespace pvt
OSL_NAMESPACE_EXIT

#if defined(__GNUC__) || defined(__clang__)
#pragma GCC visibility pop
#endif