Newer
Older
/*
* HLSL preprocessor
*
* Copyright 2020 Zebediah Figura for CodeWeavers
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
*/
%{
#include "preproc.tab.h"
#define YYSTYPE PREPROC_YYSTYPE
#define YYLTYPE PREPROC_YYLTYPE
#define YY_DECL static int preproc_lexer_lex(YYSTYPE *yylval_param, YYLTYPE *yylloc_param, yyscan_t yyscanner)
static void update_location(struct preproc_ctx *ctx);
#define YY_USER_ACTION update_location(yyget_extra(yyscanner));
%}
%option 8bit
%option bison-bridge
%option bison-locations
%option extra-type="struct preproc_ctx *"
%option never-interactive
%option noinput
%option nounput
%option noyy_top_state
%option noyywrap
%option prefix="preproc_yy"
%option reentrant
%option stack
/* Because these can both be terminated by EOF, we need states for them. */
%s C_COMMENT
%s CXX_COMMENT
NEWLINE \r?\n
WS [ \t]
IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
%%
<INITIAL>"//" {yy_push_state(CXX_COMMENT, yyscanner);}
<INITIAL>"/*" {yy_push_state(C_COMMENT, yyscanner);}
<CXX_COMMENT>\\{NEWLINE} {}
<CXX_COMMENT>\n {
yy_pop_state(yyscanner);
return T_NEWLINE;
}
<C_COMMENT>"*/" {yy_pop_state(yyscanner);}
<C_COMMENT,CXX_COMMENT><<EOF>> {yy_pop_state(yyscanner);}
<C_COMMENT,CXX_COMMENT>. {}
<INITIAL>{IDENTIFIER}/\( {return T_IDENTIFIER_PAREN;}
<INITIAL>{IDENTIFIER} {return T_IDENTIFIER;}
/* We have no use for floats, but shouldn't parse them as integers. */
<INITIAL>[0-9]*\.[0-9]+([eE][+-]?[0-9]+)?[hHfF]? {return T_TEXT;}
<INITIAL>[0-9]+\.([eE][+-]?[0-9]+)?[hHfF]? {return T_TEXT;}
<INITIAL>[0-9]+([eE][+-]?[0-9]+)?[hHfF] {return T_TEXT;}
<INITIAL>[0-9]+[eE][+-]?[0-9]+ {return T_TEXT;}
<INITIAL>0[xX][0-9a-fA-f]+[ul]{0,2} {return T_INTEGER;}
<INITIAL>0[0-7]*[ul]{0,2} {return T_INTEGER;}
<INITIAL>[1-9][0-9]*[ul]{0,2} {return T_INTEGER;}
<INITIAL>## {return T_CONCAT;}
<INITIAL>"&&" {return T_TEXT;}
<INITIAL>"||" {return T_TEXT;}
<INITIAL>"++" {return T_TEXT;}
<INITIAL>"--" {return T_TEXT;}
<INITIAL>"<<"=? {return T_TEXT;}
<INITIAL>">>"=? {return T_TEXT;}
<INITIAL>[-+*/%&|^=><!]= {return T_TEXT;}
<INCLUDE>\"[^"]*\" {return T_STRING;}
<INCLUDE>\<[^>]*\> {return T_STRING;}
/* C strings (including escaped quotes). */
<INITIAL>\"([^"\\]|\\.)*\" {return T_TEXT;}
<INITIAL>#{WS}*{IDENTIFIER} {
struct preproc_ctx *ctx = yyget_extra(yyscanner);
const char *p;
if (!ctx->last_was_newline)
return T_TEXT;
for (p = yytext + 1; strchr(" \t", *p); ++p)
;
if (!strcmp(p, "include"))
{
BEGIN(INCLUDE);
return T_INCLUDE;
}
if (!strcmp(p, "define"))
return T_DEFINE;
if (!strcmp(p, "elif"))
return T_ELIF;
if (!strcmp(p, "else"))
return T_ELSE;
if (!strcmp(p, "endif"))
return T_ENDIF;
if (!strcmp(p, "if"))
return T_IF;
if (!strcmp(p, "ifdef"))
return T_IFDEF;
if (!strcmp(p, "ifndef"))
return T_IFNDEF;
if (!strcmp(p, "undef"))
return T_UNDEF;
preproc_warning(ctx, yyget_lloc(yyscanner), VKD3D_SHADER_WARNING_PP_UNKNOWN_DIRECTIVE,
"Ignoring unknown directive \"%s\".", yytext);
return T_TEXT;
}
<INITIAL,INCLUDE>\\{NEWLINE} {}
<INITIAL,INCLUDE>{NEWLINE} {
BEGIN(INITIAL);
return T_NEWLINE;
}
<INITIAL>[()\[\]{},] {return yytext[0];}
<INITIAL>. {return T_TEXT;}
%%
static void update_location(struct preproc_ctx *ctx)
{
struct preproc_buffer *buffer = &preproc_get_top_file(ctx)->buffer;
unsigned int i, leng = yyget_leng(ctx->scanner);
const char *text = yyget_text(ctx->scanner);
/* We want to do this here, rather than before calling yylex(), because
* some tokens are skipped by the lexer. */
*yyget_lloc(ctx->scanner) = buffer->location;
for (i = 0; i < leng; ++i)
{
if (text[i] == '\n')
{
buffer->location.column = 1;
++buffer->location.line;
static bool preproc_is_writing(struct preproc_ctx *ctx)
{
const struct preproc_file *file;
/* This can happen while checking for unterminated macro invocation. */
if (!ctx->file_count)
return true;
file = preproc_get_top_file(ctx);
if (!file->if_count)
return true;
return file->if_stack[file->if_count - 1].current_true;
}
static struct preproc_macro *preproc_get_top_macro(struct preproc_ctx *ctx)
{
if (!ctx->expansion_count)
return NULL;
return ctx->expansion_stack[ctx->expansion_count - 1].macro;
}
/* Concatenation is not done for object-like macros, but is done for both
* function-like macro bodies and their arguments. */
static bool should_concat(struct preproc_ctx *ctx)
{
struct preproc_macro *macro;
if (!ctx->expansion_count)
return false;
macro = ctx->expansion_stack[ctx->expansion_count - 1].macro;
return !macro || macro->arg_count;
}
static void preproc_pop_buffer(struct preproc_ctx *ctx)
{
if (ctx->expansion_count)
{
struct preproc_expansion *exp = &ctx->expansion_stack[ctx->expansion_count - 1];
yy_delete_buffer(exp->buffer.lexer_buffer, ctx->scanner);
--ctx->expansion_count;
TRACE("Expansion stack size is now %zu.\n", ctx->expansion_count);
}
else
struct preproc_file *file = preproc_get_top_file(ctx);
if (ctx->file_count > 1)
preproc_close_include(ctx, &file->code);
if (file->if_count)
{
const struct vkd3d_shader_location loc = {.source_name = file->filename};
preproc_warning(ctx, &loc, VKD3D_SHADER_WARNING_PP_UNTERMINATED_IF, "Unterminated #if block.");
}
vkd3d_free(file->if_stack);
vkd3d_free(file->filename);
yy_delete_buffer(file->buffer.lexer_buffer, ctx->scanner);
--ctx->file_count;
TRACE("File stack size is now %zu.\n", ctx->file_count);
}
if (ctx->expansion_count)
yy_switch_to_buffer(ctx->expansion_stack[ctx->expansion_count - 1].buffer.lexer_buffer, ctx->scanner);
else if (ctx->file_count)
yy_switch_to_buffer(ctx->file_stack[ctx->file_count - 1].buffer.lexer_buffer, ctx->scanner);
}
static int return_token(int token, YYSTYPE *lval, const char *text)
{
switch (token)
{
case T_IDENTIFIER_PAREN:
case T_INTEGER:
case T_TEXT:
if (!(lval->string = vkd3d_strdup(text)))
return 0;
break;
}
return token;
}
static const struct preproc_text *find_arg_expansion(struct preproc_ctx *ctx, const char *s)
{
struct preproc_macro *macro;
unsigned int i;
if ((macro = preproc_get_top_macro(ctx)))
{
for (i = 0; i < macro->arg_count; ++i)
{
if (!strcmp(s, macro->arg_names[i]))
return ¯o->arg_values[i];
}
}
return NULL;
}
static void preproc_text_add(struct preproc_text *text, const char *string)
{
vkd3d_string_buffer_printf(&text->text, "%s", string);
}
static bool preproc_push_expansion(struct preproc_ctx *ctx,
const struct preproc_text *text, struct preproc_macro *macro)
{
struct preproc_expansion *exp;
if (!vkd3d_array_reserve((void **)&ctx->expansion_stack, &ctx->expansion_stack_size,
ctx->expansion_count + 1, sizeof(*ctx->expansion_stack)))
return false;
exp = &ctx->expansion_stack[ctx->expansion_count++];
exp->text = text;
exp->buffer.lexer_buffer = yy_scan_bytes(text->text.buffer, text->text.content_size, ctx->scanner);
exp->buffer.location = text->location;
exp->macro = macro;
TRACE("Expansion stack size is now %zu.\n", ctx->expansion_count);
return true;
}
int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner)
{
struct preproc_ctx *ctx = yyget_extra(scanner);
for (;;)
{
struct preproc_func_state *func_state;
const char *text;
int token;
if (ctx->lookahead_token)
token = ctx->lookahead_token;
text = yyget_text(scanner);
}
else
{
if (ctx->last_was_eof)
{
preproc_pop_buffer(ctx);
if (!ctx->file_count)
return 0;
}
ctx->last_was_eof = false;
assert(ctx->file_count);
if (!(token = preproc_lexer_lex(lval, lloc, scanner)))
{
ctx->last_was_eof = true;
/* If we have reached the end of an included file, inject a newline. */
if (ctx->expansion_count)
continue;
token = T_NEWLINE;
text = "\n";
}
else
{
text = yyget_text(scanner);
}
if (ctx->last_was_newline)
switch (token)
{
case T_DEFINE:
case T_ELIF:
case T_ELSE:
case T_ENDIF:
case T_IF:
case T_IFDEF:
case T_IFNDEF:
case T_INCLUDE:
case T_UNDEF:
ctx->current_directive = token;
break;
default:
ctx->current_directive = 0;
}
ctx->last_was_newline = (token == T_NEWLINE);
func_state = ctx->current_directive ? &ctx->directive_func : &ctx->text_func;
TRACE("Parsing token %d%s, line %d, in directive %d, state %#x, string %s.\n",
token, ctx->lookahead_token ? " (lookahead)" : "", lloc->line,
ctx->current_directive, func_state->state, debugstr_a(text));
ctx->lookahead_token = 0;
switch (ctx->current_directive)
{
case T_ELIF:
case T_ELSE:
case T_ENDIF:
case T_IF:
break;
default:
if (!preproc_is_writing(ctx))
continue;
}
switch (func_state->state)
case STATE_NONE:
if (token == T_CONCAT && should_concat(ctx))
{
while (ctx->buffer.content_size
&& strchr(" \t\r\n", ctx->buffer.buffer[ctx->buffer.content_size - 1]))
--ctx->buffer.content_size;
break;
}
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
if (token == T_IDENTIFIER || token == T_IDENTIFIER_PAREN)
{
const struct preproc_text *expansion;
struct preproc_macro *macro;
switch (ctx->current_directive)
{
case T_DEFINE:
case T_IFDEF:
case T_IFNDEF:
case T_UNDEF:
/* Return identifiers verbatim. */
return return_token(token, lval, text);
}
/* Otherwise, expand a macro if there is one. */
if ((expansion = find_arg_expansion(ctx, text)))
{
preproc_push_expansion(ctx, expansion, NULL);
continue;
}
if ((macro = preproc_find_macro(ctx, text)))
{
if (!macro->arg_count)
{
preproc_push_expansion(ctx, ¯o->body, macro);
}
else
{
func_state->state = STATE_IDENTIFIER;
func_state->macro = macro;
}
continue;
}
}
if (ctx->current_directive)
return return_token(token, lval, text);
vkd3d_string_buffer_printf(&ctx->buffer, "%s ", text);
break;
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
case STATE_IDENTIFIER:
if (token == '(')
{
struct preproc_text *first_arg = &func_state->macro->arg_values[0];
unsigned int i;
func_state->arg_count = 0;
func_state->paren_depth = 1;
func_state->state = STATE_ARGS;
for (i = 0; i < func_state->macro->arg_count; ++i)
func_state->macro->arg_values[i].text.content_size = 0;
first_arg->location = *lloc;
}
else
{
const char *name = func_state->macro->name;
ctx->lookahead_token = token;
func_state->macro = NULL;
func_state->state = STATE_NONE;
if (ctx->current_directive)
return return_token(T_IDENTIFIER, lval, name);
vkd3d_string_buffer_printf(&ctx->buffer, "%s ", name);
}
break;
case STATE_ARGS:
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
struct preproc_text *current_arg = NULL;
assert(func_state->macro->arg_count);
if (func_state->arg_count < func_state->macro->arg_count)
current_arg = &func_state->macro->arg_values[func_state->arg_count];
switch (token)
{
case T_NEWLINE:
if (current_arg)
preproc_text_add(current_arg, " ");
break;
case ')':
case ']':
case '}':
if (!--func_state->paren_depth)
{
if (++func_state->arg_count == func_state->macro->arg_count)
{
preproc_push_expansion(ctx, &func_state->macro->body, func_state->macro);
}
else
{
preproc_warning(ctx, lloc, VKD3D_SHADER_WARNING_PP_ARGUMENT_COUNT_MISMATCH,
"Wrong number of arguments to macro \"%s\": expected %zu, got %zu.",
func_state->macro->name, func_state->macro->arg_count, func_state->arg_count);
if (ctx->current_directive)
return return_token(T_IDENTIFIER, lval, func_state->macro->name);
vkd3d_string_buffer_printf(&ctx->buffer, "%s ", func_state->macro->name);
}
func_state->macro = NULL;
func_state->state = STATE_NONE;
}
else
{
if (current_arg)
preproc_text_add(current_arg, text);
}
break;
case ',':
if (func_state->paren_depth == 1)
{
++func_state->arg_count;
if (current_arg)
current_arg->location = *lloc;
}
else if (current_arg)
{
preproc_text_add(current_arg, text);
}
break;
case '(':
case '[':
case '{':
++func_state->paren_depth;
/* fall through */
default:
if (current_arg)
preproc_text_add(current_arg, text);
}
break;
}
}
bool preproc_push_include(struct preproc_ctx *ctx, char *filename, const struct vkd3d_shader_code *code)
{
struct preproc_file *file;
if (!vkd3d_array_reserve((void **)&ctx->file_stack, &ctx->file_stack_size,
ctx->file_count + 1, sizeof(*ctx->file_stack)))
return false;
file = &ctx->file_stack[ctx->file_count++];
memset(file, 0, sizeof(*file));
file->code = *code;
file->filename = filename;
file->buffer.lexer_buffer = yy_scan_bytes(code->code, code->size, ctx->scanner);
file->buffer.location.source_name = file->filename;
file->buffer.location.line = 1;
file->buffer.location.column = 1;
TRACE("File stack size is now %zu.\n", ctx->file_count);
ctx->last_was_newline = true;
return true;
}
static int preproc_macro_compare(const void *key, const struct rb_entry *entry)
{
const struct preproc_macro *macro = RB_ENTRY_VALUE(entry, struct preproc_macro, entry);
const char *name = key;
return strcmp(name, macro->name);
}
static void preproc_macro_rb_free(struct rb_entry *entry, void *ctx)
{
preproc_free_macro(RB_ENTRY_VALUE(entry, struct preproc_macro, entry));
}
int preproc_lexer_parse(const struct vkd3d_shader_compile_info *compile_info,
struct vkd3d_shader_code *out, struct vkd3d_shader_message_context *message_context)
{
static const struct vkd3d_shader_preprocess_info default_preprocess_info = {0};
struct preproc_ctx ctx = {0};
void *output_code;
vkd3d_string_buffer_init(&ctx.buffer);
rb_init(&ctx.macros, preproc_macro_compare);
if (!(ctx.preprocess_info = vkd3d_find_struct(compile_info->next, PREPROCESS_INFO)))
ctx.preprocess_info = &default_preprocess_info;
ctx.message_context = message_context;
if (!(source_name = vkd3d_strdup(compile_info->source_name ? compile_info->source_name : "<anonymous>")))
{
vkd3d_string_buffer_cleanup(&ctx.buffer);
return VKD3D_ERROR_OUT_OF_MEMORY;
}
yylex_init_extra(&ctx, &ctx.scanner);
if (!preproc_push_include(&ctx, source_name, &compile_info->source))
{
yylex_destroy(ctx.scanner);
vkd3d_free(source_name);
vkd3d_string_buffer_cleanup(&ctx.buffer);
return VKD3D_ERROR_OUT_OF_MEMORY;
}
preproc_yyparse(ctx.scanner, &ctx);
switch (ctx.text_func.state)
{
case STATE_NONE:
break;
case STATE_ARGS:
{
const struct vkd3d_shader_location loc = {.source_name = source_name};
preproc_warning(&ctx, &loc, VKD3D_SHADER_WARNING_PP_UNTERMINATED_MACRO,
"Unterminated macro invocation.");
}
/* fall through */
case STATE_IDENTIFIER:
if (preproc_is_writing(&ctx))
vkd3d_string_buffer_printf(&ctx.buffer, "%s ", ctx.text_func.macro->name);
break;
}
while (ctx.file_count)
preproc_pop_buffer(&ctx);
yylex_destroy(ctx.scanner);
rb_destroy(&ctx.macros, preproc_macro_rb_free, NULL);
vkd3d_free(ctx.expansion_stack);
if (ctx.error)
{
WARN("Failed to preprocess.\n");
vkd3d_string_buffer_cleanup(&ctx.buffer);
return VKD3D_ERROR_INVALID_SHADER;
}
if (!(output_code = vkd3d_malloc(ctx.buffer.content_size)))
{
vkd3d_string_buffer_cleanup(&ctx.buffer);
return VKD3D_ERROR_OUT_OF_MEMORY;
}
memcpy(output_code, ctx.buffer.buffer, ctx.buffer.content_size);
out->size = ctx.buffer.content_size;
out->code = output_code;
vkd3d_string_buffer_trace(&ctx.buffer);
vkd3d_string_buffer_cleanup(&ctx.buffer);
return VKD3D_OK;
}