代码之家 › 专栏 › 技术社区 › Rishav Sharan

c中的Tokenizer没有显示任何输出

undefined-behavior c

0

Rishav Sharan · 技术社区 · 3 年前

我正在写一个简单的标记器来自学C,并且在似乎是未定义行为的情况下挣扎了很多。代码对我来说似乎很好,我不确定问题出在哪里。当我运行程序时,我在控制台中看不到任何输出。编译器根本不抛出错误。

这是我试图标记的简单文本文件:

1
2.34


xxy =  5e6

我下面的代码没有向控制台输出:

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include <ctype.h>
// #include "hexal.h"

#define HX_VERSION "0.0.1"

typedef enum
{
    TOK_EOF,                    // End of file      0
    TOK_ILLEGAL,                // Illegal token    1
    TOK_SPACE,                  // Space            2
    TOK_VAR,                    // var              3
    TOK_CONST,                  // const            4
    TOK_ASSIGN,                 // =                5
    TOK_INT_LIT,                // Integer literal  6
    TOK_FLOAT_LIT,              // Float literal    7
    TOK_EXPO_LIT,              // Float literal    7
    TOK_BOOL_LIT,               // Bool literal     8
    TOK_IDENTIFIER,             // Identifier       9
} Hx_Token_Type;

typedef struct Token
{
    Hx_Token_Type type;

    char *value;
    fpos_t cursor;
    int line;
} Token;

typedef enum {
    // Compiler Errors
    UNEXPECTED_TOKEN,
    UNEXPECTED_EOF,
    UNCLOSED_STRING,
    UNCLOSED_COMMENT,
    UNCLOSED_PAREN,
    UNCLOSED_BRACE,
    UNCLOSED_BRACKET,
    UNCLOSED_ANGLE_BRACKET,

    // Compiler Warnings
    UNUSED_IDENTIFIER,
    DEPRECATED_FEATURE,

    // 

} Hx_Compiler_Error_Types;

typedef struct {
    Hx_Compiler_Error_Types type;
    const char* err_message;
    size_t at_pos;
    size_t at_line;
} Hx_Compiler_Error;

// a type to hold the list of Hx_Compiler_Error

// Compiler metadata
typedef struct {
    Hx_Compiler_Error* Hx_Compiler_Errors;
    
    size_t Hx_Compiling_Duration;
    size_t Hx_Bytes_Compiled;

} Hx_Compiler_Metadata;

typedef struct {
    FILE* fp;

    int line;

    Token* prev_token;
    Token* curr_token;
    Token* next_token;

} Hx_File_Parsing_Context;

typedef struct {
    char* data;
    size_t length;
    size_t capacity;
} String;

String* init_string() {
    String* string = malloc(sizeof(String));
    if (string == NULL) {
        printf("Error: Failed to allocate memory\n");
        exit(EXIT_FAILURE);
    }

    string->length = 1;
    string->capacity = 8;
    string->data = malloc(string->capacity * sizeof(char));
    if (string->data == NULL) {
        printf("Error: Failed to allocate memory\n");
        exit(EXIT_FAILURE);
    }
    string->data[0] = '\0';

    return string;
}

// Function to append a char to a String
String* append_char_to_string(String* string, int ch) {
    if (string->length + 1 >= string->capacity) {
        // Double the capacity
        string->capacity *= 2;

        // Reallocate memory with double the capacity
        string->data = realloc(string->data, string->capacity * sizeof(char));
        if (string->data == NULL) {
            printf("Error: Failed to allocate memory\n");
            exit(EXIT_FAILURE);
        }
    }

    // Append the character to the string
    string->data[string->length] = (char) ch;
    string->data[string->length + 1] = '\0';
    string->length += 1;

    return string;
}

// Function to free the memory allocated for a String
void free_string(String* string) {
    free(string->data);
    free(string);
}

Token* get_next_token(Hx_File_Parsing_Context* ctx, Hx_Compiler_Metadata* meta) {
    // get the next token
    int ch = fgetc(ctx->fp);

    // setup return token
    Hx_Token_Type tok_type;
    String* tok_val = init_string();

    // Handle end of file
    if (ch == EOF) {
        tok_type = TOK_EOF;

        // Handle spaces & newlines. Roll all spaces into a single space token
    } else if (isspace(ch)) {
        do {
            if (ch == '\n') {
                ctx->line++;
            }
            ch = fgetc(ctx->fp);
        } while (isspace(ch));

        ungetc(ch, ctx->fp);
        tok_type = TOK_SPACE;

        // Handle Floats, exponentials and Integers
    } else if (isdigit(ch)) {
        tok_val = append_char_to_string(tok_val, ch);
        tok_type = TOK_INT_LIT;

        // roll all digits into a single float
        do {
            ch = fgetc(ctx->fp);
            if (ch == '_') {
                continue;
            } else if (ch == '.') {
                if (tok_type == TOK_INT_LIT) {
                    tok_val = append_char_to_string(tok_val, ch);
                    tok_type = TOK_FLOAT_LIT;
                } else {
                    ungetc(ch, ctx->fp);
                    break;
                }
            } else if (ch == 'e') {
                if (tok_type == TOK_INT_LIT || tok_type == TOK_FLOAT_LIT) {
                    tok_val = append_char_to_string(tok_val, ch);
                    tok_type = TOK_EXPO_LIT;
                } else {
                    ungetc(ch, ctx->fp);
                    break;
                }
            } else if (isdigit(ch)) {
                tok_val = append_char_to_string(tok_val, ch);
            } else {
                ungetc(ch, ctx->fp);
                break;
            }
        } while (isdigit(ch) || ch == '.' || ch == '_' || ch == 'e');

        // Handle Illegal Characters
    } else {
        tok_val = append_char_to_string(tok_val, ch);
        tok_type = TOK_ILLEGAL;
    }

    Token* res = malloc(sizeof(Token));
    res->type = tok_type;
    res->value = tok_val->data;
    res->line = ctx->line;
    free_string(tok_val);

    return res;
}

// parse_file
int parse_file(Hx_File_Parsing_Context* ctx, Hx_Compiler_Metadata* meta) {
    // advance the tokens to fill the prev, curr, and next tokens
    ctx->prev_token = ctx->curr_token;
    ctx->curr_token = ctx->next_token;
    ctx->next_token = get_next_token(ctx, meta);

    printf("\nAll Tokens in the file = \n");

    do {
        free(ctx->prev_token->value);
        free(ctx->prev_token);

        // advance the tokens in a loop till file ends
        ctx->prev_token = ctx->curr_token;
        ctx->curr_token = ctx->next_token;
        ctx->next_token = get_next_token(ctx, meta);
        // printf("{ type: %i, value: %s, line: %i },\n", ctx->curr_token->type, ctx->curr_token->value, ctx->curr_token->line);
        printf("{ type: %i, value: %s, line: %i },\n", (int)ctx->curr_token->type, ctx->curr_token->value, ctx->curr_token->line);

    } while (ctx->curr_token->type != TOK_EOF);

    return 0;
}

// Section for handling multiple files
int compile_source(char* entry_file) {
    // parse = tokenize + parse + typecheck
    // compile = parse + generate
    // present = compile + present + run

    Hx_File_Parsing_Context ctx;
    ctx.line = 1;

    Hx_Compiler_Metadata meta;
    meta.Hx_Bytes_Compiled = 0;
    meta.Hx_Compiling_Duration = 0;
    meta.Hx_Compiler_Errors = NULL;

    ctx.fp = fopen(entry_file, "r, ccs=UTF-8");
    if (ctx.fp == NULL) {
        printf("Error: Failed to open file\n");
        exit(EXIT_FAILURE);
    }
    printf("Parsing file: %s\n", entry_file);
    int res = parse_file(&ctx, &meta);

    fclose(ctx.fp);

    return res;
}

int main(int argc, char* argv[]) {

    int res = compile_source("input/test.hex");

    return res;
}

这是我在控制台中看到的:

src\hexal.c:251:14: warning: 'fopen' is deprecated: This
      function or variable may be unsafe. Consider using
      fopen_s instead. To disable deprecation, use
      _CRT_SECURE_NO_WARNINGS. See online help for
      details. [-Wdeprecated-declarations]
    ctx.fp = fopen(entry_file, "r, ccs=UTF-8");
             ^
C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\stdio.h:212:20: note: 
      'fopen' has been explicitly marked deprecated here
    _Check_return_ _CRT_INSECURE_DEPRECATE(fopen_s)
                   ^
C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.35.32215\include\vcruntime.h:355:55: note: 
      expanded from macro '_CRT_INSECURE_DEPRECATE'
        #define _CRT_INSECURE_DEPRECATE(_Replacement) _C...
                                                      ^
C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.35.32215\include\vcruntime.h:345:47: note: 
      expanded from macro '_CRT_DEPRECATE_TEXT'
#define _CRT_DEPRECATE_TEXT(_Text) __declspec(deprecated...
                                              ^
1 warning generated.
   Creating library bin\hexal.lib and object bin\hexal.exp
Parsing file: input/test.hex

C:\Users\risharan\Documents\GitHub\seawitch>cls && clang -g3 -Wall -Wextra -Wconversion -Wdouble-promotion -Wno-unused-parameter -Wno-unused-function -Wno-sign-conversion -fsanitize=undefined src\hexal.c -o bin\hexal.exe && bin\hexal.exe

0 回复 | 直到 3 年前

1

chqrlie 3 年前

在我的系统上运行您的代码时,我收到以下运行时错误:

230520-token(77235,0x20a1d8140) malloc: *** error for object 0x40000: pointer being freed was not allocated
230520-token(77235,0x20a1d8140) malloc: *** set a breakpoint in malloc_error_break to debug
/bin/sh: line 1: 77235 Abort trap: 6           ./230520-token
make: *** [230520-token.run] Error 134

阅读您的代码,我可以看到以下问题:

[专业] 也不 ctx 也没有 meta 在中初始化 main 。您应该初始化这些结构,以确保所有指针都为null:
```
Hx_File_Parsing_Context ctx = { 0 };
Hx_Compiler_Metadata meta = { 0 };
```

[专业] 在释放前一个令牌时,您应该测试它是否确实已分配:

  if (ctx->prev_token) {
      free(ctx->prev_token->value);
      free(ctx->prev_token);
  }

[专业] 在里面 init_string ,的初始值 string->length 应该是 0 。
append_char_to_string 不更改其第一个参数,则不需要更新 tok_val 像 tok_val = append_char_to_string(tok_val, ch); ,只需使用
```
 append_char_to_string(tok_val, ch);
```
读取数字的循环应该只是 for (;;) 环使用 do / while 循环添加了冗余测试。不支持十六进制数字。
您应该将行号保存在 get_next_token 所以 line 字段设置为令牌开始的行。当您解析多行标记(如注释和带有转义换行符的多行中断标记)时,这将更加重要。
您应该解析标识符、运算符、字符串、字符常量、注释。。。
[专业] 你用释放绳子 free_string(tok_val); 但是您保存了指向令牌结构的内容指针,因此该指针无效。您应该分配一个字符串的副本
```
    res->value = strdup(tok_val->data);
```

解决了上述主要问题,我得到了以下输出:

All Tokens in the file =
{ type: 6, value: 1, line: 1 },
{ type: 2, value: , line: 2 },
{ type: 7, value: 2.34, line: 2 },
{ type: 2, value: , line: 5 },
{ type: 1, value: x, line: 5 },
{ type: 1, value: x, line: 5 },
{ type: 1, value: y, line: 5 },
{ type: 2, value: , line: 5 },
{ type: 1, value: =, line: 5 },
{ type: 2, value: , line: 5 },
{ type: 8, value: 5e6, line: 5 },
{ type: 2, value: , line: 6 },
{ type: 0, value: , line: 6 },