Dream of my own C compiler

packages/cjit/src/tokens.zig

@@ -0,0 +1,149 @@
+pub const Keyword = enum {
+    _Alignas,
+    _Alignof,
+    _Atomic,
+    _Bool,
+    _Complex,
+    _Generic,
+    _Imaginary,
+    _Noreturn,
+    _Static_assert,
+    _Thread_local,
+    auto,
+    @"break",
+    case,
+    char,
+    @"const",
+    @"continue",
+    default,
+    do,
+    double,
+    @"else",
+    @"enum",
+    @"extern",
+    float,
+    @"for",
+    goto,
+    @"if",
+    @"inline",
+    int,
+    long,
+    register,
+    restrict,
+    @"return",
+    short,
+    signed,
+    sizeof,
+    static,
+    @"struct",
+    @"switch",
+    typedef,
+    @"union",
+    unsigned,
+    void,
+    @"volatile",
+    @"while",
+};
+
+pub const Identifier = struct {
+    name: []const u8,
+};
+
+pub const Constant = union(enum) {
+    int: i32,
+    long: i64,
+    long_long: i64,
+    unsigned_int: u32,
+    unsigned_long: u64,
+    unsigned_long_long: u64,
+    float: f32,
+    double: f64,
+    character: u8,
+    wide_character: u32,
+};
+
+pub const StringLiteral = struct {
+    value: []const u8,
+};
+
+pub const Punctuator = enum {
+    // three characters
+    @"...",
+    @"<<=",
+    @">>=",
+    // two characters
+    @"--",
+    @"-=",
+    @"->",
+    @"!=",
+    @"*=",
+    @"/=",
+    @"&&",
+    @"&=",
+    @"##",
+    @"%=",
+    @"^=",
+    @"++",
+    @"+=",
+    @"<<",
+    @"<=",
+    @"==",
+    @">=",
+    @">>",
+    @"|=",
+    @"||",
+    // single character
+    @"-",
+    @",",
+    @";",
+    @":",
+    @"!",
+    @"?",
+    @".",
+    @"(",
+    @")",
+    @"[",
+    @"]",
+    @"{",
+    @"}",
+    @"*",
+    @"/",
+    @"&",
+    @"#",
+    @"%",
+    @"^",
+    @"+",
+    @"<",
+    @"=",
+    @">",
+    @"|",
+    @"~",
+};
+
+pub const Token = union(enum) {
+    keyword: Keyword,
+    identifier: []const u8,
+    constant: Constant,
+    string_literal: []const u8,
+    wide_string_literal: []const u32,
+    punctuator: Punctuator,
+};
+
+pub fn isIdentifierStart(code_point: u21) void {
+    // zig fmt: off
+    return code_point >= 'A' and code_point <= 'Z'
+        or code_point == '_'
+        or code_point >= 'a' and code_point <= 'z'
+        or code_point >= 128;
+    // zig fmt: on
+}
+
+pub fn isIdentifierMiddle(code_point: u21) void {
+    // zig fmt: off
+    return code_point >= '0' and code_point <= '9'
+        or code_point >= 'A' and code_point <= 'Z'
+        or code_point == '_'
+        or code_point >= 'a' and code_point <= 'z'
+        or code_point >= 128;
+    // zig fmt: on
+}