From e262e0082950df4abe6bd71ad1e284ee2b1dca52 Mon Sep 17 00:00:00 2001 From: Szymon Nowakowski Date: Mon, 17 Aug 2026 23:46:54 +0200 Subject: [PATCH] datatore: new project --- packages/datastore/build.zig | 27 ++ packages/datastore/build.zig.zon | 16 + packages/datastore/src/root.zig | 657 +++++++++++++++++++++++++++++++ 3 files changed, 700 insertions(+) create mode 100644 packages/datastore/build.zig create mode 100644 packages/datastore/build.zig.zon create mode 100644 packages/datastore/src/root.zig diff --git a/packages/datastore/build.zig b/packages/datastore/build.zig new file mode 100644 index 0000000..428c7a2 --- /dev/null +++ b/packages/datastore/build.zig @@ -0,0 +1,27 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + + const vm_dep = b.dependency("vecmath", .{ + .target = target, + }); + const vm_mod = vm_dep.module("vecmath"); + + const mod = b.addModule("datastore", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .imports = &.{ + .{ .name = "vecmath", .module = vm_mod }, + }, + }); + + const mod_tests = b.addTest(.{ + .root_module = mod, + }); + + const run_mod_tests = b.addRunArtifact(mod_tests); + + const test_step = b.step("test", "Run tests"); + test_step.dependOn(&run_mod_tests.step); +} diff --git a/packages/datastore/build.zig.zon b/packages/datastore/build.zig.zon new file mode 100644 index 0000000..e716269 --- /dev/null +++ b/packages/datastore/build.zig.zon @@ -0,0 +1,16 @@ +.{ + .name = .datastore, + .version = "0.0.0", + .minimum_zig_version = "0.16.0", + .paths = .{ + "src", + "build.zig", + "build.zig.zon", + }, + .fingerprint = 0x193de38ef5a38d70, + .dependencies = .{ + .vecmath = .{ + .path = "../vecmath", + }, + }, +} diff --git a/packages/datastore/src/root.zig b/packages/datastore/src/root.zig new file mode 100644 index 0000000..4e18b6d --- /dev/null +++ b/packages/datastore/src/root.zig @@ -0,0 +1,657 @@ +const std = @import("std"); +const vm = @import("vecmath"); + +pub const page_size = 4096; +pub const page_align = std.mem.Alignment.fromByteUnits(page_size); + +fn IdMixin(comptime Enum: type) type { + const Tag = switch (@typeInfo(Enum)) { + .@"enum" => |e| e.tag_type, + else => @compileError("Expected " ++ @typeName(Enum) ++ " to be an enum."), + }; + + return struct { + pub inline fn fromInt(tag: Tag) Enum { + return @enumFromInt(tag); + } + + pub inline fn toInt(self: Enum) Tag { + return @intFromEnum(self); + } + + pub inline fn next(self: Enum) error{OutOfIds}!Enum { + const tag = @intFromEnum(self); + const next_tag = std.math.add(Tag, tag, 1) catch return error.OutOfIds; + return @enumFromInt(next_tag); + } + }; +} + +pub const ColumnId = enum(u16) { + _, + + pub const fromInt = IdMixin(ColumnId).fromInt; + pub const toInt = IdMixin(ColumnId).toInt; + pub const next = IdMixin(ColumnId).next; +}; + +pub const TableId = enum(u16) { + _, + + pub const fromInt = IdMixin(TableId).fromInt; + pub const toInt = IdMixin(TableId).toInt; + pub const next = IdMixin(TableId).next; +}; + +pub const RowId = enum(u64) { + _, + + pub const fromInt = IdMixin(RowId).fromInt; + pub const toInt = IdMixin(RowId).toInt; + pub const next = IdMixin(RowId).next; +}; + +pub const ColumnType = enum { + /// 1 byte, 0x00 or 0x01 + boolean, + /// 1 bit, bit-packed + bit, + u8, + u16, + u32, + u64, + i8, + i16, + i32, + i64, + f32, + f64, + /// RGBA u8 + color, + /// RGBA f16 + color_hdr, + mat3x2, + mat4x4, + complex, + quaternion, + vec2, + vec2i, + vec3, + vec3i, + vec4, + vec4i, + /// 16 B (1 B len ++ 15 B str) + str16, + /// 64 B (1 B len ++ 63 B str) + str64, + /// 256 B (1 B len ++ 255 B str) + str256, + /// 1 kiB (2 B len ++ <= 1022 B str) + str1k, + /// 4 kiB (2 B len ++ <= 4094 B str) + str4k, + + pub fn countPerPage(self: ColumnType) usize { + return switch (self) { + .str16 => page_size / 16, + .str64 => page_size / 64, + .str256 => page_size / 256, + .str1k => page_size / 1024, + .str4k => page_size / 4096, + inline else => |x| page_size / @sizeOf(ZigTypeSimd(x)) * x.simdWidth(), + }; + } + + test countPerPage { + try std.testing.expectEqual(page_size, countPerPage(.boolean)); + try std.testing.expectEqual(8 * page_size, countPerPage(.bit)); + try std.testing.expectEqual(page_size / 4, countPerPage(.u32)); + try std.testing.expectEqual(page_size / 8, countPerPage(.u64)); + try std.testing.expectEqual(page_size / @sizeOf(vm.Vector3x8) * 8, countPerPage(.vec3)); + } + + pub fn simdWidth(self: ColumnType) usize { + return switch (self) { + .boolean => 1, + .bit => 8, + .u8 => 1, + .u16 => 1, + .u32 => 1, + .u64 => 1, + .i8 => 1, + .i16 => 1, + .i32 => 1, + .i64 => 1, + .f32 => 1, + .f64 => 1, + .color => 1, + .color_hdr => 1, + .mat3x2 => 8, + .mat4x4 => 8, + .complex => 8, + .quaternion => 8, + .vec2 => 8, + .vec2i => 8, + .vec3 => 8, + .vec3i => 8, + .vec4 => 8, + .vec4i => 8, + .str16 => 1, + .str64 => 1, + .str256 => 1, + .str1k => 1, + .str4k => 1, + }; + } + + pub fn ZigType(comptime self: ColumnType) type { + return switch (self) { + .boolean => bool, + .bit => u1, + .u8 => u8, + .u16 => u16, + .u32 => u32, + .u64 => u64, + .i8 => i8, + .i16 => i16, + .i32 => i32, + .i64 => i64, + .f32 => f32, + .f64 => f64, + .color => vm.Color, + .color_hdr => vm.ColorHdr, + .mat3x2 => vm.Matrix3x2, + .mat4x4 => vm.Matrix4x4, + .complex => vm.Complex, + .quaternion => vm.Quaternion, + .vec2 => vm.Vector2, + .vec2i => vm.Vector2Int, + .vec3 => vm.Vector3, + .vec3i => vm.Vector3Int, + .vec4 => vm.Vector4, + .vec4i => vm.Vector4Int, + .str16 => []const u8, + .str64 => []const u8, + .str256 => []const u8, + .str1k => []const u8, + .str4k => []const u8, + }; + } + + pub fn ZigTypeSimd(comptime self: ColumnType) type { + return switch (self) { + .boolean => bool, + .bit => u8, + .u8 => u8, + .u16 => u16, + .u32 => u32, + .u64 => u64, + .i8 => i8, + .i16 => i16, + .i32 => i32, + .i64 => i64, + .f32 => f32, + .f64 => f64, + .color => vm.Color, + .color_hdr => vm.ColorHdr, + .mat3x2 => vm.Matrix3x2x8, + .mat4x4 => vm.Matrix4x4x8, + .complex => vm.Complex_x8, + .quaternion => vm.Quaternion_x8, + .vec2 => vm.Vector2x8, + .vec2i => vm.Vector2Int_x8, + .vec3 => vm.Vector3x8, + .vec3i => vm.Vector3Int_x8, + .vec4 => vm.Vector4x8, + .vec4i => vm.Vector4Int_x8, + .str16 => []const u8, + .str64 => []const u8, + .str256 => []const u8, + .str1k => []const u8, + .str4k => []const u8, + }; + } +}; + +pub const TypedValue = union(enum) { + boolean: bool, + uint: u64, + int: i64, + f32: f32, + f64: f64, + color: vm.Color, + color_hdr: vm.ColorHdr, + mat3x2: vm.Matrix3x2, + mat4x4: vm.Matrix4x4, + complex: vm.Complex, + quaternion: vm.Quaternion, + vec2: vm.Vector2, + vec2i: vm.Vector2Int, + vec3: vm.Vector3, + vec3i: vm.Vector3Int, + vec4: vm.Vector4, + vec4i: vm.Vector4Int, + str: []const u8, + + pub fn initBool(x: bool) TypedValue { + return .{ .boolean = x }; + } + + pub fn initUint(x: u64) TypedValue { + return .{ .uint = x }; + } + + pub fn initColor(x: vm.Color) TypedValue { + return .{ .color = x }; + } + + pub fn initStr(x: []const u8) TypedValue { + return .{ .str = x }; + } +}; + +pub const TableDefinition = struct { + id: TableId, + column_definitions: []const ColumnDefinition, +}; + +pub const ColumnDefinition = struct { + id: ColumnId, + type: ColumnType, +}; + +pub const CellValue = struct { + id: ColumnId, + value: TypedValue, +}; + +pub const Database = struct { + allocator: std.mem.Allocator, + io: std.Io, + + tables: std.AutoHashMapUnmanaged(TableId, Table), + + pub fn init(allocator: std.mem.Allocator, io: std.Io) !Database { + return .{ + .allocator = allocator, + .io = io, + + .tables = .empty, + }; + } + + pub fn deinit(self: *Database) void { + var it = self.tables.valueIterator(); + while (it.next()) |table| { + table.deinit(self.allocator); + } + + self.tables.deinit(self.allocator); + + self.* = undefined; + } + + pub fn createTable(self: *Database, table_definition: TableDefinition) !void { + const get_or_put_result = try self.tables.getOrPut(self.allocator, table_definition.id); + + if (get_or_put_result.found_existing) { + return error.DuplicateTableId; + } + + errdefer _ = self.tables.remove(table_definition.id); + + get_or_put_result.value_ptr.* = try .init(table_definition, self.allocator); + } + + pub fn insertRow(self: *Database, table_id: TableId, row: []const CellValue) !RowId { + const table = self.tables.getPtr(table_id) orelse return error.TableNotFound; + + const row_id = table.next_row_id; + + var it = table.columns.valueIterator(); + while (it.next()) |column| { + var corresponding_cell_value: ?*const CellValue = null; + for (row) |*cell_value| { + if (column.id == cell_value.id) { + if (corresponding_cell_value != null) { + return error.DuplicateColumnId; + } + + corresponding_cell_value = cell_value; + } + } + + if (corresponding_cell_value) |cell_value| { + try column.set(row_id, cell_value.value, self.allocator); + } else { + return error.MissingColumnId; + } + } + + table.next_row_id = try row_id.next(); + return row_id; + } +}; + +pub const Table = struct { + id: TableId, + columns: std.AutoHashMapUnmanaged(ColumnId, Column), + next_row_id: RowId, + + pub fn init(table_definition: TableDefinition, allocator: std.mem.Allocator) !Table { + const column_count = std.math.cast(u32, table_definition.column_definitions.len) orelse return error.OutOfMemory; + + var columns: std.AutoHashMapUnmanaged(ColumnId, Column) = .empty; + try columns.ensureTotalCapacity(allocator, column_count); + + errdefer { + var it = columns.valueIterator(); + while (it.next()) |column| { + column.deinit(allocator); + } + + columns.deinit(allocator); + } + + for (table_definition.column_definitions) |column_definition| { + const get_or_put_result = columns.getOrPutAssumeCapacity(column_definition.id); + + if (get_or_put_result.found_existing) { + return error.DuplicateColumnId; + } + + errdefer _ = columns.remove(column_definition.id); + + get_or_put_result.value_ptr.* = .init(column_definition); + } + + return .{ + .id = table_definition.id, + .columns = columns, + .next_row_id = .fromInt(0), + }; + } + + pub fn deinit(self: *Table, allocator: std.mem.Allocator) void { + var it = self.columns.valueIterator(); + while (it.next()) |column| { + column.deinit(allocator); + } + + self.columns.deinit(allocator); + + self.* = undefined; + } +}; + +pub const Column = struct { + id: ColumnId, + type: ColumnType, + pages: std.ArrayList(Page), + + pub fn init(column_definition: ColumnDefinition) Column { + return .{ + .id = column_definition.id, + .type = column_definition.type, + .pages = .empty, + }; + } + + pub fn deinit(self: *Column, allocator: std.mem.Allocator) void { + for (self.pages.items) |*page| { + page.deinit(); + } + + self.pages.deinit(allocator); + + self.* = undefined; + } + + pub fn set(self: *Column, row_id: RowId, value: TypedValue, allocator: std.mem.Allocator) !void { + const count_per_page = self.type.countPerPage(); + + const page_i = @as(usize, row_id.toInt() / count_per_page); + const page_j = @as(usize, row_id.toInt() % count_per_page); + + try self.pages.ensureTotalCapacity(allocator, page_i + 1); + while (page_i >= self.pages.items.len) { + self.pages.appendAssumeCapacity(try .init()); + } + + const page = &self.pages.items[page_i]; + + switch (value) { + .boolean => |x| switch (self.type) { + .boolean => page.assign(bool, page_j, x), + .bit => page.setBit(page_j, x), + else => return error.TypeError, + }, + .uint => |x| switch (self.type) { + .u8 => page.assign(u8, page_j, std.math.cast(u8, x) orelse return error.RangeError), + .u16 => page.assign(u16, page_j, std.math.cast(u16, x) orelse return error.RangeError), + .u32 => page.assign(u32, page_j, std.math.cast(u32, x) orelse return error.RangeError), + .u64 => page.assign(u64, page_j, x), + else => return error.TypeError, + }, + .int => |x| switch (self.type) { + .i8 => page.assign(i8, page_j, std.math.cast(i8, x) orelse return error.RangeError), + .i16 => page.assign(i16, page_j, std.math.cast(i16, x) orelse return error.RangeError), + .i32 => page.assign(i32, page_j, std.math.cast(i32, x) orelse return error.RangeError), + .i64 => page.assign(i64, page_j, x), + else => return error.TypeError, + }, + .f32 => |x| { + if (self.type != .f32) return error.TypeError; + page.assign(f32, page_j, x); + }, + .f64 => |x| { + if (self.type != .f64) return error.TypeError; + page.assign(f64, page_j, x); + }, + .color => |x| { + if (self.type != .color) return error.TypeError; + page.assign(vm.Color, page_j, x); + }, + .color_hdr => |x| { + if (self.type != .color_hdr) return error.TypeError; + page.assign(vm.ColorHdr, page_j, x); + }, + .mat3x2 => |x| { + _ = x; + @panic("TODO"); + //if (self.type != .mat3x2) return error.TypeError; + //const simd_i = page_j / 8; + //const simd_j = page_j % 8; + //const ptr = page.getPtr(vm.Matrix3x2x8, simd_i); + //ptr.ix[simd_j] = x.ix; + //ptr.iy[simd_j] = x.iy; + //ptr.jx[simd_j] = x.jx; + //ptr.jy[simd_j] = x.jy; + //ptr.tx[simd_j] = x.tx; + //ptr.ty[simd_j] = x.ty; + }, + .mat4x4 => |x| { + _ = x; + @panic("TODO"); + }, + .complex => |x| { + _ = x; + @panic("TODO"); + }, + .quaternion => |x| { + _ = x; + @panic("TODO"); + }, + .vec2 => |x| { + _ = x; + @panic("TODO"); + }, + .vec2i => |x| { + _ = x; + @panic("TODO"); + }, + .vec3 => |x| { + _ = x; + @panic("TODO"); + }, + .vec3i => |x| { + _ = x; + @panic("TODO"); + }, + .vec4 => |x| { + _ = x; + @panic("TODO"); + }, + .vec4i => |x| { + _ = x; + @panic("TODO"); + }, + .str => |x| switch (self.type) { + .str16 => { + if (x.len > 15) return error.RangeError; + const slice = page.ptr[page_j * 16 .. (page_j + 1) * 16]; + @as(*u8, @ptrCast(slice[0..1])).* = @intCast(x.len); + @memcpy(slice[1 .. 1 + x.len], x); + @memset(slice[1 + x.len ..], 0); + }, + .str64 => { + if (x.len > 63) return error.RangeError; + const slice = page.ptr[page_j * 64 .. (page_j + 1) * 64]; + @as(*u8, @ptrCast(slice[0..1])).* = @intCast(x.len); + @memcpy(slice[1 .. 1 + x.len], x); + @memset(slice[1 + x.len ..], 0); + }, + .str256 => { + if (x.len > 255) return error.RangeError; + const slice = page.ptr[page_j * 256 .. (page_j + 1) * 256]; + @as(*u8, @ptrCast(slice[0..1])).* = @intCast(x.len); + @memcpy(slice[1 .. 1 + x.len], x); + @memset(slice[1 + x.len ..], 0); + }, + .str1k => { + if (x.len > 1022) return error.RangeError; + const slice = page.ptr[page_j * 1024 .. (page_j + 1) * 1024]; + @as(*u16, @ptrCast(@alignCast(slice[0..2]))).* = @intCast(x.len); + @memcpy(slice[2 .. 2 + x.len], x); + @memset(slice[2 + x.len ..], 0); + }, + .str4k => { + if (x.len > 4094) return error.RangeError; + const slice = page.ptr[page_j * 4096 .. (page_j + 1) * 4096]; + @as(*u16, @ptrCast(@alignCast(slice[0..2]))).* = @intCast(x.len); + @memcpy(slice[2 .. 2 + x.len], x); + @memset(slice[2 + x.len ..], 0); + }, + else => return error.TypeError, + }, + } + } +}; + +pub const Page = struct { + ptr: *align(page_align.toByteUnits()) [page_size]u8, + + pub fn init() !Page { + const bytes = try std.heap.page_allocator.alignedAlloc(u8, page_align, page_size); + return .{ + .ptr = bytes[0..page_size], + }; + } + + pub fn deinit(self: *Page) void { + const bytes = @as([]u8, std.mem.asBytes(self.ptr)); + std.heap.page_allocator.free(bytes); + + self.* = undefined; + } + + pub fn assign(self: Page, comptime T: type, index: usize, value: T) void { + const byte_offset = index * @sizeOf(T); + std.debug.assert(byte_offset + @sizeOf(T) <= page_size); + + @as([*]T, @ptrCast(self.ptr))[index] = value; + } + + pub fn getPtr(self: Page, comptime T: type, index: usize) *T { + const byte_offset = index * @sizeOf(T); + std.debug.assert(byte_offset + @sizeOf(T) <= page_size); + + return @ptrCast(@alignCast(self.ptr[byte_offset..].ptr)); + } + + pub fn setBit(self: Page, bit_index: usize, value: bool) void { + const byte_index = bit_index / 8; + const bit_shift = @as(u3, @intCast(bit_index % 8)); + + if (value) { + const mask = @as(u8, 1) << bit_shift; + self.ptr[byte_index] |= mask; + } else { + const mask = ~(@as(u8, 1) << bit_shift); + self.ptr[byte_index] &= mask; + } + } +}; + +test { + var db: Database = try .init(std.testing.allocator, std.testing.io); + defer db.deinit(); + + const table_ids = struct { + const users = TableId.fromInt(0); + }; + + const column_ids = struct { + const username = ColumnId.fromInt(0); + const password_hash = ColumnId.fromInt(1); + const color = ColumnId.fromInt(2); + }; + + try db.createTable(.{ + .id = table_ids.users, + .column_definitions = &.{ + .{ + .id = column_ids.username, + .type = .str64, + }, + .{ + .id = column_ids.password_hash, + .type = .str256, + }, + .{ + .id = column_ids.color, + .type = .color, + }, + }, + }); + + _ = try db.insertRow(table_ids.users, &.{ + .{ + .id = column_ids.username, + .value = .initStr("admin"), + }, + .{ + .id = column_ids.password_hash, + .value = .initStr("$argon2id$v=19$m=16,t=2,p=1$R0l1WjlpM1BUSXZ3dU0xbQ$muWG51NiQjaIe6RTh6LJgk/7VJMvtbtJiN5Z11fEFbI"), + }, + .{ + .id = column_ids.color, + .value = .initColor(.red), + }, + }); + + _ = try db.insertRow(table_ids.users, &.{ + .{ + .id = column_ids.username, + .value = .initStr("user"), + }, + .{ + .id = column_ids.password_hash, + .value = .initStr("$argon2id$v=19$m=16,t=2,p=1$RlNyZTR3dklRU1hVb3hHSA$8EDVcnkQouADrpvb5bONOqQWrUil1Jc/YaZGe3t34q0"), + }, + .{ + .id = column_ids.color, + .value = .initColor(.blue), + }, + }); +}