renati/castle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

datatore: new project

Browse Source
This commit is contained in:
Szymon Nowakowski
2026-08-17 23:46:54 +02:00
parent cbbe7f2ea1
commit e262e00829
3 changed files with 700 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
packages/datastore/build.zig Normal file
View File

@@ -0,0 +1,27 @@
const std = @import("std");
pub fn build(b: *std.Build) void {
const target = b.standardTargetOptions(.{});
const vm_dep = b.dependency("vecmath", .{
.target = target,
});
const vm_mod = vm_dep.module("vecmath");
const mod = b.addModule("datastore", .{
.root_source_file = b.path("src/root.zig"),
.target = target,
.imports = &.{
.{ .name = "vecmath", .module = vm_mod },
},
});
const mod_tests = b.addTest(.{
.root_module = mod,
});
const run_mod_tests = b.addRunArtifact(mod_tests);
const test_step = b.step("test", "Run tests");
test_step.dependOn(&run_mod_tests.step);
}

16
packages/datastore/build.zig.zon Normal file
View File

@@ -0,0 +1,16 @@
.{
.name = .datastore,
.version = "0.0.0",
.minimum_zig_version = "0.16.0",
.paths = .{
"src",
"build.zig",
"build.zig.zon",
},
.fingerprint = 0x193de38ef5a38d70,
.dependencies = .{
.vecmath = .{
.path = "../vecmath",
},
},
}

657
packages/datastore/src/root.zig Normal file
View File

@@ -0,0 +1,657 @@
const std = @import("std");
const vm = @import("vecmath");
pub const page_size = 4096;
pub const page_align = std.mem.Alignment.fromByteUnits(page_size);
fn IdMixin(comptime Enum: type) type {
const Tag = switch (@typeInfo(Enum)) {
.@"enum" => |e| e.tag_type,
else => @compileError("Expected " ++ @typeName(Enum) ++ " to be an enum."),
};
return struct {
pub inline fn fromInt(tag: Tag) Enum {
return @enumFromInt(tag);
}
pub inline fn toInt(self: Enum) Tag {
return @intFromEnum(self);
}
pub inline fn next(self: Enum) error{OutOfIds}!Enum {
const tag = @intFromEnum(self);
const next_tag = std.math.add(Tag, tag, 1) catch return error.OutOfIds;
return @enumFromInt(next_tag);
}
};
}
pub const ColumnId = enum(u16) {
_,
pub const fromInt = IdMixin(ColumnId).fromInt;
pub const toInt = IdMixin(ColumnId).toInt;
pub const next = IdMixin(ColumnId).next;
};
pub const TableId = enum(u16) {
_,
pub const fromInt = IdMixin(TableId).fromInt;
pub const toInt = IdMixin(TableId).toInt;
pub const next = IdMixin(TableId).next;
};
pub const RowId = enum(u64) {
_,
pub const fromInt = IdMixin(RowId).fromInt;
pub const toInt = IdMixin(RowId).toInt;
pub const next = IdMixin(RowId).next;
};
pub const ColumnType = enum {
/// 1 byte, 0x00 or 0x01
boolean,
/// 1 bit, bit-packed
bit,
u8,
u16,
u32,
u64,
i8,
i16,
i32,
i64,
f32,
f64,
/// RGBA u8
color,
/// RGBA f16
color_hdr,
mat3x2,
mat4x4,
complex,
quaternion,
vec2,
vec2i,
vec3,
vec3i,
vec4,
vec4i,
/// 16 B (1 B len ++ 15 B str)
str16,
/// 64 B (1 B len ++ 63 B str)
str64,
/// 256 B (1 B len ++ 255 B str)
str256,
/// 1 kiB (2 B len ++ <= 1022 B str)
str1k,
/// 4 kiB (2 B len ++ <= 4094 B str)
str4k,
pub fn countPerPage(self: ColumnType) usize {
return switch (self) {
.str16 => page_size / 16,
.str64 => page_size / 64,
.str256 => page_size / 256,
.str1k => page_size / 1024,
.str4k => page_size / 4096,
inline else => |x| page_size / @sizeOf(ZigTypeSimd(x)) * x.simdWidth(),
};
}
test countPerPage {
try std.testing.expectEqual(page_size, countPerPage(.boolean));
try std.testing.expectEqual(8 * page_size, countPerPage(.bit));
try std.testing.expectEqual(page_size / 4, countPerPage(.u32));
try std.testing.expectEqual(page_size / 8, countPerPage(.u64));
try std.testing.expectEqual(page_size / @sizeOf(vm.Vector3x8) * 8, countPerPage(.vec3));
}
pub fn simdWidth(self: ColumnType) usize {
return switch (self) {
.boolean => 1,
.bit => 8,
.u8 => 1,
.u16 => 1,
.u32 => 1,
.u64 => 1,
.i8 => 1,
.i16 => 1,
.i32 => 1,
.i64 => 1,
.f32 => 1,
.f64 => 1,
.color => 1,
.color_hdr => 1,
.mat3x2 => 8,
.mat4x4 => 8,
.complex => 8,
.quaternion => 8,
.vec2 => 8,
.vec2i => 8,
.vec3 => 8,
.vec3i => 8,
.vec4 => 8,
.vec4i => 8,
.str16 => 1,
.str64 => 1,
.str256 => 1,
.str1k => 1,
.str4k => 1,
};
}
pub fn ZigType(comptime self: ColumnType) type {
return switch (self) {
.boolean => bool,
.bit => u1,
.u8 => u8,
.u16 => u16,
.u32 => u32,
.u64 => u64,
.i8 => i8,
.i16 => i16,
.i32 => i32,
.i64 => i64,
.f32 => f32,
.f64 => f64,
.color => vm.Color,
.color_hdr => vm.ColorHdr,
.mat3x2 => vm.Matrix3x2,
.mat4x4 => vm.Matrix4x4,
.complex => vm.Complex,
.quaternion => vm.Quaternion,
.vec2 => vm.Vector2,
.vec2i => vm.Vector2Int,
.vec3 => vm.Vector3,
.vec3i => vm.Vector3Int,
.vec4 => vm.Vector4,
.vec4i => vm.Vector4Int,
.str16 => []const u8,
.str64 => []const u8,
.str256 => []const u8,
.str1k => []const u8,
.str4k => []const u8,
};
}
pub fn ZigTypeSimd(comptime self: ColumnType) type {
return switch (self) {
.boolean => bool,
.bit => u8,
.u8 => u8,
.u16 => u16,
.u32 => u32,
.u64 => u64,
.i8 => i8,
.i16 => i16,
.i32 => i32,
.i64 => i64,
.f32 => f32,
.f64 => f64,
.color => vm.Color,
.color_hdr => vm.ColorHdr,
.mat3x2 => vm.Matrix3x2x8,
.mat4x4 => vm.Matrix4x4x8,
.complex => vm.Complex_x8,
.quaternion => vm.Quaternion_x8,
.vec2 => vm.Vector2x8,
.vec2i => vm.Vector2Int_x8,
.vec3 => vm.Vector3x8,
.vec3i => vm.Vector3Int_x8,
.vec4 => vm.Vector4x8,
.vec4i => vm.Vector4Int_x8,
.str16 => []const u8,
.str64 => []const u8,
.str256 => []const u8,
.str1k => []const u8,
.str4k => []const u8,
};
}
};
pub const TypedValue = union(enum) {
boolean: bool,
uint: u64,
int: i64,
f32: f32,
f64: f64,
color: vm.Color,
color_hdr: vm.ColorHdr,
mat3x2: vm.Matrix3x2,
mat4x4: vm.Matrix4x4,
complex: vm.Complex,
quaternion: vm.Quaternion,
vec2: vm.Vector2,
vec2i: vm.Vector2Int,
vec3: vm.Vector3,
vec3i: vm.Vector3Int,
vec4: vm.Vector4,
vec4i: vm.Vector4Int,
str: []const u8,
pub fn initBool(x: bool) TypedValue {
return .{ .boolean = x };
}
pub fn initUint(x: u64) TypedValue {
return .{ .uint = x };
}
pub fn initColor(x: vm.Color) TypedValue {
return .{ .color = x };
}
pub fn initStr(x: []const u8) TypedValue {
return .{ .str = x };
}
};
pub const TableDefinition = struct {
id: TableId,
column_definitions: []const ColumnDefinition,
};
pub const ColumnDefinition = struct {
id: ColumnId,
type: ColumnType,
};
pub const CellValue = struct {
id: ColumnId,
value: TypedValue,
};
pub const Database = struct {
allocator: std.mem.Allocator,
io: std.Io,
tables: std.AutoHashMapUnmanaged(TableId, Table),
pub fn init(allocator: std.mem.Allocator, io: std.Io) !Database {
return .{
.allocator = allocator,
.io = io,
.tables = .empty,
};
}
pub fn deinit(self: *Database) void {
var it = self.tables.valueIterator();
while (it.next()) |table| {
table.deinit(self.allocator);
}
self.tables.deinit(self.allocator);
self.* = undefined;
}
pub fn createTable(self: *Database, table_definition: TableDefinition) !void {
const get_or_put_result = try self.tables.getOrPut(self.allocator, table_definition.id);
if (get_or_put_result.found_existing) {
return error.DuplicateTableId;
}
errdefer _ = self.tables.remove(table_definition.id);
get_or_put_result.value_ptr.* = try .init(table_definition, self.allocator);
}
pub fn insertRow(self: *Database, table_id: TableId, row: []const CellValue) !RowId {
const table = self.tables.getPtr(table_id) orelse return error.TableNotFound;
const row_id = table.next_row_id;
var it = table.columns.valueIterator();
while (it.next()) |column| {
var corresponding_cell_value: ?*const CellValue = null;
for (row) |*cell_value| {
if (column.id == cell_value.id) {
if (corresponding_cell_value != null) {
return error.DuplicateColumnId;
}
corresponding_cell_value = cell_value;
}
}
if (corresponding_cell_value) |cell_value| {
try column.set(row_id, cell_value.value, self.allocator);
} else {
return error.MissingColumnId;
}
}
table.next_row_id = try row_id.next();
return row_id;
}
};
pub const Table = struct {
id: TableId,
columns: std.AutoHashMapUnmanaged(ColumnId, Column),
next_row_id: RowId,
pub fn init(table_definition: TableDefinition, allocator: std.mem.Allocator) !Table {
const column_count = std.math.cast(u32, table_definition.column_definitions.len) orelse return error.OutOfMemory;
var columns: std.AutoHashMapUnmanaged(ColumnId, Column) = .empty;
try columns.ensureTotalCapacity(allocator, column_count);
errdefer {
var it = columns.valueIterator();
while (it.next()) |column| {
column.deinit(allocator);
}
columns.deinit(allocator);
}
for (table_definition.column_definitions) |column_definition| {
const get_or_put_result = columns.getOrPutAssumeCapacity(column_definition.id);
if (get_or_put_result.found_existing) {
return error.DuplicateColumnId;
}
errdefer _ = columns.remove(column_definition.id);
get_or_put_result.value_ptr.* = .init(column_definition);
}
return .{
.id = table_definition.id,
.columns = columns,
.next_row_id = .fromInt(0),
};
}
pub fn deinit(self: *Table, allocator: std.mem.Allocator) void {
var it = self.columns.valueIterator();
while (it.next()) |column| {
column.deinit(allocator);
}
self.columns.deinit(allocator);
self.* = undefined;
}
};
pub const Column = struct {
id: ColumnId,
type: ColumnType,
pages: std.ArrayList(Page),
pub fn init(column_definition: ColumnDefinition) Column {
return .{
.id = column_definition.id,
.type = column_definition.type,
.pages = .empty,
};
}
pub fn deinit(self: *Column, allocator: std.mem.Allocator) void {
for (self.pages.items) |*page| {
page.deinit();
}
self.pages.deinit(allocator);
self.* = undefined;
}
pub fn set(self: *Column, row_id: RowId, value: TypedValue, allocator: std.mem.Allocator) !void {
const count_per_page = self.type.countPerPage();
const page_i = @as(usize, row_id.toInt() / count_per_page);
const page_j = @as(usize, row_id.toInt() % count_per_page);
try self.pages.ensureTotalCapacity(allocator, page_i + 1);
while (page_i >= self.pages.items.len) {
self.pages.appendAssumeCapacity(try .init());
}
const page = &self.pages.items[page_i];
switch (value) {
.boolean => |x| switch (self.type) {
.boolean => page.assign(bool, page_j, x),
.bit => page.setBit(page_j, x),
else => return error.TypeError,
},
.uint => |x| switch (self.type) {
.u8 => page.assign(u8, page_j, std.math.cast(u8, x) orelse return error.RangeError),
.u16 => page.assign(u16, page_j, std.math.cast(u16, x) orelse return error.RangeError),
.u32 => page.assign(u32, page_j, std.math.cast(u32, x) orelse return error.RangeError),
.u64 => page.assign(u64, page_j, x),
else => return error.TypeError,
},
.int => |x| switch (self.type) {
.i8 => page.assign(i8, page_j, std.math.cast(i8, x) orelse return error.RangeError),
.i16 => page.assign(i16, page_j, std.math.cast(i16, x) orelse return error.RangeError),
.i32 => page.assign(i32, page_j, std.math.cast(i32, x) orelse return error.RangeError),
.i64 => page.assign(i64, page_j, x),
else => return error.TypeError,
},
.f32 => |x| {
if (self.type != .f32) return error.TypeError;
page.assign(f32, page_j, x);
},
.f64 => |x| {
if (self.type != .f64) return error.TypeError;
page.assign(f64, page_j, x);
},
.color => |x| {
if (self.type != .color) return error.TypeError;
page.assign(vm.Color, page_j, x);
},
.color_hdr => |x| {
if (self.type != .color_hdr) return error.TypeError;
page.assign(vm.ColorHdr, page_j, x);
},
.mat3x2 => |x| {
_ = x;
@panic("TODO");
//if (self.type != .mat3x2) return error.TypeError;
//const simd_i = page_j / 8;
//const simd_j = page_j % 8;
//const ptr = page.getPtr(vm.Matrix3x2x8, simd_i);
//ptr.ix[simd_j] = x.ix;
//ptr.iy[simd_j] = x.iy;
//ptr.jx[simd_j] = x.jx;
//ptr.jy[simd_j] = x.jy;
//ptr.tx[simd_j] = x.tx;
//ptr.ty[simd_j] = x.ty;
},
.mat4x4 => |x| {
_ = x;
@panic("TODO");
},
.complex => |x| {
_ = x;
@panic("TODO");
},
.quaternion => |x| {
_ = x;
@panic("TODO");
},
.vec2 => |x| {
_ = x;
@panic("TODO");
},
.vec2i => |x| {
_ = x;
@panic("TODO");
},
.vec3 => |x| {
_ = x;
@panic("TODO");
},
.vec3i => |x| {
_ = x;
@panic("TODO");
},
.vec4 => |x| {
_ = x;
@panic("TODO");
},
.vec4i => |x| {
_ = x;
@panic("TODO");
},
.str => |x| switch (self.type) {
.str16 => {
if (x.len > 15) return error.RangeError;
const slice = page.ptr[page_j * 16 .. (page_j + 1) * 16];
@as(*u8, @ptrCast(slice[0..1])).* = @intCast(x.len);
@memcpy(slice[1 .. 1 + x.len], x);
@memset(slice[1 + x.len ..], 0);
},
.str64 => {
if (x.len > 63) return error.RangeError;
const slice = page.ptr[page_j * 64 .. (page_j + 1) * 64];
@as(*u8, @ptrCast(slice[0..1])).* = @intCast(x.len);
@memcpy(slice[1 .. 1 + x.len], x);
@memset(slice[1 + x.len ..], 0);
},
.str256 => {
if (x.len > 255) return error.RangeError;
const slice = page.ptr[page_j * 256 .. (page_j + 1) * 256];
@as(*u8, @ptrCast(slice[0..1])).* = @intCast(x.len);
@memcpy(slice[1 .. 1 + x.len], x);
@memset(slice[1 + x.len ..], 0);
},
.str1k => {
if (x.len > 1022) return error.RangeError;
const slice = page.ptr[page_j * 1024 .. (page_j + 1) * 1024];
@as(*u16, @ptrCast(@alignCast(slice[0..2]))).* = @intCast(x.len);
@memcpy(slice[2 .. 2 + x.len], x);
@memset(slice[2 + x.len ..], 0);
},
.str4k => {
if (x.len > 4094) return error.RangeError;
const slice = page.ptr[page_j * 4096 .. (page_j + 1) * 4096];
@as(*u16, @ptrCast(@alignCast(slice[0..2]))).* = @intCast(x.len);
@memcpy(slice[2 .. 2 + x.len], x);
@memset(slice[2 + x.len ..], 0);
},
else => return error.TypeError,
},
}
}
};
pub const Page = struct {
ptr: *align(page_align.toByteUnits()) [page_size]u8,
pub fn init() !Page {
const bytes = try std.heap.page_allocator.alignedAlloc(u8, page_align, page_size);
return .{
.ptr = bytes[0..page_size],
};
}
pub fn deinit(self: *Page) void {
const bytes = @as([]u8, std.mem.asBytes(self.ptr));
std.heap.page_allocator.free(bytes);
self.* = undefined;
}
pub fn assign(self: Page, comptime T: type, index: usize, value: T) void {
const byte_offset = index * @sizeOf(T);
std.debug.assert(byte_offset + @sizeOf(T) <= page_size);
@as([*]T, @ptrCast(self.ptr))[index] = value;
}
pub fn getPtr(self: Page, comptime T: type, index: usize) *T {
const byte_offset = index * @sizeOf(T);
std.debug.assert(byte_offset + @sizeOf(T) <= page_size);
return @ptrCast(@alignCast(self.ptr[byte_offset..].ptr));
}
pub fn setBit(self: Page, bit_index: usize, value: bool) void {
const byte_index = bit_index / 8;
const bit_shift = @as(u3, @intCast(bit_index % 8));
if (value) {
const mask = @as(u8, 1) << bit_shift;
self.ptr[byte_index] |= mask;
} else {
const mask = ~(@as(u8, 1) << bit_shift);
self.ptr[byte_index] &= mask;
}
}
};
test {
var db: Database = try .init(std.testing.allocator, std.testing.io);
defer db.deinit();
const table_ids = struct {
const users = TableId.fromInt(0);
};
const column_ids = struct {
const username = ColumnId.fromInt(0);
const password_hash = ColumnId.fromInt(1);
const color = ColumnId.fromInt(2);
};
try db.createTable(.{
.id = table_ids.users,
.column_definitions = &.{
.{
.id = column_ids.username,
.type = .str64,
},
.{
.id = column_ids.password_hash,
.type = .str256,
},
.{
.id = column_ids.color,
.type = .color,
},
},
});
_ = try db.insertRow(table_ids.users, &.{
.{
.id = column_ids.username,
.value = .initStr("admin"),
},
.{
.id = column_ids.password_hash,
.value = .initStr("$argon2id$v=19$m=16,t=2,p=1$R0l1WjlpM1BUSXZ3dU0xbQ$muWG51NiQjaIe6RTh6LJgk/7VJMvtbtJiN5Z11fEFbI"),
},
.{
.id = column_ids.color,
.value = .initColor(.red),
},
});
_ = try db.insertRow(table_ids.users, &.{
.{
.id = column_ids.username,
.value = .initStr("user"),
},
.{
.id = column_ids.password_hash,
.value = .initStr("$argon2id$v=19$m=16,t=2,p=1$RlNyZTR3dklRU1hVb3hHSA$8EDVcnkQouADrpvb5bONOqQWrUil1Jc/YaZGe3t34q0"),
},
.{
.id = column_ids.color,
.value = .initColor(.blue),
},
});
}