HAXY

▄ ▄▄▄▄▄ ▄ ▄▀ ▄▀▀▀▀▀▀ ▄ ▀▄ ▀▀▀▀▀▀▀▀ ▀▀ ▀▀▄ ▀▀▀▀▀▀▀▀ ▀▀▀ ▀▀▀▀ ▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀ ▀▄ ▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀ ▀ ▀▀▀▀▀▀▀▀▀▀▄ ▄▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▄ ▄▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀▀▀▀ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▄ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▄▄▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▄ ▄▀▀▀▀▀▀▀▀ ▀▀▀▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▄▄▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀ ▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▄▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀ ▀▀▀▀ ▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀▀▀▄▄▀▀▀▄▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▄ ▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▄▄▀▀▀▀▀▀▀▀▀▀ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀

╭ event.zig ────────────────────────────────────────────────────╮ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ╰───────────────────────────────────────────────────────────────╯

╭────╮ ╭ clone ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╮ │http│ ssh │ │ ╰────╯ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╯

.. test/ command.zig ╭─────────────────────────────────╮ │event.zig │ ╰─────────────────────────────────╯ lib.zig main.zig serve.zig ssh_helper.zig test.zig testnet.zig

1 const std = @import("std"); 2 const xit = @import("xit"); 3 const rp = xit.repo; 4 const hash = xit.hash; 5 6 const event_id_size: usize = 32; 7 8 pub const EventKind = enum { 9 issue, 10 }; 11 12 pub const EventData = union(EventKind) { 13 issue: struct { 14 title: []const u8, 15 description: []const u8, 16 tags: []const u8, 17 }, 18 19 pub fn read( 20 comptime DB: type, 21 comptime hash_kind: hash.HashKind, 22 allocator: std.mem.Allocator, 23 map: DB.HashMap(.read_only), 24 kind: EventKind, 25 ) !EventData { 26 return switch (kind) { 27 .issue => .{ 28 .issue = .{ 29 .title = try readBytes(DB, hash_kind, allocator, map, "title"), 30 .description = try readBytes(DB, hash_kind, allocator, map, "description"), 31 .tags = try readBytes(DB, hash_kind, allocator, map, "tags"), 32 }, 33 }, 34 }; 35 } 36 37 fn readBytes( 38 comptime DB: type, 39 comptime hash_kind: hash.HashKind, 40 allocator: std.mem.Allocator, 41 map: DB.HashMap(.read_only), 42 field_name: []const u8, 43 ) ![]const u8 { 44 const cursor = try map.getCursor(hash.hashInt(hash_kind, field_name)) orelse return error.NotFound; 45 return try cursor.readBytesAlloc(allocator, null); 46 } 47 }; 48 49 pub const Event = struct { 50 id: [event_id_size * 2]u8, 51 data: EventData, 52 }; 53 54 /// associates an Event with a git object id. 55 /// we save this oid in the database so we can reliably re-consume 56 /// events if their oid changed. this can happen if the branch 57 /// is rebased and force-pushed. 58 pub fn RepoEvent(comptime hash_kind: hash.HashKind) type { 59 return struct { 60 parent_oid: ?[hash.byteLen(hash_kind)]u8, 61 oid: [hash.byteLen(hash_kind)]u8, 62 event: Event, 63 }; 64 } 65 66 pub fn randomId(random: std.Random) [event_id_size]u8 { 67 var id_bytes: [event_id_size]u8 = undefined; 68 random.bytes(&id_bytes); 69 return id_bytes; 70 } 71 72 pub fn consume( 73 comptime DB: type, 74 comptime hash_kind: hash.HashKind, 75 io: std.Io, 76 db: *DB, 77 db_file: std.Io.File, 78 repo_events: []RepoEvent(hash_kind), 79 ) !void { 80 const Ctx = struct { 81 repo_events: []RepoEvent(hash_kind), 82 83 pub fn run(ctx: @This(), cursor: *DB.Cursor(.read_write)) !void { 84 const moment = try DB.HashMap(.read_write).init(cursor.*); 85 86 // the last_object_id represents the object id that was last consumed 87 var last_object_id_maybe: ?[hash.byteLen(hash_kind)]u8 = null; 88 if (try moment.getCursor(hash.hashInt(hash_kind, "haxy-last-object-id"))) |last_object_id_cursor| { 89 var last_object_id_buffer: [hash.byteLen(hash_kind)]u8 = undefined; 90 _ = try last_object_id_cursor.readBytes(&last_object_id_buffer); 91 last_object_id_maybe = last_object_id_buffer; 92 } 93 94 // the list with all of haxy's state, including materialized views. 95 // the reason it is a list is so we can keep every previous haxy 96 // state, making it easy to revert to an older state if the user 97 // rebases some of the past commits. 98 const haxy_cursor = try moment.putCursor(hash.hashInt(hash_kind, "haxy")); 99 const haxy = try DB.ArrayList(.read_write).init(haxy_cursor); 100 101 // add a new item to the haxy list created above. 102 // we call the item haxy_moments. it is a map of object id to haxy moment. 103 // in other words, it maps each object id to a hash map containing the 104 // state that the database was in when that object id was consumed. 105 var haxy_moments_cursor = try haxy.appendCursor(); 106 // use the previous haxy_moments as the basis for this one if it exists 107 if (try haxy.getCursor(-2)) |last_haxy_moments_cursor| { 108 try haxy_moments_cursor.write(.{ .slot = last_haxy_moments_cursor.slot() }); 109 } 110 var haxy_moments = try DB.HashMap(.read_write).init(haxy_moments_cursor); 111 112 // for each event we want to consume... 113 for (ctx.repo_events) |repo_event| { 114 // if this object id has already been consumed, skip it 115 if (null != try haxy_moments.getCursor(hash.bytesToInt(hash_kind, &repo_event.oid))) { 116 continue; 117 } 118 119 // compare last_object_id to the event's parent. this is important 120 // in situations where the branch was rebased, because in that case 121 // the last_object_id may no longer be valid. 122 if (last_object_id_maybe) |*last_object_id| { 123 if (repo_event.parent_oid) |*parent_oid| { 124 if (!std.mem.eql(u8, last_object_id, parent_oid)) { 125 // the last_object_id does not match the current event's parent id. 126 // this means that a rebase occurred. we just need to revert the 127 // haxy list to the state it was in what parent_oid was consumed, 128 // and then create a new haxy_moments map to work with. 129 130 const old_moment_cursor = try haxy_moments.getCursor(hash.bytesToInt(hash_kind, parent_oid)) orelse return error.CursorNotFound; 131 const old_moment = try DB.HashMap(.read_only).init(old_moment_cursor); 132 133 const old_moment_index_cursor = try old_moment.getCursor(hash.hashInt(hash_kind, "index")) orelse return error.CursorNotFound; 134 const old_moment_index = try old_moment_index_cursor.readUint(); 135 136 // resize the haxy list so we truncate all the moments that were 137 // created after the parent_oid was consumed 138 try haxy.slice(old_moment_index + 1); 139 140 // make a new haxy moment and set its initial value to the last haxy moment 141 haxy_moments_cursor = try haxy.appendCursor(); 142 const old_haxy_moments_cursor = try haxy.getCursor(old_moment_index) orelse return error.CursorNotFound; 143 try haxy_moments_cursor.write(.{ .slot = old_haxy_moments_cursor.slot() }); 144 haxy_moments = try DB.HashMap(.read_write).init(haxy_moments_cursor); 145 146 last_object_id_maybe = parent_oid.*; 147 } 148 } else { 149 // the branch was rebased all the way to the very beginning. 150 // we have a repo event with no parent, which means it is now 151 // the very first event. all we need to do is set the haxy list 152 // to be empty and make a new haxy_moments map to work with. 153 154 try haxy.slice(0); 155 haxy_moments_cursor = try haxy.appendCursor(); 156 haxy_moments = try DB.HashMap(.read_write).init(haxy_moments_cursor); 157 158 last_object_id_maybe = null; 159 } 160 } else { 161 if (repo_event.parent_oid) |_| { 162 // there is no last_object_id, but this event has a parent. 163 // this is an invalid state. if the event has a parent, that 164 // implies that an event has already been processed, but 165 // if that's true then there would be a last_object_id. 166 return error.UnexpectedParent; 167 } 168 } 169 170 // create a moment for this object id 171 var haxy_moment_cursor = try haxy_moments.putCursor(hash.bytesToInt(hash_kind, &repo_event.oid)); 172 173 // if there was a previous object id, make this haxy moment's initial value to it. 174 // this efficiently "clones" the map so we make further modifications based on it. 175 if (last_object_id_maybe) |*last_object_id| { 176 if (try haxy_moments.getCursor(hash.bytesToInt(hash_kind, last_object_id))) |last_haxy_moment_cursor| { 177 try haxy_moment_cursor.write(.{ .slot = last_haxy_moment_cursor.slot() }); 178 } 179 } 180 181 const haxy_moment = try DB.HashMap(.read_write).init(haxy_moment_cursor); 182 183 // associate this moment with the index it will first appear at in the haxy list. 184 // this will be important later so we can truncate that list if the user ever 185 // rebases starting at this object id. 186 try haxy_moment.put(hash.hashInt(hash_kind, "index"), .{ .uint = try haxy.count() - 1 }); 187 188 // consume the event into the views map 189 { 190 const event = repo_event.event; 191 192 // get the id of the current event as bytes 193 var current_event_id: [event_id_size]u8 = undefined; 194 _ = try std.fmt.hexToBytes(&current_event_id, &event.id); 195 196 // create a new views map for the current event we are consuming 197 const views_cursor = try haxy_moment.putCursor(hash.hashInt(hash_kind, "views")); 198 const views = try DB.HashMap(.read_write).init(views_cursor); 199 200 switch (event.data) { 201 .issue => |data| { 202 const event_id_to_issue_cursor = try views.putCursor(hash.hashInt(hash_kind, "event-id->issue")); 203 const event_id_to_issue = try DB.HashMap(.read_write).init(event_id_to_issue_cursor); 204 205 const issue_cursor = try event_id_to_issue.putCursor(hash.hashInt(hash_kind, &current_event_id)); 206 const issue = try DB.HashMap(.read_write).init(issue_cursor); 207 208 try upsert(DB, hash_kind, issue, @TypeOf(data), data); 209 }, 210 } 211 } 212 213 // the current object id is now the last one 214 last_object_id_maybe = repo_event.oid; 215 216 // prevent any of the data created above from being mutated by future iterations of this loop 217 try cursor.db.freeze(); 218 } 219 220 if (last_object_id_maybe) |*last_object_id| { 221 try moment.put(hash.hashInt(hash_kind, "haxy-last-object-id"), .{ .bytes = last_object_id }); 222 } 223 } 224 }; 225 226 try db_file.lock(io, .exclusive); 227 defer db_file.unlock(io); 228 229 // create a new transaction in the database that runs the above-defined Ctx function 230 const history = try DB.ArrayList(.read_write).init(db.rootCursor()); 231 try history.appendContext(.{ .slot = try history.getSlot(-1) }, Ctx{ .repo_events = repo_events }); 232 } 233 234 fn upsert( 235 comptime DB: type, 236 comptime hash_kind: hash.HashKind, 237 map: DB.HashMap(.read_write), 238 comptime Data: type, 239 data: Data, 240 ) !void { 241 switch (@typeInfo(Data)) { 242 .@"struct" => |struct_info| { 243 inline for (struct_info.fields) |field| { 244 try upsertField(DB, hash_kind, map, field.name, field.type, @field(data, field.name)); 245 } 246 }, 247 else => @compileError("upsert expects a struct"), 248 } 249 } 250 251 fn upsertField( 252 comptime DB: type, 253 comptime hash_kind: hash.HashKind, 254 map: DB.HashMap(.read_write), 255 comptime field_name: []const u8, 256 comptime Field: type, 257 value: Field, 258 ) !void { 259 const key = hash.hashInt(hash_kind, field_name); 260 261 switch (@typeInfo(Field)) { 262 .pointer => |pointer_info| { 263 if (pointer_info.size == .slice and pointer_info.child == u8) { 264 try upsertBytes(DB, hash_kind, map, key, value); 265 } else { 266 @compileError("unsupported upsert field type: " ++ @typeName(Field)); 267 } 268 }, 269 .array => |array_info| { 270 if (array_info.child == u8) { 271 try upsertBytes(DB, hash_kind, map, key, &value); 272 } else { 273 @compileError("unsupported upsert field type: " ++ @typeName(Field)); 274 } 275 }, 276 .int => |int_info| switch (int_info.signedness) { 277 .unsigned => { 278 if (try map.getCursor(key)) |value_cursor| { 279 if (try value_cursor.readUint() == value) { 280 return; 281 } 282 } 283 284 try map.put(key, .{ .uint = value }); 285 }, 286 .signed => { 287 if (try map.getCursor(key)) |value_cursor| { 288 if (try value_cursor.readInt() == value) { 289 return; 290 } 291 } 292 293 try map.put(key, .{ .int = value }); 294 }, 295 }, 296 else => @compileError("unsupported upsert field type: " ++ @typeName(Field)), 297 } 298 } 299 300 fn upsertBytes( 301 comptime DB: type, 302 comptime hash_kind: hash.HashKind, 303 map: DB.HashMap(.read_write), 304 key: hash.HashInt(hash_kind), 305 value: []const u8, 306 ) !void { 307 var existing_cursor_maybe = try map.getCursor(key); 308 if (existing_cursor_maybe) |*existing_cursor| { 309 if (try bytesEqual(DB, existing_cursor, value)) { 310 return; 311 } 312 } 313 314 var value_cursor = try map.putCursor(key); 315 var write_buffer: [1024]u8 = undefined; 316 var writer = try value_cursor.writer(&write_buffer); 317 try writer.interface.writeAll(value); 318 try writer.finish(); 319 } 320 321 fn bytesEqual( 322 comptime DB: type, 323 cursor: *DB.Cursor(.read_only), 324 value: []const u8, 325 ) !bool { 326 var read_buffer: [1024]u8 = undefined; 327 var reader = try cursor.reader(&read_buffer); 328 if (reader.size != value.len) { 329 return false; 330 } 331 332 var chunk_buffer: [1024]u8 = undefined; 333 var offset: usize = 0; 334 while (offset < value.len) { 335 const chunk_len = @min(chunk_buffer.len, value.len - offset); 336 try reader.interface.readSliceAll(chunk_buffer[0..chunk_len]); 337 if (!std.mem.eql(u8, chunk_buffer[0..chunk_len], value[offset .. offset + chunk_len])) { 338 return false; 339 } 340 offset += chunk_len; 341 } 342 343 return true; 344 } 345