HAXY

▄ ▄▀▀▀▀ ▀ ▄▀▀▀▀▀ ▄▀▀▀ ▄▀▀▀▄▀▀ ▄▄▀▀▀▀▄▀ ▄▀▀▄▀▀▀▀ ▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀ ▄ ▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀ ▀▄▄▀▄▀▀▀▀ ▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀ ▄ ▀▀▀▄▀▀▀▀▀▀▀ ▀▀▀▀▀ ▄▀▀▀▀▀▀▀▀▀▄▀▀▄ ▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▄ ▀▀▄ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▄▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▄▄▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀▄▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀▀ ▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀ ▀▀▀▀▀▀▄▀▀▀▀▀▀▀▀▀▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▄▀▄ ▀▀ ▀▀▀▀▀▀▄▀▀▀▀▀▄▄▀▀▀▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▄▀▄▄▀▀▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀▀ ▀▀ ▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▀ ▀ ▀▀▀▀▀▀▀▄▄▀▀▀▀▀▀▀▀▀▀▀ ▀▀ ▀▀▀▀▀▀▀▄▄▀▀▀▄▄▀▄▀▀▀▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▄▄▀▀▀ ▀▀ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄▀▀▄▀▀▄ ▀▀▄▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▄ ▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ ▀▀▀▀

╭ event.zig ────────────────────────────────────────────────────╮ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ╰───────────────────────────────────────────────────────────────╯

╭────╮ ╭ clone ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╮ │http│ ssh │ │ ╰────╯ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╯

.. test/ command.zig ╭─────────────────────────────────╮ │event.zig │ ╰─────────────────────────────────╯ lib.zig main.zig serve.zig ssh_helper.zig test.zig testnet.zig

1 const std = @import("std"); 2 const xit = @import("xit"); 3 const rp = xit.repo; 4 const hash = xit.hash; 5 const mrg = xit.merge; 6 7 pub const event_id_size: usize = 32; 8 9 pub const EventKind = enum { 10 issue, 11 }; 12 13 pub const EventData = union(EventKind) { 14 issue: struct { 15 title: []const u8, 16 description: []const u8, 17 tags: []const u8, 18 }, 19 20 pub fn read( 21 comptime DB: type, 22 comptime hash_kind: hash.HashKind, 23 allocator: std.mem.Allocator, 24 map: DB.HashMap(.read_only), 25 kind: EventKind, 26 ) !EventData { 27 return switch (kind) { 28 .issue => .{ 29 .issue = .{ 30 .title = try readBytes(DB, hash_kind, allocator, map, "title"), 31 .description = try readBytes(DB, hash_kind, allocator, map, "description"), 32 .tags = try readBytes(DB, hash_kind, allocator, map, "tags"), 33 }, 34 }, 35 }; 36 } 37 38 fn readBytes( 39 comptime DB: type, 40 comptime hash_kind: hash.HashKind, 41 allocator: std.mem.Allocator, 42 map: DB.HashMap(.read_only), 43 field_name: []const u8, 44 ) ![]const u8 { 45 const cursor = try map.getCursor(hash.hashInt(hash_kind, field_name)) orelse return error.NotFound; 46 return try cursor.readBytesAlloc(allocator, null); 47 } 48 }; 49 50 pub const Event = struct { 51 id: [event_id_size * 2]u8, 52 data: EventData, 53 }; 54 55 /// associates an Event with a git object id. 56 /// we save this oid in the database so we can reliably re-consume 57 /// events if their oid changed. this can happen if the branch 58 /// is rebased and force-pushed. 59 pub fn RepoEvent(comptime hash_kind: hash.HashKind) type { 60 return struct { 61 parent_oid: ?[hash.byteLen(hash_kind)]u8, 62 oid: [hash.byteLen(hash_kind)]u8, 63 event: ?Event, // if null, this is a merge commit 64 }; 65 } 66 67 pub fn randomId(random: std.Random) [event_id_size]u8 { 68 var id_bytes: [event_id_size]u8 = undefined; 69 random.bytes(&id_bytes); 70 return id_bytes; 71 } 72 73 pub fn consume( 74 comptime repo_opts: rp.RepoOpts(.xit), 75 io: std.Io, 76 allocator: std.mem.Allocator, 77 repo: *rp.Repo(.xit, repo_opts), 78 repo_events: []RepoEvent(repo_opts.hash), 79 ) !void { 80 const DB = rp.Repo(.xit, repo_opts).DB; 81 82 const Ctx = struct { 83 io: std.Io, 84 allocator: std.mem.Allocator, 85 repo: *rp.Repo(.xit, repo_opts), 86 repo_events: []RepoEvent(repo_opts.hash), 87 88 pub fn run(ctx: @This(), cursor: *DB.Cursor(.read_write)) !void { 89 const moment = try DB.HashMap(.read_write).init(cursor.*); 90 91 // the last_object_id represents the object id that was last consumed 92 var last_object_id_maybe: ?[hash.byteLen(repo_opts.hash)]u8 = null; 93 if (try moment.getCursor(hash.hashInt(repo_opts.hash, "haxy-last-object-id"))) |last_object_id_cursor| { 94 var last_object_id_buffer: [hash.byteLen(repo_opts.hash)]u8 = undefined; 95 _ = try last_object_id_cursor.readBytes(&last_object_id_buffer); 96 last_object_id_maybe = last_object_id_buffer; 97 } 98 99 // the list with all of haxy's state, including materialized views. 100 // the reason it is a list is so we can keep every previous haxy 101 // state, making it easy to revert to an older state if the user 102 // rebases some of the past commits. 103 const haxy_cursor = try moment.putCursor(hash.hashInt(repo_opts.hash, "haxy")); 104 const haxy = try DB.ArrayList(.read_write).init(haxy_cursor); 105 106 // add a new item to the haxy list created above. 107 // we call the item haxy_moments. it is a map of object id to haxy moment. 108 // in other words, it maps each object id to a hash map containing the 109 // state that the database was in when that object id was consumed. 110 var haxy_moments_cursor = try haxy.appendCursor(); 111 // use the previous haxy_moments as the basis for this one if it exists 112 if (try haxy.getCursor(-2)) |last_haxy_moments_cursor| { 113 try haxy_moments_cursor.write(.{ .slot = last_haxy_moments_cursor.slot() }); 114 } 115 var haxy_moments = try DB.HashMap(.read_write).init(haxy_moments_cursor); 116 117 // if this branch was rebased and force pushed, we need to detect that and 118 // properly revert the haxy state to the last valid state. we detect this 119 // by simply asking if the last event is a descendent of the last event 120 // we consumed. if it isn't, then we know the haxy state needs to be reverted. 121 // from that point on, the rest of the events should be descendents so we no 122 // longer need to even check. this boolean lets us know if that has happened. 123 var checked_for_rebase = false; 124 125 // iterate over the events in reverse order 126 // (they are sorted by most recent events first) 127 for (0..ctx.repo_events.len) |i| { 128 const repo_event = ctx.repo_events[ctx.repo_events.len - i - 1]; 129 130 // if this object id has already been consumed, skip it 131 if (null != try haxy_moments.getCursor(hash.bytesToInt(repo_opts.hash, &repo_event.oid))) { 132 continue; 133 } 134 135 if (!checked_for_rebase) { 136 checked_for_rebase = true; 137 138 if (last_object_id_maybe) |*last_object_id| { 139 var moment_read_only = moment.readOnly(); 140 const state = rp.Repo(.xit, repo_opts).State(.read_only){ .core = &ctx.repo.core, .extra = .{ .moment = &moment_read_only } }; 141 142 // if the last event isn't a descendent of the last event that 143 // we consumed, the branch was rebased so we need to revert the 144 // haxy state back to where it was when this event's parent was 145 // consumed. from that point on, we can consume the rest of the 146 // events on top of that state. 147 _ = mrg.getDescendent( 148 .xit, 149 repo_opts, 150 state, 151 ctx.io, 152 ctx.allocator, 153 &std.fmt.bytesToHex(last_object_id, .lower), 154 &std.fmt.bytesToHex(ctx.repo_events[0].oid, .lower), 155 ) catch |err| switch (err) { 156 error.DescendentNotFound => { 157 if (repo_event.parent_oid) |*parent_oid| { 158 const old_moment_cursor = try haxy_moments.getCursor(hash.bytesToInt(repo_opts.hash, parent_oid)) orelse return error.CursorNotFound; 159 const old_moment = try DB.HashMap(.read_only).init(old_moment_cursor); 160 161 const old_moment_index_cursor = try old_moment.getCursor(hash.hashInt(repo_opts.hash, "moment-index")) orelse return error.CursorNotFound; 162 const old_moment_index = try old_moment_index_cursor.readUint(); 163 164 // resize the haxy list so we truncate all the moments that were 165 // created after the parent_oid was consumed 166 try haxy.slice(old_moment_index + 1); 167 168 // make a new haxy moment and set its initial value to the last haxy moment 169 haxy_moments_cursor = try haxy.appendCursor(); 170 const old_haxy_moments_cursor = try haxy.getCursor(old_moment_index) orelse return error.CursorNotFound; 171 try haxy_moments_cursor.write(.{ .slot = old_haxy_moments_cursor.slot() }); 172 haxy_moments = try DB.HashMap(.read_write).init(haxy_moments_cursor); 173 174 last_object_id_maybe = parent_oid.*; 175 } else { 176 // the branch was rebased all the way to the very beginning. 177 // we have a repo event with no parent, which means it is now 178 // the very first event. all we need to do is set the haxy list 179 // to be empty and make a new haxy_moments map to work with. 180 181 try haxy.slice(0); 182 haxy_moments_cursor = try haxy.appendCursor(); 183 haxy_moments = try DB.HashMap(.read_write).init(haxy_moments_cursor); 184 185 last_object_id_maybe = null; 186 } 187 }, 188 else => |e| return e, 189 }; 190 } else { 191 if (repo_event.parent_oid) |_| { 192 // there is no last_object_id, but this event has a parent. 193 // this is an invalid state. if the event has a parent, that 194 // implies that an event has already been processed, but 195 // if that's true then there would be a last_object_id. 196 return error.UnexpectedParent; 197 } 198 } 199 } 200 201 // create a moment for this object id 202 var haxy_moment_cursor = try haxy_moments.putCursor(hash.bytesToInt(repo_opts.hash, &repo_event.oid)); 203 204 // if there was a previous object id, make this haxy moment's initial value to it. 205 // this efficiently "clones" the map so we make further modifications based on it. 206 if (last_object_id_maybe) |*last_object_id| { 207 if (try haxy_moments.getCursor(hash.bytesToInt(repo_opts.hash, last_object_id))) |last_haxy_moment_cursor| { 208 try haxy_moment_cursor.write(.{ .slot = last_haxy_moment_cursor.slot() }); 209 } 210 } 211 212 const haxy_moment = try DB.HashMap(.read_write).init(haxy_moment_cursor); 213 214 // associate this moment with the index it will first appear at in the haxy list. 215 // this will be important later so we can truncate that list if the user ever 216 // rebases starting at this object id. 217 try haxy_moment.put(hash.hashInt(repo_opts.hash, "moment-index"), .{ .uint = try haxy.count() - 1 }); 218 219 // consume the event into the views map. 220 // if the event is null, it was just a merge commit 221 // so we don't need to update any views. 222 if (repo_event.event) |event| { 223 // get the id of the current event as bytes 224 var current_event_id: [event_id_size]u8 = undefined; 225 _ = try std.fmt.hexToBytes(&current_event_id, &event.id); 226 227 switch (event.data) { 228 .issue => |data| { 229 const event_id_to_issue_cursor = try haxy_moment.putCursor(hash.hashInt(repo_opts.hash, "event-id->issue")); 230 const event_id_to_issue = try DB.HashMap(.read_write).init(event_id_to_issue_cursor); 231 232 const issue_cursor = try event_id_to_issue.putCursor(hash.hashInt(repo_opts.hash, &current_event_id)); 233 const issue = try DB.HashMap(.read_write).init(issue_cursor); 234 235 try upsert(DB, repo_opts.hash, issue, @TypeOf(data), data); 236 }, 237 } 238 } 239 240 // the current object id is now the last one 241 last_object_id_maybe = repo_event.oid; 242 243 // prevent any of the data created above from being mutated by future iterations of this loop 244 try cursor.db.freeze(); 245 } 246 247 if (last_object_id_maybe) |*last_object_id| { 248 try moment.put(hash.hashInt(repo_opts.hash, "haxy-last-object-id"), .{ .bytes = last_object_id }); 249 } 250 } 251 }; 252 253 try repo.core.db_file.lock(io, .exclusive); 254 defer repo.core.db_file.unlock(io); 255 256 // create a new transaction in the database that runs the above-defined Ctx function 257 const history = try DB.ArrayList(.read_write).init(repo.core.db.rootCursor()); 258 try history.appendContext(.{ .slot = try history.getSlot(-1) }, Ctx{ 259 .io = io, 260 .allocator = allocator, 261 .repo = repo, 262 .repo_events = repo_events, 263 }); 264 } 265 266 fn upsert( 267 comptime DB: type, 268 comptime hash_kind: hash.HashKind, 269 map: DB.HashMap(.read_write), 270 comptime Data: type, 271 data: Data, 272 ) !void { 273 switch (@typeInfo(Data)) { 274 .@"struct" => |struct_info| { 275 inline for (struct_info.fields) |field| { 276 try upsertField(DB, hash_kind, map, field.name, field.type, @field(data, field.name)); 277 } 278 }, 279 else => @compileError("upsert expects a struct"), 280 } 281 } 282 283 fn upsertField( 284 comptime DB: type, 285 comptime hash_kind: hash.HashKind, 286 map: DB.HashMap(.read_write), 287 comptime field_name: []const u8, 288 comptime Field: type, 289 value: Field, 290 ) !void { 291 const key = hash.hashInt(hash_kind, field_name); 292 293 switch (@typeInfo(Field)) { 294 .pointer => |pointer_info| { 295 if (pointer_info.size == .slice and pointer_info.child == u8) { 296 try upsertBytes(DB, hash_kind, map, key, value); 297 } else { 298 @compileError("unsupported upsert field type: " ++ @typeName(Field)); 299 } 300 }, 301 .array => |array_info| { 302 if (array_info.child == u8) { 303 try upsertBytes(DB, hash_kind, map, key, &value); 304 } else { 305 @compileError("unsupported upsert field type: " ++ @typeName(Field)); 306 } 307 }, 308 .int => |int_info| switch (int_info.signedness) { 309 .unsigned => { 310 if (try map.getCursor(key)) |value_cursor| { 311 if (try value_cursor.readUint() == value) { 312 return; 313 } 314 } 315 316 try map.put(key, .{ .uint = value }); 317 }, 318 .signed => { 319 if (try map.getCursor(key)) |value_cursor| { 320 if (try value_cursor.readInt() == value) { 321 return; 322 } 323 } 324 325 try map.put(key, .{ .int = value }); 326 }, 327 }, 328 else => @compileError("unsupported upsert field type: " ++ @typeName(Field)), 329 } 330 } 331 332 fn upsertBytes( 333 comptime DB: type, 334 comptime hash_kind: hash.HashKind, 335 map: DB.HashMap(.read_write), 336 key: hash.HashInt(hash_kind), 337 value: []const u8, 338 ) !void { 339 var existing_cursor_maybe = try map.getCursor(key); 340 if (existing_cursor_maybe) |*existing_cursor| { 341 if (try bytesEqual(DB, existing_cursor, value)) { 342 return; 343 } 344 } 345 346 var value_cursor = try map.putCursor(key); 347 var write_buffer: [1024]u8 = undefined; 348 var writer = try value_cursor.writer(&write_buffer); 349 try writer.interface.writeAll(value); 350 try writer.finish(); 351 } 352 353 fn bytesEqual( 354 comptime DB: type, 355 cursor: *DB.Cursor(.read_only), 356 value: []const u8, 357 ) !bool { 358 var read_buffer: [1024]u8 = undefined; 359 var reader = try cursor.reader(&read_buffer); 360 if (reader.size != value.len) { 361 return false; 362 } 363 364 var chunk_buffer: [1024]u8 = undefined; 365 var offset: usize = 0; 366 while (offset < value.len) { 367 const chunk_len = @min(chunk_buffer.len, value.len - offset); 368 try reader.interface.readSliceAll(chunk_buffer[0..chunk_len]); 369 if (!std.mem.eql(u8, chunk_buffer[0..chunk_len], value[offset .. offset + chunk_len])) { 370 return false; 371 } 372 offset += chunk_len; 373 } 374 375 return true; 376 } 377