╭─────────────╮ │scroll to top│ ╰─────────────╯ ╭ event.zig ────────────────────────────────────────────────────╮ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ╰───────────────────────────────────────────────────────────────╯
🬐 🬠 🬞🬂🬏 🬟 🬑 🬟 🬑 ╭03fd8b80..╮ 03fd8b80.. see-em: 🬋🬋🬋 🬋🬋🬋 🬋 🬋 │files │ commits refs discuss issues new login 🬐 🬠 🬐 🬠 🬑 🬟 🬰 ╰──────────╯ (55) user
╭────╮ ╭ clone ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╮ │http│ ssh │ │ ╰────╯ ╰ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ╯
.. event/ test/ command.zig ╭─────────────────────────────────╮ │event.zig │ ╰─────────────────────────────────╯ lib.zig main.zig serve.zig ssh_helper.zig test.zig testnet.zig
464 } 465 } 466 467 fn mergeChangedMapEntries( 468 comptime DB: type, 469 target: DB.HashMap(.read_write), 470 parent: DB.HashMap(.read_only), 471 baseline: DB.HashMap(.read_only), 472 comptime is_top_level: bool, 473 ) !void { 474 var parent_iter = try parent.iterator(); 475 while (try parent_iter.next()) |kv_pair_cursor| { 476 const kv_pair = try kv_pair_cursor.readKeyValuePair(); 477 478 const baseline_value_cursor = try baseline.getCursor(kv_pair.hash) orelse { 479 if ((try target.getCursor(kv_pair.hash)) != null) { 480 return error.MergeConflict; 481 } 482 483 try target.put(kv_pair.hash, .{ .slot = kv_pair.value_cursor.slot() }); 484 continue; 485 }; 486 487 const tag = kv_pair.value_cursor.slot().tag; 488 489 if (tag != baseline_value_cursor.slot().tag) { 490 return error.UnexpectedTag; 491 } 492 493 if (kv_pair.value_cursor.slot().value == baseline_value_cursor.slot().value) { 494 continue; 495 } 496 497 if (tag == .hash_map) { 498 const target_existing_cursor = try target.getCursor(kv_pair.hash) orelse return error.MergeConflict; 499 if (target_existing_cursor.slot().tag != .hash_map) { 500 return error.MergeConflict; 501 } 502 503 const target_child_cursor = try target.putCursor(kv_pair.hash); 504 const target_child = try DB.HashMap(.read_write).init(target_child_cursor); 505 const parent_child = try DB.HashMap(.read_only).init(kv_pair.value_cursor); 506 const baseline_child = try DB.HashMap(.read_only).init(baseline_value_cursor); 507 try mergeChangedMapEntries(DB, target_child, parent_child, baseline_child, false); 508 } else if (is_top_level) { 509 // the moment-index is a top-level key in the haxy moment 510 // and is not a map. we don't want to do any merge of this. 511 continue; 512 } else { 513 const target_value_cursor = try target.getCursor(kv_pair.hash) orelse return error.MergeConflict; 514 if (target_value_cursor.slot().value != baseline_value_cursor.slot().value) { 515 return error.MergeConflict; 516 } 517 518 try target.put(kv_pair.hash, .{ .slot = kv_pair.value_cursor.slot() }); 519 } 520 } 521 } 522 523 fn upsertField( 524 comptime DB: type, 525 comptime hash_kind: hash.HashKind, 526 map: DB.HashMap(.read_write), 527 comptime field_name: []const u8, 528 comptime Field: type, 529 value: Field, 530 ) !void { 531 const key = hash.hashInt(hash_kind, field_name); 532 533 switch (@typeInfo(Field)) { 534 .pointer => |pointer_info| { 535 if (pointer_info.size == .slice and pointer_info.child == u8) { 536 try upsertBytes(DB, hash_kind, map, key, value); 537 } else { 538 @compileError("unsupported upsert field type: " ++ @typeName(Field)); 539 } 540 }, 541 .array => |array_info| { 542 if (array_info.child == u8) { 543 try upsertBytes(DB, hash_kind, map, key, &value); 544 } else { 545 @compileError("unsupported upsert field type: " ++ @typeName(Field)); 546 } 547 }, 548 .int => |int_info| switch (int_info.signedness) { 549 .unsigned => { 550 if (try map.getCursor(key)) |value_cursor| { 551 if (try value_cursor.readUint() == value) { 552 return; 553 } 554 } 555 556 try map.put(key, .{ .uint = value }); 557 }, 558 .signed => { 559 if (try map.getCursor(key)) |value_cursor| { 560 if (try value_cursor.readInt() == value) { 561 return; 562 } 563 } 564 565 try map.put(key, .{ .int = value }); 566 }, 567 }, 568 .bool => { 569 const bytes = if (value) "true" else "false"; 570 if (try map.getCursor(key)) |value_cursor| { 571 var buffer: [5]u8 = undefined; 572 const existing = try value_cursor.readBytesObject(&buffer); 573 if (existing.format_tag) |format_tag| { 574 if (std.mem.eql(u8, &format_tag, "bl") and std.mem.eql(u8, existing.value, bytes)) { 575 return; 576 } 577 } 578 } 579 580 try map.put(key, .{ .bytes_object = .{ .value = bytes, .format_tag = "bl".* } }); 581 }, 582 else => @compileError("unsupported upsert field type: " ++ @typeName(Field)), 583 } 584 } 585 586 fn upsertBytes( 587 comptime DB: type, 588 comptime hash_kind: hash.HashKind, 589 map: DB.HashMap(.read_write), 590 key: hash.HashInt(hash_kind), 591 value: []const u8, 592 ) !void { 593 var existing_cursor_maybe = try map.getCursor(key); 594 if (existing_cursor_maybe) |*existing_cursor| { 595 if (try bytesEqual(DB, existing_cursor, value)) { 596 return; 597 } 598 } 599 600 var value_cursor = try map.putCursor(key); 601 var write_buffer: [1024]u8 = undefined; 602 var writer = try value_cursor.writer(&write_buffer); 603 try writer.interface.writeAll(value); 604 try writer.finish(); 605 } 606 607 fn bytesEqual( 608 comptime DB: type, 609 cursor: *DB.Cursor(.read_only), 610 value: []const u8, 611 ) !bool { 612 var read_buffer: [1024]u8 = undefined; 613 var reader = try cursor.reader(&read_buffer); 614 if (reader.size != value.len) { 615 return false; 616 } 617 618 var chunk_buffer: [1024]u8 = undefined; 619 var offset: usize = 0; 620 while (offset < value.len) { 621 const chunk_len = @min(chunk_buffer.len, value.len - offset); 622 try reader.interface.readSliceAll(chunk_buffer[0..chunk_len]); 623 if (!std.mem.eql(u8, chunk_buffer[0..chunk_len], value[offset .. offset + chunk_len])) { 624 return false; 625 } 626 offset += chunk_len; 627 } 628 629 return true; 630 } 631 632 // 633 // CommitIterator 634 // 635 // differs from xit's ObjectIterator in a few ways: 636 // 637 // 1. iterates parents first, rather than children first 638 // 2. `next` only returns the object id, not the full Object 639 // 3. stores its temporary state in a file on disk to avoid OOMs 640 // 641 642 pub fn CommitIterator(comptime repo_opts: rp.RepoOpts(.xit)) type { 643 return struct { 644 const DB = rp.Repo(.xit, repo_opts).DB; 645 const db_name = "haxy-repo-events.db"; 646 647 repo_dir: std.Io.Dir, 648 db_file: std.Io.File, 649 db: *DB, 650 // map of each object id to its children 651 parent_to_children: DB.HashMap(.read_write), 652 // tracking pending parents is necessary because a child can only 653 // be returned after all of its unconsumed parents have been returned 654 child_to_pending_parents: DB.HashMap(.read_write), 655 // queue of object ids that are ready to be returned from `next` 656 ready_queue: DB.ArrayList(.read_write), 657 ready_queue_index: u64, 658 // the object id at the tip of this branch 659 newest_object_id: ?[hash.byteLen(repo_opts.hash)]u8, 660 // the object id furthest back in the history that hasn't been consumed 661 oldest_object_id: ?[hash.byteLen(repo_opts.hash)]u8, 662 663 pub fn init( 664 state: rp.Repo(.xit, repo_opts).State(.read_only), 665 io: std.Io, 666 allocator: std.mem.Allocator, 667 consumed_object_ids: DB.HashMap(.read_only), 668 ref: rf.Ref, 669 ) !CommitIterator(repo_opts) { 670 const db_file = try state.core.repo_dir.createFile(io, db_name, .{ .truncate = true, .lock = .exclusive, .read = true }); 671 errdefer { 672 db_file.close(io); 673 state.core.repo_dir.deleteFile(io, db_name) catch {}; 674 } 675 676 const buffer_ptr = try allocator.create(std.Io.Writer.Allocating); 677 errdefer allocator.destroy(buffer_ptr); 678 679 buffer_ptr.* = std.Io.Writer.Allocating.init(allocator); 680 errdefer buffer_ptr.deinit(); 681 682 const db_ptr = try allocator.create(DB); 683 errdefer allocator.destroy(db_ptr); 684 db_ptr.* = try DB.init(.{ .io = io, .file = db_file, .buffer = buffer_ptr }); 685 686 const map = try DB.HashMap(.read_write).init(db_ptr.rootCursor()); 687 688 const parent_to_children_cursor = try map.putCursor(hash.hashInt(repo_opts.hash, "parent->children")); 689 const parent_to_children = try DB.HashMap(.read_write).init(parent_to_children_cursor); 690 691 const child_to_pending_parents_cursor = try map.putCursor(hash.hashInt(repo_opts.hash, "child->pending-parents")); 692 const child_to_pending_parents = try DB.HashMap(.read_write).init(child_to_pending_parents_cursor); 693 694 const ready_queue_cursor = try map.putCursor(hash.hashInt(repo_opts.hash, "ready-queue")); 695 const ready_queue = try DB.ArrayList(.read_write).init(ready_queue_cursor); 696 697 var self = CommitIterator(repo_opts){ 698 .repo_dir = state.core.repo_dir, 699 .db_file = db_file, 700 .db = db_ptr, 701 .parent_to_children = parent_to_children, 702 .child_to_pending_parents = child_to_pending_parents, 703 .ready_queue = ready_queue, 704 .ready_queue_index = 0, 705 .newest_object_id = null, 706 .oldest_object_id = null, 707 }; 708 errdefer self.deinit(io, allocator); 709 710 try self.collect(state, io, allocator, consumed_object_ids, ref); 711 712 return self; 713 } 714 715 pub fn deinit(self: *CommitIterator(repo_opts), io: std.Io, allocator: std.mem.Allocator) void { 716 self.db_file.close(io); 717 self.db.core.memory.buffer.deinit(); 718 allocator.destroy(self.db.core.memory.buffer); 719 self.repo_dir.deleteFile(io, db_name) catch {}; 720 allocator.destroy(self.db); 721 } 722 723 pub fn next(self: *CommitIterator(repo_opts)) !?[hash.byteLen(repo_opts.hash)]u8 { 724 if (self.ready_queue_index >= try self.ready_queue.count()) return null; 725 726 const oid = try readOidFromList(self.ready_queue.readOnly(), self.ready_queue_index); 727 self.ready_queue_index += 1; 728 729 try self.enqueueChildren(&oid); 730 return oid; 731 } 732 733 fn collect( 734 self: *CommitIterator(repo_opts), 735 state: rp.Repo(.xit, repo_opts).State(.read_only), 736 io: std.Io, 737 allocator: std.mem.Allocator, 738 consumed_object_ids: DB.HashMap(.read_only), 739 ref: rf.Ref, 740 ) !void { 741 const map = try DB.HashMap(.read_write).init(self.db.rootCursor()); 742 743 const walk_queue_cursor = try map.putCursor(hash.hashInt(repo_opts.hash, "walk-queue")); 744 const walk_queue = try DB.ArrayList(.read_write).init(walk_queue_cursor); 745 var walk_queue_index: u64 = 0; 746 747 const seen_object_ids_cursor = try map.putCursor(hash.hashInt(repo_opts.hash, "seen-object-ids")); 748 const seen_object_ids = try DB.HashSet(.read_write).init(seen_object_ids_cursor); 749 750 const head_oid_hex = (try rf.readRecur(.xit, repo_opts, state, io, .{ .ref = ref })) orelse return error.OidNotFound; 751 var head_oid: [hash.byteLen(repo_opts.hash)]u8 = undefined; 752 _ = try std.fmt.hexToBytes(&head_oid, &head_oid_hex); 753 try walk_queue.append(.{ .bytes = &head_oid }); 754 755 while (walk_queue_index < try walk_queue.count()) { 756 const oid = try readOidFromList(walk_queue.readOnly(), walk_queue_index); 757 const oid_int = hash.bytesToInt(repo_opts.hash, &oid); 758 759 walk_queue_index += 1; 760 761 // if we've seen this object id, skip it 762 if (null != try seen_object_ids.getCursor(oid_int)) { 763 continue; 764 } 765 766 try seen_object_ids.put(oid_int, .{ .bytes = &oid }); 767 768 var commit_object = try obj.Object(.xit, repo_opts, .full).init(state, io, allocator, &std.fmt.bytesToHex(oid, .lower)); 769 defer commit_object.deinit(); 770 771 // if this object id has already been consumed, skip it 772 if (null != try consumed_object_ids.getCursor(oid_int)) { 773 continue; 774 } 775 776 if (self.newest_object_id == null) self.newest_object_id = oid; 777 self.oldest_object_id = oid; 778 779 const parent_oids = commit_object.content.commit.metadata.parent_oids orelse return error.ParentOidsNotFound; 780 var pending_parent_count: u64 = 0; 781 for (parent_oids) |*parent_oid| { 782 var parent_oid_bytes: [hash.byteLen(repo_opts.hash)]u8 = undefined; 783 _ = try std.fmt.hexToBytes(&parent_oid_bytes, parent_oid); 784 const parent_oid_int = hash.bytesToInt(repo_opts.hash, &parent_oid_bytes); 785 786 // if this object id has already been consumed, skip it 787 if (null != try consumed_object_ids.getCursor(parent_oid_int)) { 788 continue; 789 } 790 791 // if this object id hasn't already been seen, add it to the walk queue 792 if (null == try seen_object_ids.getCursor(parent_oid_int)) { 793 try walk_queue.append(.{ .bytes = &parent_oid_bytes }); 794 } 795 796 const children_cursor = try self.parent_to_children.putCursor(parent_oid_int); 797 const children = try DB.HashSet(.read_write).init(children_cursor); 798 try children.put(oid_int, .{ .bytes = &oid }); 799 pending_parent_count += 1; 800 } 801 802 if (pending_parent_count == 0) { 803 try self.ready_queue.append(.{ .bytes = &oid }); 804 } else { 805 try self.child_to_pending_parents.put(oid_int, .{ .uint = pending_parent_count }); 806 } 807 } 808 } 809 810 fn enqueueChildren(self: *CommitIterator(repo_opts), oid: *const [hash.byteLen(repo_opts.hash)]u8) !void { 811 const oid_int = hash.bytesToInt(repo_opts.hash, oid); 812 const children_cursor = try self.parent_to_children.getCursor(oid_int) orelse return; 813 const children = try DB.HashSet(.read_only).init(children_cursor); 814 var children_iter = try children.iterator(); 815 816 while (try children_iter.next()) |*child_cursor| { 817 const kv_pair = try child_cursor.readKeyValuePair(); 818 const child_oid_int = kv_pair.hash; 819 const pending_parent_cursor = try self.child_to_pending_parents.getCursor(child_oid_int) orelse return error.CursorNotFound; 820 const pending_parent_count = try pending_parent_cursor.readUint(); 821 822 if (pending_parent_count <= 1) { 823 _ = try self.child_to_pending_parents.remove(child_oid_int); 824 const child_oid = hash.intToBytes(hash.HashInt(repo_opts.hash), child_oid_int); 825 try self.ready_queue.append(.{ .bytes = &child_oid }); 826 } else { 827 try self.child_to_pending_parents.put(child_oid_int, .{ .uint = pending_parent_count - 1 }); 828 } 829 } 830 } 831 832 fn readOidFromList(list: DB.ArrayList(.read_only), index: u64) ![hash.byteLen(repo_opts.hash)]u8 { 833 const oid_cursor = try list.getCursor(index) orelse return error.CursorNotFound; 834 var oid: [hash.byteLen(repo_opts.hash)]u8 = undefined; 835 _ = try oid_cursor.readBytes(&oid); 836 return oid; 837 } 838 }; 839 } 840