+9 -3

ufos/src/lib.rs

Reviewed

··· 277 277 }, 278 278 } 279 279 280 280 - #[derive(Debug, Serialize, JsonSchema)] 280 280 + #[derive(Debug, PartialEq, Serialize, JsonSchema)] 281 281 pub struct NsidCount { 282 282 nsid: String, 283 283 creates: u64, ··· 285 285 dids_estimate: u64, 286 286 } 287 287 288 288 - #[derive(Debug, Serialize, JsonSchema)] 288 288 + #[derive(Debug, PartialEq, Serialize, JsonSchema)] 289 289 pub struct PrefixCount { 290 290 prefix: String, 291 291 creates: u64, ··· 293 293 dids_estimate: u64, 294 294 } 295 295 296 296 - #[derive(Debug, Serialize, JsonSchema)] 296 296 + #[derive(Debug, PartialEq, Serialize, JsonSchema)] 297 297 #[serde(tag = "type", rename_all = "camelCase")] 298 298 pub enum PrefixChild { 299 299 Collection(NsidCount), ··· 303 303 #[derive(Debug, Serialize, JsonSchema)] 304 304 pub struct NsidPrefix(String); 305 305 impl NsidPrefix { 306 306 + /// Input must not include a trailing dot. 306 307 pub fn new(pre: &str) -> EncodingResult<Self> { 307 308 // it's a valid prefix if appending `.name` makes it a valid NSID 308 309 Nsid::new(format!("{pre}.name")).map_err(EncodingError::BadAtriumStringType)?; ··· 319 320 ); 320 321 self.0 == other.domain_authority() 321 322 } 323 323 + /// The prefix as initialized (no trailing dot) 322 324 pub fn as_str(&self) -> &str { 323 325 self.0.as_str() 326 326 + } 327 327 + /// The prefix with a trailing `.` appended to avoid matching a longer segment 328 328 + pub fn terminated(&self) -> String { 329 329 + format!("{}.", self.0) 324 330 } 325 331 } 326 332

+346 -14

ufos/src/storage_fjall.rs

Reviewed

··· 665 665 cursor: Option<Vec<u8>>, 666 666 buckets: Vec<CursorBucket>, 667 667 ) -> StorageResult<(JustCount, Vec<PrefixChild>, Option<Vec<u8>>)> { 668 668 - let prefix_sub = String::sub_prefix(prefix.as_str())?; 668 668 + // let prefix_sub_with_null = prefix.as_str().to_string().to_db_bytes()?; 669 669 + let prefix_sub = String::sub_prefix(&prefix.terminated())?; // with trailing dot to ensure full segment match 669 670 let cursor_child = cursor 670 671 .as_deref() 671 672 .map(|encoded_bytes| { 672 673 let decoded: String = db_complete(encoded_bytes)?; 673 673 - let as_sub_prefix = String::sub_prefix(&decoded)?; 674 674 - Ok::<_, EncodingError>(as_sub_prefix) 674 674 + // TODO: write some tests for cursors, there's probably bugs here 675 675 + let as_sub_prefix_with_null = decoded.to_db_bytes()?; 676 676 + Ok::<_, EncodingError>(as_sub_prefix_with_null) 675 677 }) 676 678 .transpose()?; 677 679 let mut iters: Vec<NsidCounter> = Vec::with_capacity(buckets.len()); ··· 709 711 let mut iters: Vec<_> = iters 710 712 .into_iter() 711 713 .map(|it| { 712 712 - it.map(|bla| bla.map(|(nsid, v)| (Child::from_prefix(&nsid, &prefix), v))) 713 713 - .peekable() 714 714 + it.map(|bla| { 715 715 + bla.map(|(nsid, v)| { 716 716 + let Some(child) = Child::from_prefix(&nsid, &prefix) else { 717 717 + panic!("failed from_prefix: {nsid:?} {prefix:?} (bad iter bounds?)"); 718 718 + }; 719 719 + (child, v) 720 720 + }) 721 721 + }) 722 722 + .peekable() 714 723 }) 715 724 .collect(); 716 725 ··· 722 731 ChildPrefix(String), 723 732 } 724 733 impl Child { 725 725 - fn from_prefix(nsid: &Nsid, prefix: &NsidPrefix) -> Self { 734 734 + fn from_prefix(nsid: &Nsid, prefix: &NsidPrefix) -> Option<Self> { 726 735 if prefix.is_group_of(nsid) { 727 727 - Child::FullNsid(nsid.to_string()) 728 728 - } else { 729 729 - let suffix = nsid 730 730 - .as_str() 731 731 - .strip_prefix(&format!("{}.", prefix.0)) 732 732 - .unwrap(); 733 733 - let (segment, _) = suffix.split_once('.').unwrap(); 734 734 - Child::ChildPrefix(format!("{}.{segment}", prefix.0)) 736 736 + return Some(Child::FullNsid(nsid.to_string())); 735 737 } 738 738 + let suffix = nsid.as_str().strip_prefix(&format!("{}.", prefix.0))?; 739 739 + let (segment, _) = suffix.split_once('.').unwrap(); 740 740 + let child_prefix = format!("{}.{segment}", prefix.0); 741 741 + Some(Child::ChildPrefix(child_prefix)) 736 742 } 737 743 fn is_before(&self, other: &Child) -> bool { 738 744 match (self, other) { ··· 2464 2470 let (n, _) = write.step_rollup()?; 2465 2471 assert_eq!(n, 0); 2466 2472 2473 2473 + Ok(()) 2474 2474 + } 2475 2475 + 2476 2476 + #[test] 2477 2477 + fn get_prefix_children_lexi_empty() { 2478 2478 + let (read, _) = fjall_db(); 2479 2479 + let ( 2480 2480 + JustCount { 2481 2481 + creates, 2482 2482 + dids_estimate, 2483 2483 + .. 2484 2484 + }, 2485 2485 + children, 2486 2486 + cursor, 2487 2487 + ) = read 2488 2488 + .get_prefix( 2489 2489 + NsidPrefix::new("aaa.aaa").unwrap(), 2490 2490 + 10, 2491 2491 + OrderCollectionsBy::Lexi { cursor: None }, 2492 2492 + None, 2493 2493 + None, 2494 2494 + ) 2495 2495 + .unwrap(); 2496 2496 + 2497 2497 + assert_eq!(creates, 0); 2498 2498 + assert_eq!(dids_estimate, 0); 2499 2499 + assert_eq!(children, vec![]); 2500 2500 + assert_eq!(cursor, None); 2501 2501 + } 2502 2502 + 2503 2503 + #[test] 2504 2504 + fn get_prefix_excludes_exact_collection() -> anyhow::Result<()> { 2505 2505 + let (read, mut write) = fjall_db(); 2506 2506 + 2507 2507 + let mut batch = TestBatch::default(); 2508 2508 + batch.create( 2509 2509 + "did:plc:person-a", 2510 2510 + "a.a.a", 2511 2511 + "rkey-aaa", 2512 2512 + "{}", 2513 2513 + Some("rev-aaa"), 2514 2514 + None, 2515 2515 + 10_000, 2516 2516 + ); 2517 2517 + write.insert_batch(batch.batch)?; 2518 2518 + write.step_rollup()?; 2519 2519 + 2520 2520 + let ( 2521 2521 + JustCount { 2522 2522 + creates, 2523 2523 + dids_estimate, 2524 2524 + .. 2525 2525 + }, 2526 2526 + children, 2527 2527 + cursor, 2528 2528 + ) = read.get_prefix( 2529 2529 + NsidPrefix::new("a.a.a").unwrap(), 2530 2530 + 10, 2531 2531 + OrderCollectionsBy::Lexi { cursor: None }, 2532 2532 + None, 2533 2533 + None, 2534 2534 + )?; 2535 2535 + assert_eq!(creates, 0); 2536 2536 + assert_eq!(dids_estimate, 0); 2537 2537 + assert_eq!(children, vec![]); 2538 2538 + assert_eq!(cursor, None); 2539 2539 + Ok(()) 2540 2540 + } 2541 2541 + 2542 2542 + #[test] 2543 2543 + fn get_prefix_excludes_neighbour_collection() -> anyhow::Result<()> { 2544 2544 + let (read, mut write) = fjall_db(); 2545 2545 + 2546 2546 + let mut batch = TestBatch::default(); 2547 2547 + batch.create( 2548 2548 + "did:plc:person-a", 2549 2549 + "a.a.aa", 2550 2550 + "rkey-aaa", 2551 2551 + "{}", 2552 2552 + Some("rev-aaa"), 2553 2553 + None, 2554 2554 + 10_000, 2555 2555 + ); 2556 2556 + write.insert_batch(batch.batch)?; 2557 2557 + write.step_rollup()?; 2558 2558 + 2559 2559 + let ( 2560 2560 + JustCount { 2561 2561 + creates, 2562 2562 + dids_estimate, 2563 2563 + .. 2564 2564 + }, 2565 2565 + children, 2566 2566 + cursor, 2567 2567 + ) = read.get_prefix( 2568 2568 + NsidPrefix::new("a.a.a").unwrap(), 2569 2569 + 10, 2570 2570 + OrderCollectionsBy::Lexi { cursor: None }, 2571 2571 + None, 2572 2572 + None, 2573 2573 + )?; 2574 2574 + assert_eq!(creates, 0); 2575 2575 + assert_eq!(dids_estimate, 0); 2576 2576 + assert_eq!(children, vec![]); 2577 2577 + assert_eq!(cursor, None); 2578 2578 + Ok(()) 2579 2579 + } 2580 2580 + 2581 2581 + #[test] 2582 2582 + fn get_prefix_includes_child_collection() -> anyhow::Result<()> { 2583 2583 + let (read, mut write) = fjall_db(); 2584 2584 + 2585 2585 + let mut batch = TestBatch::default(); 2586 2586 + batch.create( 2587 2587 + "did:plc:person-a", 2588 2588 + "a.a.a", 2589 2589 + "rkey-aaa", 2590 2590 + "{}", 2591 2591 + Some("rev-aaa"), 2592 2592 + None, 2593 2593 + 10_000, 2594 2594 + ); 2595 2595 + write.insert_batch(batch.batch)?; 2596 2596 + write.step_rollup()?; 2597 2597 + 2598 2598 + let ( 2599 2599 + JustCount { 2600 2600 + creates, 2601 2601 + dids_estimate, 2602 2602 + .. 2603 2603 + }, 2604 2604 + children, 2605 2605 + cursor, 2606 2606 + ) = read.get_prefix( 2607 2607 + NsidPrefix::new("a.a").unwrap(), 2608 2608 + 10, 2609 2609 + OrderCollectionsBy::Lexi { cursor: None }, 2610 2610 + None, 2611 2611 + None, 2612 2612 + )?; 2613 2613 + assert_eq!(creates, 1); 2614 2614 + assert_eq!(dids_estimate, 1); 2615 2615 + assert_eq!( 2616 2616 + children, 2617 2617 + vec![PrefixChild::Collection(NsidCount { 2618 2618 + nsid: "a.a.a".to_string(), 2619 2619 + creates: 1, 2620 2620 + dids_estimate: 1 2621 2621 + }),] 2622 2622 + ); 2623 2623 + assert_eq!(cursor, None); 2624 2624 + Ok(()) 2625 2625 + } 2626 2626 + 2627 2627 + #[test] 2628 2628 + fn get_prefix_includes_child_prefix() -> anyhow::Result<()> { 2629 2629 + let (read, mut write) = fjall_db(); 2630 2630 + 2631 2631 + let mut batch = TestBatch::default(); 2632 2632 + batch.create( 2633 2633 + "did:plc:person-a", 2634 2634 + "a.a.a.a", 2635 2635 + "rkey-aaaa", 2636 2636 + "{}", 2637 2637 + Some("rev-aaaa"), 2638 2638 + None, 2639 2639 + 10_000, 2640 2640 + ); 2641 2641 + write.insert_batch(batch.batch)?; 2642 2642 + write.step_rollup()?; 2643 2643 + 2644 2644 + let ( 2645 2645 + JustCount { 2646 2646 + creates, 2647 2647 + dids_estimate, 2648 2648 + .. 2649 2649 + }, 2650 2650 + children, 2651 2651 + cursor, 2652 2652 + ) = read.get_prefix( 2653 2653 + NsidPrefix::new("a.a").unwrap(), 2654 2654 + 10, 2655 2655 + OrderCollectionsBy::Lexi { cursor: None }, 2656 2656 + None, 2657 2657 + None, 2658 2658 + )?; 2659 2659 + assert_eq!(creates, 1); 2660 2660 + assert_eq!(dids_estimate, 1); 2661 2661 + assert_eq!( 2662 2662 + children, 2663 2663 + vec![PrefixChild::Prefix(PrefixCount { 2664 2664 + prefix: "a.a.a".to_string(), 2665 2665 + creates: 1, 2666 2666 + dids_estimate: 1 2667 2667 + }),] 2668 2668 + ); 2669 2669 + assert_eq!(cursor, None); 2670 2670 + Ok(()) 2671 2671 + } 2672 2672 + 2673 2673 + #[test] 2674 2674 + fn get_prefix_merges_child_prefixes() -> anyhow::Result<()> { 2675 2675 + let (read, mut write) = fjall_db(); 2676 2676 + 2677 2677 + let mut batch = TestBatch::default(); 2678 2678 + batch.create( 2679 2679 + "did:plc:person-a", 2680 2680 + "a.a.a.a", 2681 2681 + "rkey-aaaa", 2682 2682 + "{}", 2683 2683 + Some("rev-aaaa"), 2684 2684 + None, 2685 2685 + 10_000, 2686 2686 + ); 2687 2687 + batch.create( 2688 2688 + "did:plc:person-a", 2689 2689 + "a.a.a.b", 2690 2690 + "rkey-aaab", 2691 2691 + "{}", 2692 2692 + Some("rev-aaab"), 2693 2693 + None, 2694 2694 + 10_001, 2695 2695 + ); 2696 2696 + write.insert_batch(batch.batch)?; 2697 2697 + write.step_rollup()?; 2698 2698 + 2699 2699 + let ( 2700 2700 + JustCount { 2701 2701 + creates, 2702 2702 + dids_estimate, 2703 2703 + .. 2704 2704 + }, 2705 2705 + children, 2706 2706 + cursor, 2707 2707 + ) = read.get_prefix( 2708 2708 + NsidPrefix::new("a.a").unwrap(), 2709 2709 + 10, 2710 2710 + OrderCollectionsBy::Lexi { cursor: None }, 2711 2711 + None, 2712 2712 + None, 2713 2713 + )?; 2714 2714 + assert_eq!(creates, 2); 2715 2715 + assert_eq!(dids_estimate, 1); 2716 2716 + assert_eq!( 2717 2717 + children, 2718 2718 + vec![PrefixChild::Prefix(PrefixCount { 2719 2719 + prefix: "a.a.a".to_string(), 2720 2720 + creates: 2, 2721 2721 + dids_estimate: 1 2722 2722 + }),] 2723 2723 + ); 2724 2724 + assert_eq!(cursor, None); 2725 2725 + Ok(()) 2726 2726 + } 2727 2727 + 2728 2728 + #[test] 2729 2729 + fn get_prefix_exact_and_child_and_prefix() -> anyhow::Result<()> { 2730 2730 + let (read, mut write) = fjall_db(); 2731 2731 + 2732 2732 + let mut batch = TestBatch::default(); 2733 2733 + // exact: 2734 2734 + batch.create( 2735 2735 + "did:plc:person-a", 2736 2736 + "a.a.a", 2737 2737 + "rkey-aaa", 2738 2738 + "{}", 2739 2739 + Some("rev-aaa"), 2740 2740 + None, 2741 2741 + 10_000, 2742 2742 + ); 2743 2743 + // child: 2744 2744 + batch.create( 2745 2745 + "did:plc:person-a", 2746 2746 + "a.a.a.a", 2747 2747 + "rkey-aaaa", 2748 2748 + "{}", 2749 2749 + Some("rev-aaaa"), 2750 2750 + None, 2751 2751 + 10_001, 2752 2752 + ); 2753 2753 + // prefix: 2754 2754 + batch.create( 2755 2755 + "did:plc:person-a", 2756 2756 + "a.a.a.a.a", 2757 2757 + "rkey-aaaaa", 2758 2758 + "{}", 2759 2759 + Some("rev-aaaaa"), 2760 2760 + None, 2761 2761 + 10_002, 2762 2762 + ); 2763 2763 + write.insert_batch(batch.batch)?; 2764 2764 + write.step_rollup()?; 2765 2765 + 2766 2766 + let ( 2767 2767 + JustCount { 2768 2768 + creates, 2769 2769 + dids_estimate, 2770 2770 + .. 2771 2771 + }, 2772 2772 + children, 2773 2773 + cursor, 2774 2774 + ) = read.get_prefix( 2775 2775 + NsidPrefix::new("a.a.a").unwrap(), 2776 2776 + 10, 2777 2777 + OrderCollectionsBy::Lexi { cursor: None }, 2778 2778 + None, 2779 2779 + None, 2780 2780 + )?; 2781 2781 + assert_eq!(creates, 2); 2782 2782 + assert_eq!(dids_estimate, 1); 2783 2783 + assert_eq!( 2784 2784 + children, 2785 2785 + vec![ 2786 2786 + PrefixChild::Collection(NsidCount { 2787 2787 + nsid: "a.a.a.a".to_string(), 2788 2788 + creates: 1, 2789 2789 + dids_estimate: 1 2790 2790 + }), 2791 2791 + PrefixChild::Prefix(PrefixCount { 2792 2792 + prefix: "a.a.a.a".to_string(), 2793 2793 + creates: 1, 2794 2794 + dids_estimate: 1 2795 2795 + }), 2796 2796 + ] 2797 2797 + ); 2798 2798 + assert_eq!(cursor, None); 2467 2799 Ok(()) 2468 2800 } 2469 2801 }