fix lower bound for nsid-prefix iterating · microcosm.blue/microcosm-rs@eb4beb9

+206 -62

1 changed file

Expand all

ufos

src

+206 -62

ufos/src/storage_fjall.rs

··· 665 665 cursor: Option<Vec<u8>>, 666 666 buckets: Vec<CursorBucket>, 667 667 ) -> StorageResult<(JustCount, Vec<PrefixChild>, Option<Vec<u8>>)> { 668 + // TODO: fix up this mess 669 + // the lower/start bound is tricky because `Exclusive()` _without_ a null terminator _will_ include the null- 670 + // terminated exact match. so we actually need the null terminator for an exclusive lower bound! 671 + // 672 + // but for upper/end bound, we *cannot* have a null terminator after the prefix, else we'll only get everything up 673 + // until prefix+null (aka nothing). 674 + // anywayyyyyyyy 675 + let prefix_sub_with_null = prefix.as_str().to_string().to_db_bytes()?; 668 676 let prefix_sub = String::sub_prefix(prefix.as_str())?; 669 677 let cursor_child = cursor 670 678 .as_deref() 671 679 .map(|encoded_bytes| { 672 680 let decoded: String = db_complete(encoded_bytes)?; 673 - let as_sub_prefix = String::sub_prefix(&decoded)?; 674 - Ok::<_, EncodingError>(as_sub_prefix) 681 + let as_sub_prefix_with_null = decoded.to_db_bytes()?; 682 + Ok::<_, EncodingError>(as_sub_prefix_with_null) 675 683 }) 676 684 .transpose()?; 677 685 let mut iters: Vec<NsidCounter> = Vec::with_capacity(buckets.len()); ··· 681 689 let start = cursor_child 682 690 .as_ref() 683 691 .map(|child| HourlyRollupKey::after_nsid_prefix(*t, child)) 684 - .unwrap_or_else(|| HourlyRollupKey::after_nsid_prefix(*t, &prefix_sub))?; 692 + .unwrap_or_else(|| { 693 + HourlyRollupKey::after_nsid_prefix(*t, &prefix_sub_with_null) 694 + })?; 685 695 let end = HourlyRollupKey::nsid_prefix_end(*t, &prefix_sub)?; 686 696 get_lexi_iter::<HourlyRollupKey>(&snapshot, start, end)? 687 697 } ··· 689 699 let start = cursor_child 690 700 .as_ref() 691 701 .map(|child| WeeklyRollupKey::after_nsid_prefix(*t, child)) 692 - .unwrap_or_else(|| WeeklyRollupKey::after_nsid_prefix(*t, &prefix_sub))?; 702 + .unwrap_or_else(|| { 703 + WeeklyRollupKey::after_nsid_prefix(*t, &prefix_sub_with_null) 704 + })?; 693 705 let end = WeeklyRollupKey::nsid_prefix_end(*t, &prefix_sub)?; 694 706 get_lexi_iter::<WeeklyRollupKey>(&snapshot, start, end)? 695 707 } ··· 697 709 let start = cursor_child 698 710 .as_ref() 699 711 .map(|child| AllTimeRollupKey::after_nsid_prefix(child)) 700 - .unwrap_or_else(|| AllTimeRollupKey::after_nsid_prefix(&prefix_sub))?; 712 + .unwrap_or_else(|| { 713 + AllTimeRollupKey::after_nsid_prefix(&prefix_sub_with_null) 714 + })?; 701 715 let end = AllTimeRollupKey::nsid_prefix_end(&prefix_sub)?; 702 716 get_lexi_iter::<AllTimeRollupKey>(&snapshot, start, end)? 703 717 } ··· 709 723 let mut iters: Vec<_> = iters 710 724 .into_iter() 711 725 .map(|it| { 712 - it.map(|bla| bla.map(|(nsid, v)| (Child::from_prefix(&nsid, &prefix), v))) 713 - .peekable() 726 + it.map(|bla| { 727 + bla.map(|(nsid, v)| { 728 + let Some(child) = Child::from_prefix(&nsid, &prefix) else { 729 + panic!("failed from_prefix: {nsid:?} {prefix:?} (bad iter bounds?)"); 730 + }; 731 + (child, v) 732 + }) 733 + }) 734 + .peekable() 714 735 }) 715 736 .collect(); 716 737 ··· 722 743 ChildPrefix(String), 723 744 } 724 745 impl Child { 725 - fn from_prefix(nsid: &Nsid, prefix: &NsidPrefix) -> Self { 746 + fn from_prefix(nsid: &Nsid, prefix: &NsidPrefix) -> Option<Self> { 726 747 if prefix.is_group_of(nsid) { 727 - Child::FullNsid(nsid.to_string()) 728 - } else { 729 - let suffix = nsid 730 - .as_str() 731 - .strip_prefix(&format!("{}.", prefix.0)) 732 - .unwrap(); 733 - let (segment, _) = suffix.split_once('.').unwrap(); 734 - Child::ChildPrefix(format!("{}.{segment}", prefix.0)) 748 + return Some(Child::FullNsid(nsid.to_string())); 735 749 } 750 + let suffix = nsid.as_str().strip_prefix(&format!("{}.", prefix.0))?; 751 + let (segment, _) = suffix.split_once('.').unwrap(); 752 + let child_prefix = format!("{}.{segment}", prefix.0); 753 + Some(Child::ChildPrefix(child_prefix)) 736 754 } 737 755 fn is_before(&self, other: &Child) -> bool { 738 756 match (self, other) { ··· 2468 2486 } 2469 2487 2470 2488 #[test] 2471 - fn get_nsid_prefix_children_lexi_empty() { 2489 + fn get_prefix_children_lexi_empty() { 2472 2490 let (read, _) = fjall_db(); 2473 2491 let ( 2474 2492 JustCount { ··· 2495 2513 } 2496 2514 2497 2515 #[test] 2498 - fn get_nsid_prefix_children_lexi() -> anyhow::Result<()> { 2516 + fn get_prefix_excludes_exact_collection() -> anyhow::Result<()> { 2499 2517 let (read, mut write) = fjall_db(); 2500 2518 2501 2519 let mut batch = TestBatch::default(); ··· 2508 2526 None, 2509 2527 10_000, 2510 2528 ); 2529 + write.insert_batch(batch.batch)?; 2530 + write.step_rollup()?; 2531 + 2532 + let ( 2533 + JustCount { 2534 + creates, 2535 + dids_estimate, 2536 + .. 2537 + }, 2538 + children, 2539 + cursor, 2540 + ) = read.get_prefix( 2541 + NsidPrefix::new("a.a.a").unwrap(), 2542 + 10, 2543 + OrderCollectionsBy::Lexi { cursor: None }, 2544 + None, 2545 + None, 2546 + )?; 2547 + assert_eq!(creates, 0); 2548 + assert_eq!(dids_estimate, 0); 2549 + assert_eq!(children, vec![]); 2550 + assert_eq!(cursor, None); 2551 + Ok(()) 2552 + } 2553 + 2554 + #[test] 2555 + fn get_prefix_includes_child_collection() -> anyhow::Result<()> { 2556 + let (read, mut write) = fjall_db(); 2557 + 2558 + let mut batch = TestBatch::default(); 2511 2559 batch.create( 2512 2560 "did:plc:person-a", 2513 - "a.a.a.a", 2514 - "rkey-aaaa", 2561 + "a.a.a", 2562 + "rkey-aaa", 2515 2563 "{}", 2516 - Some("rev-aaaa"), 2564 + Some("rev-aaa"), 2565 + None, 2566 + 10_000, 2567 + ); 2568 + write.insert_batch(batch.batch)?; 2569 + write.step_rollup()?; 2570 + 2571 + let ( 2572 + JustCount { 2573 + creates, 2574 + dids_estimate, 2575 + .. 2576 + }, 2577 + children, 2578 + cursor, 2579 + ) = read.get_prefix( 2580 + NsidPrefix::new("a.a").unwrap(), 2581 + 10, 2582 + OrderCollectionsBy::Lexi { cursor: None }, 2583 + None, 2517 2584 None, 2518 - 10_001, 2585 + )?; 2586 + assert_eq!(creates, 1); 2587 + assert_eq!(dids_estimate, 1); 2588 + assert_eq!( 2589 + children, 2590 + vec![PrefixChild::Collection(NsidCount { 2591 + nsid: "a.a.a".to_string(), 2592 + creates: 1, 2593 + dids_estimate: 1 2594 + }),] 2519 2595 ); 2596 + assert_eq!(cursor, None); 2597 + Ok(()) 2598 + } 2599 + 2600 + #[test] 2601 + fn get_prefix_includes_child_prefix() -> anyhow::Result<()> { 2602 + let (read, mut write) = fjall_db(); 2603 + 2604 + let mut batch = TestBatch::default(); 2520 2605 batch.create( 2521 - "did:plc:person-b", 2606 + "did:plc:person-a", 2522 2607 "a.a.a.a", 2523 2608 "rkey-aaaa", 2524 2609 "{}", 2525 2610 Some("rev-aaaa"), 2526 2611 None, 2527 - 10_002, 2612 + 10_000, 2528 2613 ); 2529 - batch.create( 2530 - "did:plc:person-a", 2531 - "a.a.a.c", 2532 - "rkey-aaac", 2533 - "{}", 2534 - Some("rev-aaac"), 2614 + write.insert_batch(batch.batch)?; 2615 + write.step_rollup()?; 2616 + 2617 + let ( 2618 + JustCount { 2619 + creates, 2620 + dids_estimate, 2621 + .. 2622 + }, 2623 + children, 2624 + cursor, 2625 + ) = read.get_prefix( 2626 + NsidPrefix::new("a.a").unwrap(), 2627 + 10, 2628 + OrderCollectionsBy::Lexi { cursor: None }, 2629 + None, 2535 2630 None, 2536 - 10_003, 2631 + )?; 2632 + assert_eq!(creates, 1); 2633 + assert_eq!(dids_estimate, 1); 2634 + assert_eq!( 2635 + children, 2636 + vec![PrefixChild::Prefix(PrefixCount { 2637 + prefix: "a.a.a".to_string(), 2638 + creates: 1, 2639 + dids_estimate: 1 2640 + }),] 2537 2641 ); 2642 + assert_eq!(cursor, None); 2643 + Ok(()) 2644 + } 2645 + 2646 + #[test] 2647 + fn get_prefix_merges_child_prefixes() -> anyhow::Result<()> { 2648 + let (read, mut write) = fjall_db(); 2649 + 2650 + let mut batch = TestBatch::default(); 2538 2651 batch.create( 2539 - "did:plc:person-b", 2540 - "a.b.c.d", 2541 - "rkey-abcd", 2652 + "did:plc:person-a", 2653 + "a.a.a.a", 2654 + "rkey-aaaa", 2542 2655 "{}", 2543 - Some("rev-abcd"), 2656 + Some("rev-aaaa"), 2544 2657 None, 2545 - 10_004, 2658 + 10_000, 2546 2659 ); 2547 2660 batch.create( 2548 2661 "did:plc:person-a", 2549 - "w.x.y.z", 2550 - "rkey-wxyz", 2662 + "a.a.a.b", 2663 + "rkey-aaab", 2551 2664 "{}", 2552 - Some("rev-wxyz"), 2665 + Some("rev-aaab"), 2553 2666 None, 2554 - 10_005, 2667 + 10_001, 2555 2668 ); 2556 2669 write.insert_batch(batch.batch)?; 2557 - 2558 2670 write.step_rollup()?; 2559 2671 2560 2672 let ( ··· 2572 2684 None, 2573 2685 None, 2574 2686 )?; 2575 - assert_eq!(creates, 4); 2576 - assert_eq!(dids_estimate, 2); 2687 + assert_eq!(creates, 2); 2688 + assert_eq!(dids_estimate, 1); 2577 2689 assert_eq!( 2578 2690 children, 2579 - vec![ 2580 - PrefixChild::Collection(NsidCount { 2581 - nsid: "a.a.a".to_string(), 2582 - creates: 1, 2583 - dids_estimate: 1 2584 - }), 2585 - PrefixChild::Prefix(PrefixCount { 2586 - prefix: "a.a.a".to_string(), 2587 - creates: 3, 2588 - dids_estimate: 2 2589 - }), 2590 - ] 2691 + vec![PrefixChild::Prefix(PrefixCount { 2692 + prefix: "a.a.a".to_string(), 2693 + creates: 2, 2694 + dids_estimate: 1 2695 + }),] 2591 2696 ); 2592 2697 assert_eq!(cursor, None); 2698 + Ok(()) 2699 + } 2700 + 2701 + #[test] 2702 + fn get_prefix_exact_and_child_and_prefix() -> anyhow::Result<()> { 2703 + let (read, mut write) = fjall_db(); 2704 + 2705 + let mut batch = TestBatch::default(); 2706 + // exact: 2707 + batch.create( 2708 + "did:plc:person-a", 2709 + "a.a.a", 2710 + "rkey-aaa", 2711 + "{}", 2712 + Some("rev-aaa"), 2713 + None, 2714 + 10_000, 2715 + ); 2716 + // child: 2717 + batch.create( 2718 + "did:plc:person-a", 2719 + "a.a.a.a", 2720 + "rkey-aaaa", 2721 + "{}", 2722 + Some("rev-aaaa"), 2723 + None, 2724 + 10_001, 2725 + ); 2726 + // prefix: 2727 + batch.create( 2728 + "did:plc:person-a", 2729 + "a.a.a.a.a", 2730 + "rkey-aaaaa", 2731 + "{}", 2732 + Some("rev-aaaaa"), 2733 + None, 2734 + 10_002, 2735 + ); 2736 + write.insert_batch(batch.batch)?; 2737 + write.step_rollup()?; 2593 2738 2594 2739 let ( 2595 2740 JustCount { ··· 2606 2751 None, 2607 2752 None, 2608 2753 )?; 2609 - assert_eq!(creates, 4); 2610 - assert_eq!(dids_estimate, 2); 2754 + assert_eq!(creates, 2); 2755 + assert_eq!(dids_estimate, 1); 2611 2756 assert_eq!( 2612 2757 children, 2613 2758 vec![ 2614 2759 PrefixChild::Collection(NsidCount { 2615 - nsid: "a.a.a".to_string(), 2760 + nsid: "a.a.a.a".to_string(), 2616 2761 creates: 1, 2617 2762 dids_estimate: 1 2618 2763 }), 2619 2764 PrefixChild::Prefix(PrefixCount { 2620 - prefix: "a.a.a".to_string(), 2621 - creates: 3, 2622 - dids_estimate: 2 2765 + prefix: "a.a.a.a".to_string(), 2766 + creates: 1, 2767 + dids_estimate: 1 2623 2768 }), 2624 2769 ] 2625 2770 ); 2626 2771 assert_eq!(cursor, None); 2627 - 2628 2772 Ok(()) 2629 2773 } 2630 2774 }

Configure Feed

Configure Feed