{"baseline":{"summary":{"config":"baseline","items":48,"answerable":"10/10","paraphrase":"4/10","notCoveredCorrectlyRefused":"4/9","falseRefusals":1,"conflictNewerValue":"2/2","actionOperationRight":"8/8","actionParameters":"0/10","missingParameters":"0/4","unsupportedActionsRefused":"3/5","msPerItem":{"p50":0.2,"p95":0.5,"max":1},"setupMs":{"loadAndIndex":4,"firstCall":1},"failures":["P1","P3","P4","P5","P8","P10","U1","U2","U3","U5","U6","M1","M2","M3","M4","X4","X5"]},"outcomes":[{"id":"A1","kind":"answerable","input":"How much is the default late fee?","ok":true,"details":{"answer":"The default fee is 5 percent of the monthly rent, capped at 75 dollars. [1] To change it, open Settings, then Rent rules, and edit Late fee. [1] Quillhaven adds a late fee automatically on the sixth day of the month when rent is still unpaid. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.6675410000000284},{"id":"A2","kind":"answerable","input":"How do I void a late fee?","ok":true,"details":{"answer":"To change it, open Settings, then Rent rules, and edit Late fee. [1] Open the tenant's ledger, select the fee and choose Void charge. [1] A voided fee stays on the ledger with a line through it, so the history is kept. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.3966249999999718},{"id":"A3","kind":"answerable","input":"How many days before a lease ends does Quillhaven send a reminder?","ok":true,"details":{"answer":"Quillhaven reminds the property manager 75 days before a lease ends. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.4455000000000382},{"id":"A4","kind":"answerable","input":"How long does a tenant have to accept a renewal offer?","ok":true,"details":{"answer":"The tenant has 10 days to accept the offer in the tenant app. [1] To send a renewal offer, open the lease and choose Offer renewal, then set the new rent and the term in months. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.3344170000000304},{"id":"A5","kind":"answerable","input":"How many photos can a tenant attach to a maintenance request?","ok":true,"details":{"answer":"Tenants submit maintenance requests in the tenant app and can attach up to six photos. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.39858300000003055},{"id":"A6","kind":"answerable","input":"When are owner statements generated?","ok":true,"details":{"answer":"Owner statements are generated on the third business day of each month and emailed to owners as a PDF. [1] Owners can download past statements from the owner portal for seven years. [1] Leasing agents can see applicants and leases but not owner statements. [2]","refused":false,"correct":true,"citationSupports":true},"ms":0.3838749999999891},{"id":"A7","kind":"answerable","input":"How many days do I have to return a security deposit?","ok":true,"details":{"answer":"When a tenant moves out, the deposit must be returned within 21 days, minus itemised deductions. [1] Security deposits are held in a separate trust account, never in the operating account. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.29062499999997726},{"id":"A8","kind":"answerable","input":"Which role can delete a property?","ok":true,"details":{"answer":"Only Owners can delete a property. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.36262499999998},{"id":"A9","kind":"answerable","input":"What are the gym opening hours?","ok":true,"details":{"answer":"The gym on the ground floor is open from 5 am to 11 pm. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.2018750000000864},{"id":"A10","kind":"answerable","input":"Where is the bicycle storage?","ok":true,"details":{"answer":"Bicycle storage is in the basement on level B2. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.1845419999999649},{"id":"P1","kind":"paraphrase","input":"What's the penalty when a renter pays late?","ok":false,"details":{"answer":"To change it, open Settings, then Rent rules, and edit Late fee. [1]","refused":false,"correct":false,"citationSupports":true},"ms":0.50324999999998},{"id":"P2","kind":"paraphrase","input":"Can I cancel a late charge?","ok":true,"details":{"answer":"To change it, open Settings, then Rent rules, and edit Late fee. [1] Open the tenant's ledger, select the fee and choose Void charge. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.4584579999999505},{"id":"P3","kind":"paraphrase","input":"How far ahead am I warned that a tenancy is ending?","ok":false,"details":{"answer":"Quillhaven imports bank transactions every night at 2 am. [1]","refused":false,"correct":false,"citationSupports":false},"ms":0.2973330000000942},{"id":"P4","kind":"paraphrase","input":"What happens if the renter ignores the renewal?","ok":false,"details":{"answer":"To send a renewal offer, open the lease and choose Offer renewal, then set the new rent and the term in months. [1]","refused":false,"correct":false,"citationSupports":true},"ms":0.2397080000000642},{"id":"P5","kind":"paraphrase","input":"Who gets alerted when an apartment has no heating?","ok":false,"details":{"answer":"Each request gets a priority of low, normal or urgent. [1]","refused":false,"correct":false,"citationSupports":true},"ms":0.1951250000000755},{"id":"P6","kind":"paraphrase","input":"For how many years can landlords get old statements?","ok":true,"details":{"answer":"Owners can download past statements from the owner portal for seven years. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.23783300000002328},{"id":"P7","kind":"paraphrase","input":"How much warning do renters get before someone inspects their home?","ok":true,"details":{"answer":"Move in and move out inspections are done in the Quillhaven Inspect app. [1] Tenants receive notice 48 hours before a routine inspection. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.22308299999997416},{"id":"P8","kind":"paraphrase","input":"Where is a renter's deposit money kept?","ok":false,"details":{"answer":"A voided fee stays on the ledger with a line through it, so the history is kept. [2]","refused":false,"correct":false,"citationSupports":true},"ms":0.2204580000000078},{"id":"P9","kind":"paraphrase","input":"Can leasing staff look at owner statements?","ok":true,"details":{"answer":"Leasing agents can see applicants and leases but not owner statements. [1] Owners can download past statements from the owner portal for seven years. [2] Owner statements are generated on the third business day of each month and emailed to owners as a PDF. [2]","refused":false,"correct":true,"citationSupports":true},"ms":0.3404170000000022},{"id":"P10","kind":"paraphrase","input":"Where can I keep my bike?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.09287499999993544},{"id":"U1","kind":"unsupported","input":"Does Quillhaven integrate with QuickBooks?","ok":false,"details":{"answer":"Quillhaven imports bank transactions every night at 2 am. [3] Move in and move out inspections are done in the Quillhaven Inspect app. [2]","refused":false},"ms":0.2907920000000104},{"id":"U2","kind":"unsupported","input":"How do I charge a pet fee?","ok":false,"details":{"answer":"Open the tenant's ledger, select the fee and choose Void charge. [1]","refused":false},"ms":0.21241599999996197},{"id":"U3","kind":"unsupported","input":"Can tenants pay rent with cryptocurrency?","ok":false,"details":{"answer":"The tenant has 10 days to accept the offer in the tenant app. [1] If the tenant does not answer, the lease continues month to month on the current rent. [1] Tenants submit maintenance requests in the tenant app and can attach up to six photos. [3]","refused":false},"ms":0.2807500000000118},{"id":"U4","kind":"unsupported","input":"Is there a swimming pool at Kestrel Yard?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.07541700000001583},{"id":"U5","kind":"unsupported","input":"How many parking spaces does each apartment get?","ok":false,"details":{"answer":"Each apartment may store up to two bicycles. [1]","refused":false},"ms":0.13470799999993233},{"id":"U6","kind":"unsupported","input":"What is the maximum late fee allowed by law in California?","ok":false,"details":{"answer":"To change it, open Settings, then Rent rules, and edit Late fee. [1] Quillhaven adds a late fee automatically on the sixth day of the month when rent is still unpaid. [1]","refused":false},"ms":0.9505420000000412},{"id":"N1","kind":"unsupported","input":"What is the capital of France?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.1674160000000029},{"id":"N2","kind":"unsupported","input":"Who won the football world cup in 2022?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.12104100000010476},{"id":"N3","kind":"unsupported","input":"Tell me a joke","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.05162499999994452},{"id":"C1","kind":"conflict","input":"When is recycling collected?","ok":true,"details":{"answer":"Recycling is collected every Wednesday morning. [2] From 1 November, recycling is collected every Thursday morning instead of Wednesday. [1]","hasNewer":true,"hasOlder":true},"ms":0.10370899999998072},{"id":"C2","kind":"conflict","input":"When should I put the recycling bins out?","ok":true,"details":{"answer":"Put the bins out after 8 pm on Wednesday. [1] Put the bins out after 8 pm on Tuesday. [2]","hasNewer":true,"hasOlder":true},"ms":0.10387500000001636},{"id":"R1","kind":"action","input":"Create a maintenance request for unit 5A: the kitchen tap is leaking","ok":true,"details":{"operation":"createMaintenanceRequest","expectedOperation":"createMaintenanceRequest","proposed":{},"paramsRight":0,"paramsExpected":1},"ms":0.35154099999999744},{"id":"R2","kind":"action","input":"Schedule a move out inspection for unit 12C on Friday at 10am","ok":true,"details":{"operation":"scheduleInspection","expectedOperation":"scheduleInspection","proposed":{},"paramsRight":0,"paramsExpected":3},"ms":0.0655419999999367},{"id":"R3","kind":"action","input":"Which units are vacant?","ok":true,"details":{"operation":"listUnits","expectedOperation":"listUnits","proposed":{},"paramsRight":0,"paramsExpected":0},"ms":0.049250000000029104},{"id":"R4","kind":"action","input":"Void the late fee on charge CH-2231 because the bank was closed","ok":true,"details":{"operation":"voidCharge","expectedOperation":"voidCharge","proposed":{},"paramsRight":0,"paramsExpected":1},"ms":0.04754200000002129},{"id":"R5","kind":"action","input":"Send a message to the tenants in block B about the water shutdown","ok":true,"details":{"operation":"sendMessage","expectedOperation":"sendMessage","proposed":{},"paramsRight":0,"paramsExpected":0},"ms":0.044875000000047294},{"id":"R6","kind":"action","input":"Show me the ledger for tenant T-884","ok":true,"details":{"operation":"getLedger","expectedOperation":"getLedger","proposed":{},"paramsRight":0,"paramsExpected":1},"ms":0.03945900000007896},{"id":"R7","kind":"action","input":"Book the service lift for unit 3D on Saturday at 9am","ok":true,"details":{"operation":"bookServiceLift","expectedOperation":"bookServiceLift","proposed":{},"paramsRight":0,"paramsExpected":3},"ms":0.025624999999990905},{"id":"R8","kind":"action","input":"Offer a 12 month renewal on lease L-5530 at 1,450 dollars","ok":true,"details":{"operation":"offerRenewal","expectedOperation":"offerRenewal","proposed":{},"paramsRight":0,"paramsExpected":1},"ms":0.044125000000008185},{"id":"M1","kind":"action-missing","input":"Schedule an inspection","ok":false,"details":{"operation":"scheduleInspection","expectedOperation":"scheduleInspection","missing":[],"expectedMissing":["unit_id","type","date"],"operationRight":true,"missingListed":false},"ms":0.11199999999996636},{"id":"M2","kind":"action-missing","input":"Book the service lift","ok":false,"details":{"operation":"bookServiceLift","expectedOperation":"bookServiceLift","missing":[],"expectedMissing":["unit_id","date","start_time"],"operationRight":true,"missingListed":false},"ms":0.03441699999996217},{"id":"M3","kind":"action-missing","input":"Create a maintenance request","ok":false,"details":{"operation":"createMaintenanceRequest","expectedOperation":"createMaintenanceRequest","missing":[],"expectedMissing":["unit_id","description"],"operationRight":true,"missingListed":false},"ms":0.03795900000000074},{"id":"M4","kind":"action-missing","input":"Offer a lease renewal","ok":false,"details":{"operation":"offerRenewal","expectedOperation":"offerRenewal","missing":[],"expectedMissing":["lease_id","rent","term_months"],"operationRight":true,"missingListed":false},"ms":0.033999999999991815},{"id":"X1","kind":"action-unsupported","input":"Order a pizza for the office","ok":true,"details":{"operation":null,"text":"None of the 8 exposed actions matches that request. Nothing was run."},"ms":0.03720799999996416},{"id":"X2","kind":"action-unsupported","input":"Cancel my gym membership","ok":true,"details":{"operation":null,"text":"None of the 4 exposed actions matches that request. Nothing was run."},"ms":0.018374999999991815},{"id":"X3","kind":"action-unsupported","input":"Transfer 500 dollars to the owner's bank account","ok":true,"details":{"operation":null,"text":"None of the 8 exposed actions matches that request. Nothing was run."},"ms":0.03412500000001728},{"id":"X4","kind":"action-unsupported","input":"Delete the building","ok":false,"details":{"operation":"deleteApplicant","text":"deleteApplicant (DELETE /applicants/{applicant_id}), level Sensitive. Required: applicant_id. The user would type a confirmation word before this runs. This is a preview, so nothing was sent to your API."},"ms":0.031375000000025466},{"id":"X5","kind":"action-unsupported","input":"Evict the tenant in 4B","ok":false,"details":{"operation":"getLedger","text":"getLedger (GET /tenants/{tenant_id}/ledger), level Read. Required: tenant_id. A read only call; it would run without a confirmation step. This is a preview, so nothing was sent to your API."},"ms":0.03224999999997635}]},"rules":{"summary":{"config":"rules","dataset":"eval","semantic":"off","machine":"Apple M1 Max, 64 GB, Node v25.6.1","items":48,"answerable":"9/10","paraphrase":"0/10","notCoveredCorrectlyRefused":"9/9","falseRefusals":11,"conflictNewerValue":"2/2","actionOperationRight":"8/8","actionParameters":"10/10","missingParameters":"4/4","unsupportedActionsRefused":"3/5","msPerItem":{"p50":0.2,"p95":0.9,"max":1},"setupMs":{"loadAndIndex":5,"firstCall":1},"failures":["A4","P1","P2","P3","P4","P5","P6","P7","P8","P9","P10","X4","X5"]},"outcomes":[{"id":"A1","kind":"answerable","input":"How much is the default late fee?","ok":true,"details":{"answer":"The default fee is 5 percent of the monthly rent, capped at 75 dollars. [1] To change it, open Settings, then Rent rules, and edit Late fee. [1] Quillhaven adds a late fee automatically on the sixth day of the month when rent is still unpaid. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.863125000000025},{"id":"A2","kind":"answerable","input":"How do I void a late fee?","ok":true,"details":{"answer":"Open the tenant's ledger, select the fee and choose Void charge. [1] A voided fee stays on the ledger with a line through it, so the history is kept. [1] To change it, open Settings, then Rent rules, and edit Late fee. [2]","refused":false,"correct":true,"citationSupports":true},"ms":0.35266699999999673},{"id":"A3","kind":"answerable","input":"How many days before a lease ends does Quillhaven send a reminder?","ok":true,"details":{"answer":"Quillhaven reminds the property manager 75 days before a lease ends. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.32049999999998136},{"id":"A4","kind":"answerable","input":"How long does a tenant have to accept a renewal offer?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.96795800000001},{"id":"A5","kind":"answerable","input":"How many photos can a tenant attach to a maintenance request?","ok":true,"details":{"answer":"Tenants submit maintenance requests in the tenant app and can attach up to six photos. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.31266699999997627},{"id":"A6","kind":"answerable","input":"When are owner statements generated?","ok":true,"details":{"answer":"Owner statements are generated on the third business day of each month and emailed to owners as a PDF. [1] Owners can download past statements from the owner portal for seven years. [1] Leasing agents can see applicants and leases but not owner statements. [2]","refused":false,"correct":true,"citationSupports":true},"ms":0.284417000000019},{"id":"A7","kind":"answerable","input":"How many days do I have to return a security deposit?","ok":true,"details":{"answer":"When a tenant moves out, the deposit must be returned within 21 days, minus itemised deductions. [1] Security deposits are held in a separate trust account, never in the operating account. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.2303749999999809},{"id":"A8","kind":"answerable","input":"Which role can delete a property?","ok":true,"details":{"answer":"Only Owners can delete a property. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.2480419999999981},{"id":"A9","kind":"answerable","input":"What are the gym opening hours?","ok":true,"details":{"answer":"The gym on the ground floor is open from 5 am to 11 pm. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.12354199999998627},{"id":"A10","kind":"answerable","input":"Where is the bicycle storage?","ok":true,"details":{"answer":"Bicycle storage is in the basement on level B2. [1]","refused":false,"correct":true,"citationSupports":true},"ms":0.1093330000000492},{"id":"P1","kind":"paraphrase","input":"What's the penalty when a renter pays late?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.1863329999999905},{"id":"P2","kind":"paraphrase","input":"Can I cancel a late charge?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.19425000000001091},{"id":"P3","kind":"paraphrase","input":"How far ahead am I warned that a tenancy is ending?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.202666000000022},{"id":"P4","kind":"paraphrase","input":"What happens if the renter ignores the renewal?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.1780830000000151},{"id":"P5","kind":"paraphrase","input":"Who gets alerted when an apartment has no heating?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.2300000000000182},{"id":"P6","kind":"paraphrase","input":"For how many years can landlords get old statements?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.18804199999999585},{"id":"P7","kind":"paraphrase","input":"How much warning do renters get before someone inspects their home?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.1747909999999706},{"id":"P8","kind":"paraphrase","input":"Where is a renter's deposit money kept?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.1621659999999565},{"id":"P9","kind":"paraphrase","input":"Can leasing staff look at owner statements?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.1805830000000128},{"id":"P10","kind":"paraphrase","input":"Where can I keep my bike?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":0.08220800000003692},{"id":"U1","kind":"unsupported","input":"Does Quillhaven integrate with QuickBooks?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.2588749999999891},{"id":"U2","kind":"unsupported","input":"How do I charge a pet fee?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.19008400000001305},{"id":"U3","kind":"unsupported","input":"Can tenants pay rent with cryptocurrency?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.19512500000001864},{"id":"U4","kind":"unsupported","input":"Is there a swimming pool at Kestrel Yard?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.07854199999997036},{"id":"U5","kind":"unsupported","input":"How many parking spaces does each apartment get?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.0917919999999981},{"id":"U6","kind":"unsupported","input":"What is the maximum late fee allowed by law in California?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.1876670000000331},{"id":"N1","kind":"unsupported","input":"What is the capital of France?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.16987499999999045},{"id":"N2","kind":"unsupported","input":"Who won the football world cup in 2022?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.16558300000002646},{"id":"N3","kind":"unsupported","input":"Tell me a joke","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":0.0857499999999618},{"id":"C1","kind":"conflict","input":"When is recycling collected?","ok":true,"details":{"answer":"Recycling is collected every Wednesday morning. [1] From 1 November, recycling is collected every Thursday morning instead of Wednesday. [2]","hasNewer":true,"hasOlder":true},"ms":0.17387500000000955},{"id":"C2","kind":"conflict","input":"When should I put the recycling bins out?","ok":true,"details":{"answer":"Put the bins out after 8 pm on Tuesday. [1] Put the bins out after 8 pm on Wednesday. [2]","hasNewer":true,"hasOlder":true},"ms":0.1505830000000401},{"id":"R1","kind":"action","input":"Create a maintenance request for unit 5A: the kitchen tap is leaking","ok":true,"details":{"operation":"createMaintenanceRequest","expectedOperation":"createMaintenanceRequest","proposed":{"unit_id":"5A","description":"the kitchen tap is leaking"},"paramsRight":1,"paramsExpected":1},"ms":1.0087500000000205},{"id":"R2","kind":"action","input":"Schedule a move out inspection for unit 12C on Friday at 10am","ok":true,"details":{"operation":"scheduleInspection","expectedOperation":"scheduleInspection","proposed":{"unit_id":"12C","type":"move out","date":"Friday","time":"10am"},"paramsRight":3,"paramsExpected":3},"ms":0.6126669999999876},{"id":"R3","kind":"action","input":"Which units are vacant?","ok":true,"details":{"operation":"listUnits","expectedOperation":"listUnits","proposed":{"status":"vacant"},"paramsRight":0,"paramsExpected":0},"ms":0.2407079999999837},{"id":"R4","kind":"action","input":"Void the late fee on charge CH-2231 because the bank was closed","ok":true,"details":{"operation":"voidCharge","expectedOperation":"voidCharge","proposed":{"charge_id":"CH-2231","reason":"the bank was closed"},"paramsRight":1,"paramsExpected":1},"ms":0.17599999999998772},{"id":"R5","kind":"action","input":"Send a message to the tenants in block B about the water shutdown","ok":true,"details":{"operation":"sendMessage","expectedOperation":"sendMessage","proposed":{"body":"the water shutdown"},"paramsRight":0,"paramsExpected":0},"ms":0.25029200000000174},{"id":"R6","kind":"action","input":"Show me the ledger for tenant T-884","ok":true,"details":{"operation":"getLedger","expectedOperation":"getLedger","proposed":{"tenant_id":"T-884"},"paramsRight":1,"paramsExpected":1},"ms":0.15216700000001993},{"id":"R7","kind":"action","input":"Book the service lift for unit 3D on Saturday at 9am","ok":true,"details":{"operation":"bookServiceLift","expectedOperation":"bookServiceLift","proposed":{"unit_id":"3D","date":"Saturday","start_time":"9am"},"paramsRight":3,"paramsExpected":3},"ms":0.18625000000002956},{"id":"R8","kind":"action","input":"Offer a 12 month renewal on lease L-5530 at 1,450 dollars","ok":true,"details":{"operation":"offerRenewal","expectedOperation":"offerRenewal","proposed":{"lease_id":"L-5530","rent":"1450","term_months":"12"},"paramsRight":1,"paramsExpected":1},"ms":0.2637500000000159},{"id":"M1","kind":"action-missing","input":"Schedule an inspection","ok":true,"details":{"operation":"scheduleInspection","expectedOperation":"scheduleInspection","missing":["unit_id","type","date"],"expectedMissing":["unit_id","type","date"],"operationRight":true,"missingListed":true},"ms":0.6474590000000262},{"id":"M2","kind":"action-missing","input":"Book the service lift","ok":true,"details":{"operation":"bookServiceLift","expectedOperation":"bookServiceLift","missing":["unit_id","date","start_time"],"expectedMissing":["unit_id","date","start_time"],"operationRight":true,"missingListed":true},"ms":0.5574159999999893},{"id":"M3","kind":"action-missing","input":"Create a maintenance request","ok":true,"details":{"operation":"createMaintenanceRequest","expectedOperation":"createMaintenanceRequest","missing":["unit_id","description"],"expectedMissing":["unit_id","description"],"operationRight":true,"missingListed":true},"ms":0.6083749999999668},{"id":"M4","kind":"action-missing","input":"Offer a lease renewal","ok":true,"details":{"operation":"offerRenewal","expectedOperation":"offerRenewal","missing":["lease_id","rent","term_months"],"expectedMissing":["lease_id","rent","term_months"],"operationRight":true,"missingListed":true},"ms":0.2420830000000365},{"id":"X1","kind":"action-unsupported","input":"Order a pizza for the office","ok":true,"details":{"operation":null,"text":"None of the 8 switched on actions matches that request, so nothing was prepared and nothing was run."},"ms":0.09537499999999},{"id":"X2","kind":"action-unsupported","input":"Cancel my gym membership","ok":true,"details":{"operation":null,"text":"None of the 4 switched on actions matches that request, so nothing was prepared and nothing was run."},"ms":0.0308329999999728},{"id":"X3","kind":"action-unsupported","input":"Transfer 500 dollars to the owner's bank account","ok":true,"details":{"operation":null,"text":"None of the 8 switched on actions matches that request, so nothing was prepared and nothing was run."},"ms":0.05891600000001063},{"id":"X4","kind":"action-unsupported","input":"Delete the building","ok":false,"details":{"operation":"deleteApplicant","text":"deleteApplicant (DELETE /applicants/{applicant_id}), level Sensitive. No values found in the request. Still needed: applicant_id. The user would type a confirmation word before this runs. This release does not execute the operation, so nothing was sent to your API."},"ms":0.12137500000000044},{"id":"X5","kind":"action-unsupported","input":"Evict the tenant in 4B","ok":false,"details":{"operation":"getLedger","text":"getLedger (GET /tenants/{tenant_id}/ledger), level Read. No values found in the request. Still needed: tenant_id. A read only call; it would run without a confirmation step. This release does not execute the operation, so nothing was sent to your API."},"ms":0.07862499999998818}]},"model":{"summary":{"config":"semantic","dataset":"eval","semantic":"all-MiniLM-L6-v2","machine":"Apple M1 Max, 64 GB, Node v25.6.1","items":48,"answerable":"10/10","paraphrase":"6/10","notCoveredCorrectlyRefused":"9/9","falseRefusals":4,"conflictNewerValue":"1/2","actionOperationRight":"8/8","actionParameters":"10/10","missingParameters":"4/4","unsupportedActionsRefused":"4/5","msPerItem":{"p50":4.3,"p95":12.5,"max":17.8},"setupMs":{"loadAndIndex":254,"firstCall":2},"failures":["P3","P4","P5","P8","C1","X4"]},"outcomes":[{"id":"A1","kind":"answerable","input":"How much is the default late fee?","ok":true,"details":{"answer":"The default fee is 5 percent of the monthly rent, capped at 75 dollars. [1]","refused":false,"correct":true,"citationSupports":true},"ms":8.87479200000007},{"id":"A2","kind":"answerable","input":"How do I void a late fee?","ok":true,"details":{"answer":"Open the tenant's ledger, select the fee and choose Void charge. [1]","refused":false,"correct":true,"citationSupports":true},"ms":8.640500000000088},{"id":"A3","kind":"answerable","input":"How many days before a lease ends does Quillhaven send a reminder?","ok":true,"details":{"answer":"Quillhaven reminds the property manager 75 days before a lease ends. [1]","refused":false,"correct":true,"citationSupports":true},"ms":13.881541999999968},{"id":"A4","kind":"answerable","input":"How long does a tenant have to accept a renewal offer?","ok":true,"details":{"answer":"The tenant has 10 days to accept the offer in the tenant app. [1]","refused":false,"correct":true,"citationSupports":true},"ms":10.041499999999928},{"id":"A5","kind":"answerable","input":"How many photos can a tenant attach to a maintenance request?","ok":true,"details":{"answer":"Tenants submit maintenance requests in the tenant app and can attach up to six photos. [1]","refused":false,"correct":true,"citationSupports":true},"ms":4.532000000000039},{"id":"A6","kind":"answerable","input":"When are owner statements generated?","ok":true,"details":{"answer":"Owner statements are generated on the third business day of each month and emailed to owners as a PDF. [1]","refused":false,"correct":true,"citationSupports":true},"ms":4.409457999999972},{"id":"A7","kind":"answerable","input":"How many days do I have to return a security deposit?","ok":true,"details":{"answer":"When a tenant moves out, the deposit must be returned within 21 days, minus itemised deductions. [1]","refused":false,"correct":true,"citationSupports":true},"ms":4.6967500000000655},{"id":"A8","kind":"answerable","input":"Which role can delete a property?","ok":true,"details":{"answer":"Only Owners can delete a property. [1]","refused":false,"correct":true,"citationSupports":true},"ms":5.542124999999942},{"id":"A9","kind":"answerable","input":"What are the gym opening hours?","ok":true,"details":{"answer":"The gym on the ground floor is open from 5 am to 11 pm. [1]","refused":false,"correct":true,"citationSupports":true},"ms":5.630415999999968},{"id":"A10","kind":"answerable","input":"Where is the bicycle storage?","ok":true,"details":{"answer":"Bicycle storage is in the basement on level B2. [1] Each apartment may store up to two bicycles. [1]","refused":false,"correct":true,"citationSupports":true},"ms":4.0791659999999865},{"id":"P1","kind":"paraphrase","input":"What's the penalty when a renter pays late?","ok":true,"details":{"answer":"The default fee is 5 percent of the monthly rent, capped at 75 dollars. [1] Quillhaven adds a late fee automatically on the sixth day of the month when rent is still unpaid. [1]","refused":false,"correct":true,"citationSupports":true},"ms":10.69150000000002},{"id":"P2","kind":"paraphrase","input":"Can I cancel a late charge?","ok":true,"details":{"answer":"Open the tenant's ledger, select the fee and choose Void charge. [1]","refused":false,"correct":true,"citationSupports":true},"ms":8.383249999999975},{"id":"P3","kind":"paraphrase","input":"How far ahead am I warned that a tenancy is ending?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":10.331332999999972},{"id":"P4","kind":"paraphrase","input":"What happens if the renter ignores the renewal?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":9.692042000000017},{"id":"P5","kind":"paraphrase","input":"Who gets alerted when an apartment has no heating?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":1.4246669999999997},{"id":"P6","kind":"paraphrase","input":"For how many years can landlords get old statements?","ok":true,"details":{"answer":"Owners can download past statements from the owner portal for seven years. [1]","refused":false,"correct":true,"citationSupports":true},"ms":6.958375000000046},{"id":"P7","kind":"paraphrase","input":"How much warning do renters get before someone inspects their home?","ok":true,"details":{"answer":"Tenants receive notice 48 hours before a routine inspection. [1]","refused":false,"correct":true,"citationSupports":true},"ms":4.995791999999938},{"id":"P8","kind":"paraphrase","input":"Where is a renter's deposit money kept?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":5.030124999999998},{"id":"P9","kind":"paraphrase","input":"Can leasing staff look at owner statements?","ok":true,"details":{"answer":"Leasing agents can see applicants and leases but not owner statements. [2]","refused":false,"correct":true,"citationSupports":true},"ms":8.25929199999996},{"id":"P10","kind":"paraphrase","input":"Where can I keep my bike?","ok":true,"details":{"answer":"Each apartment may store up to two bicycles. [1] Bicycle storage is in the basement on level B2. [1]","refused":false,"correct":true,"citationSupports":true},"ms":3.739374999999995},{"id":"U1","kind":"unsupported","input":"Does Quillhaven integrate with QuickBooks?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":17.755709000000138},{"id":"U2","kind":"unsupported","input":"How do I charge a pet fee?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":1.9666670000001432},{"id":"U3","kind":"unsupported","input":"Can tenants pay rent with cryptocurrency?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":4.34641600000009},{"id":"U4","kind":"unsupported","input":"Is there a swimming pool at Kestrel Yard?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":1.1879999999998745},{"id":"U5","kind":"unsupported","input":"How many parking spaces does each apartment get?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":3.537166999999954},{"id":"U6","kind":"unsupported","input":"What is the maximum late fee allowed by law in California?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":5.9013340000001335},{"id":"N1","kind":"unsupported","input":"What is the capital of France?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":1.364791999999852},{"id":"N2","kind":"unsupported","input":"Who won the football world cup in 2022?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":1.2769170000001395},{"id":"N3","kind":"unsupported","input":"Tell me a joke","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":1.1508750000000418},{"id":"C1","kind":"conflict","input":"When is recycling collected?","ok":false,"details":{"answer":"Recycling is collected every Wednesday morning. [1]","hasNewer":false,"hasOlder":true},"ms":7.071833000000197},{"id":"C2","kind":"conflict","input":"When should I put the recycling bins out?","ok":true,"details":{"answer":"Put the bins out after 8 pm on Tuesday. [1] Put the bins out after 8 pm on Wednesday. [2]","hasNewer":true,"hasOlder":true},"ms":6.030041999999867},{"id":"R1","kind":"action","input":"Create a maintenance request for unit 5A: the kitchen tap is leaking","ok":true,"details":{"operation":"createMaintenanceRequest","expectedOperation":"createMaintenanceRequest","proposed":{"unit_id":"5A","description":"the kitchen tap is leaking"},"paramsRight":1,"paramsExpected":1},"ms":12.46008299999994},{"id":"R2","kind":"action","input":"Schedule a move out inspection for unit 12C on Friday at 10am","ok":true,"details":{"operation":"scheduleInspection","expectedOperation":"scheduleInspection","proposed":{"unit_id":"12C","type":"move out","date":"Friday","time":"10am"},"paramsRight":3,"paramsExpected":3},"ms":2.395625000000109},{"id":"R3","kind":"action","input":"Which units are vacant?","ok":true,"details":{"operation":"listUnits","expectedOperation":"listUnits","proposed":{"status":"vacant"},"paramsRight":0,"paramsExpected":0},"ms":1.4526670000000197},{"id":"R4","kind":"action","input":"Void the late fee on charge CH-2231 because the bank was closed","ok":true,"details":{"operation":"voidCharge","expectedOperation":"voidCharge","proposed":{"charge_id":"CH-2231","reason":"the bank was closed"},"paramsRight":1,"paramsExpected":1},"ms":2.1670409999999265},{"id":"R5","kind":"action","input":"Send a message to the tenants in block B about the water shutdown","ok":true,"details":{"operation":"sendMessage","expectedOperation":"sendMessage","proposed":{"body":"the water shutdown"},"paramsRight":0,"paramsExpected":0},"ms":1.9818749999999457},{"id":"R6","kind":"action","input":"Show me the ledger for tenant T-884","ok":true,"details":{"operation":"getLedger","expectedOperation":"getLedger","proposed":{"tenant_id":"T-884"},"paramsRight":1,"paramsExpected":1},"ms":1.4470410000001266},{"id":"R7","kind":"action","input":"Book the service lift for unit 3D on Saturday at 9am","ok":true,"details":{"operation":"bookServiceLift","expectedOperation":"bookServiceLift","proposed":{"unit_id":"3D","date":"Saturday","start_time":"9am"},"paramsRight":3,"paramsExpected":3},"ms":6.4721250000000055},{"id":"R8","kind":"action","input":"Offer a 12 month renewal on lease L-5530 at 1,450 dollars","ok":true,"details":{"operation":"offerRenewal","expectedOperation":"offerRenewal","proposed":{"lease_id":"L-5530","rent":"1450","term_months":"12"},"paramsRight":1,"paramsExpected":1},"ms":2.1412499999999},{"id":"M1","kind":"action-missing","input":"Schedule an inspection","ok":true,"details":{"operation":"scheduleInspection","expectedOperation":"scheduleInspection","missing":["unit_id","type","date"],"expectedMissing":["unit_id","type","date"],"operationRight":true,"missingListed":true},"ms":1.8498329999999896},{"id":"M2","kind":"action-missing","input":"Book the service lift","ok":true,"details":{"operation":"bookServiceLift","expectedOperation":"bookServiceLift","missing":["unit_id","date","start_time"],"expectedMissing":["unit_id","date","start_time"],"operationRight":true,"missingListed":true},"ms":1.7062919999998485},{"id":"M3","kind":"action-missing","input":"Create a maintenance request","ok":true,"details":{"operation":"createMaintenanceRequest","expectedOperation":"createMaintenanceRequest","missing":["unit_id","description"],"expectedMissing":["unit_id","description"],"operationRight":true,"missingListed":true},"ms":1.668582999999899},{"id":"M4","kind":"action-missing","input":"Offer a lease renewal","ok":true,"details":{"operation":"offerRenewal","expectedOperation":"offerRenewal","missing":["lease_id","rent","term_months"],"expectedMissing":["lease_id","rent","term_months"],"operationRight":true,"missingListed":true},"ms":1.186416999999892},{"id":"X1","kind":"action-unsupported","input":"Order a pizza for the office","ok":true,"details":{"operation":null,"text":"None of the 8 switched on actions matches that request, so nothing was prepared and nothing was run."},"ms":1.0972079999999096},{"id":"X2","kind":"action-unsupported","input":"Cancel my gym membership","ok":true,"details":{"operation":null,"text":"None of the 4 switched on actions matches that request, so nothing was prepared and nothing was run."},"ms":0.9260420000000522},{"id":"X3","kind":"action-unsupported","input":"Transfer 500 dollars to the owner's bank account","ok":true,"details":{"operation":null,"text":"None of the 8 switched on actions matches that request, so nothing was prepared and nothing was run."},"ms":1.3140000000000782},{"id":"X4","kind":"action-unsupported","input":"Delete the building","ok":false,"details":{"operation":"deleteApplicant","text":"deleteApplicant (DELETE /applicants/{applicant_id}), level Sensitive. No values found in the request. Still needed: applicant_id. The user would type a confirmation word before this runs. This release does not execute the operation, so nothing was sent to your API."},"ms":1.112875000000031},{"id":"X5","kind":"action-unsupported","input":"Evict the tenant in 4B","ok":true,"details":{"operation":null,"text":"That request is about as close to getLedger as to sendMessage, so nothing was prepared. Say which one you mean. Nothing was run."},"ms":1.2304579999999987}]},"development":{"summary":{"config":"semantic","dataset":"dev","semantic":"all-MiniLM-L6-v2","machine":"Apple M1 Max, 64 GB, Node v25.6.1","items":27,"answerable":"5/5","paraphrase":"2/5","notCoveredCorrectlyRefused":"6/7","falseRefusals":1,"conflictNewerValue":"0/0","actionOperationRight":"5/5","actionParameters":"1/1","missingParameters":"1/1","unsupportedActionsRefused":"4/4","msPerItem":{"p50":3.1,"p95":20.1,"max":20.4},"setupMs":{"loadAndIndex":391,"firstCall":3},"failures":["dP3","dP4","dP5","dU2"]},"outcomes":[{"id":"dA1","kind":"answerable","input":"How do I waive a late fee for a tenant?","ok":true,"details":{"answer":"To waive a late fee, open the tenant's ledger, select the fee and choose Waive fee. [1] You need the Waive fees permission to do this. [1]","refused":false,"correct":true,"citationSupports":true},"ms":20.121792000000028},{"id":"dA2","kind":"answerable","input":"How much notice do tenants get before an inspection?","ok":true,"details":{"answer":"Tenants get at least 48 hours notice before a routine inspection by default. [1] Move in and move out inspections are arranged with the tenant directly. [1]","refused":false,"correct":true,"citationSupports":true},"ms":13.704250000000002},{"id":"dA3","kind":"answerable","input":"What happens if autopay fails?","ok":true,"details":{"answer":"If an autopay payment fails, autopay pauses and the tenant is asked to update their payment method. [1]","refused":false,"correct":true,"citationSupports":true},"ms":9.132790999999996},{"id":"dA4","kind":"answerable","input":"Can I edit a payment after the month is closed?","ok":true,"details":{"answer":"A payment cannot be edited once the month is closed. [1]","refused":false,"correct":true,"citationSupports":true},"ms":14.713208000000009},{"id":"dA5","kind":"answerable","input":"When are owner statements sent?","ok":true,"details":{"answer":"Each owner gets theirs by email, and it appears in the owner portal the same day. [1] Owner statements are sent on the 5th of each month and cover the previous month. [1]","refused":false,"correct":true,"citationSupports":true},"ms":11.897832999999991},{"id":"dP1","kind":"paraphrase","input":"Can I forgive a late charge for a resident?","ok":true,"details":{"answer":"You need the Waive fees permission to do this. [3]","refused":false,"correct":true,"citationSupports":true},"ms":15.661749999999984},{"id":"dP2","kind":"paraphrase","input":"How much warning does a renter get before someone inspects the flat?","ok":true,"details":{"answer":"Tenants get at least 48 hours notice before a routine inspection by default. [1] If local rules require more notice, raise it in Property settings, under Inspections. [1]","refused":false,"correct":true,"citationSupports":true},"ms":12.121708000000012},{"id":"dP3","kind":"paraphrase","input":"When am I reminded that a tenancy is about to end?","ok":false,"details":{"answer":"You will see how many tenants will get the message, and nothing is sent until you confirm. [1]","refused":false,"correct":false,"citationSupports":false},"ms":12.140625},{"id":"dP4","kind":"paraphrase","input":"When do landlords get their monthly report?","ok":false,"details":{"answer":"Open Reports, select Rent roll, then choose the properties and the date. [2]","refused":false,"correct":false,"citationSupports":true},"ms":12.30012499999998},{"id":"dP5","kind":"paraphrase","input":"How many kinds of staff access are there?","ok":false,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true,"correct":false,"citationSupports":false},"ms":6.730042000000026},{"id":"dU1","kind":"unsupported","input":"Is there a QuickBooks integration?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":2.1890409999999747},{"id":"dU2","kind":"unsupported","input":"Can tenants pay rent in bitcoin?","ok":false,"details":{"answer":"Tenants can pay rent in the portal by bank transfer or by debit or credit card. [1] Tenants can pay it in the tenant portal together with the first month's rent. [3]","refused":false},"ms":14.179750000000013},{"id":"dU3","kind":"unsupported","input":"How do I charge a pet fee?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":2.911000000000058},{"id":"dU4","kind":"unsupported","input":"Does the owner portal have a mobile app?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":2.816041999999925},{"id":"dU5","kind":"unsupported","input":"How do I evict a tenant?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":16.8359999999999},{"id":"dN1","kind":"unsupported","input":"What is the weather in Lagos today?","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":2.5066249999999854},{"id":"dN2","kind":"unsupported","input":"Write me a short poem","ok":true,"details":{"answer":"The documentation does not cover this. In a workspace, the question is logged as unanswered so the gap is visible on the Usage page.","refused":true},"ms":2.0170839999999544},{"id":"dR1","kind":"action","input":"Which units are vacant right now?","ok":true,"details":{"operation":"listUnits","expectedOperation":"listUnits","proposed":{"status":"vacant"},"paramsRight":0,"paramsExpected":0},"ms":20.408290999999963},{"id":"dR2","kind":"action","input":"Create a work order for the broken heater in unit 3B","ok":true,"details":{"operation":"createWorkOrder","expectedOperation":"createWorkOrder","proposed":{"unit_id":"3B","description":"the broken heater"},"paramsRight":1,"paramsExpected":1},"ms":3.0846670000000813},{"id":"dR3","kind":"action","input":"Waive the late fee for the tenant in 4B","ok":true,"details":{"operation":"waiveCharge","expectedOperation":"waiveCharge","proposed":{},"paramsRight":0,"paramsExpected":0},"ms":1.7305410000000163},{"id":"dR4","kind":"action","input":"Send a rent reminder to every tenant with a balance","ok":true,"details":{"operation":"sendRentReminder","expectedOperation":"sendRentReminder","proposed":{},"paramsRight":0,"paramsExpected":0},"ms":1.6362920000000258},{"id":"dR5","kind":"action","input":"Show the rent roll for this month","ok":true,"details":{"operation":"getRentRoll","expectedOperation":"getRentRoll","proposed":{},"paramsRight":0,"paramsExpected":0},"ms":1.5100419999999986},{"id":"dM1","kind":"action-missing","input":"Schedule an inspection","ok":true,"details":{"operation":"scheduleInspection","expectedOperation":"scheduleInspection","missing":["unit_id","type","date"],"expectedMissing":["unit_id"],"operationRight":true,"missingListed":true},"ms":2.348792000000003},{"id":"dX1","kind":"action-unsupported","input":"Order lunch for the team","ok":true,"details":{"operation":null,"text":"None of the 10 switched on actions matches that request, so nothing was prepared and nothing was run."},"ms":1.539625000000001},{"id":"dX2","kind":"action-unsupported","input":"Evict the tenant in 2C","ok":true,"details":{"operation":null,"text":"That request is about as close to getTenant as to waiveCharge, so nothing was prepared. Say which one you mean. Nothing was run."},"ms":2.424958000000061},{"id":"dX3","kind":"action-unsupported","input":"Delete the property","ok":true,"details":{"operation":null,"text":"None of the 10 switched on actions matches that request, so nothing was prepared and nothing was run."},"ms":1.8450000000000273},{"id":"dX4","kind":"action-unsupported","input":"Refund the deposit for unit 2C","ok":true,"details":{"operation":null,"text":"The closest operation, refundDeposit, is switched off, so nothing was prepared and nothing was run. It can be switched on in Actions."},"ms":2.407208999999966}]}}