durable_workflow/
cooperative_cancellation.rs

1use super::*;
2use std::collections::BTreeSet;
3
4const CONTROL_BUDGET: Duration = Duration::from_secs(5);
5const MAX_REFRESH_PAGES: usize = 128;
6
7pub(super) fn assert_cancellation_scope_replay_supported(
8    events: &[HistoryEvent],
9    allow_authoring: bool,
10    allow_delivery: bool,
11) -> Result<()> {
12    if allow_delivery && !allow_authoring {
13        return Err(Error::CancellationScopeExecutionUnavailable);
14    }
15    for event in events {
16        let scope_marker = matches!(
17            event.event_type.as_str(),
18            "CancellationScopeRequested"
19                | "CancellationScopeDeliveryPrepared"
20                | "CancellationScopeDelivered"
21                | "CancellationScopeRequestConflicted"
22        );
23        let scoped_membership = std::iter::once(&event.payload)
24            .chain(
25                ["activity", "timer", "child_workflow"]
26                    .iter()
27                    .filter_map(|name| event.payload.get(name)),
28            )
29            .any(|payload| {
30                payload
31                    .get("cancellation_scope_id")
32                    .is_some_and(|scope| scope.as_str() != Some("root"))
33            });
34        if (scope_marker && !allow_delivery)
35            || (!allow_authoring
36                && (event.event_type == "CancellationScopeOpened" || scoped_membership))
37        {
38            return Err(Error::CancellationScopeExecutionUnavailable);
39        }
40    }
41    Ok(())
42}
43
44/// A delivery acknowledgment bound to one workflow task and durable run.
45#[derive(Clone, Debug, PartialEq, Eq)]
46pub struct CancellationDeliveryReceipt {
47    pub task_id: String,
48    pub run_id: String,
49    pub delivery: CancellationDelivery,
50}
51
52/// Delivery either commits or parks the parent until canonical child cleanup.
53#[derive(Clone, Debug, PartialEq, Eq)]
54pub enum CancellationDeliveryReply {
55    Delivered(CancellationDeliveryReceipt),
56    /// The current claim is completed. Return to polling without publishing
57    /// completion or failure. A child terminal event wakes a successor claim.
58    ClaimReleased {
59        task_id: String,
60        run_id: String,
61    },
62}
63
64/// Successful renewal of the exact workflow-task claim with optional observation.
65#[derive(Clone, Debug, PartialEq, Eq)]
66pub struct WorkflowTaskHeartbeat {
67    pub task_id: String,
68    pub workflow_task_attempt: u64,
69    pub lease_owner: String,
70    pub lease_expires_at: String,
71    pub run_status: String,
72    pub cancellation_request: Option<CancellationRequest>,
73}
74
75/// One actual protocol 1.20 claim and its original pending observation.
76///
77/// The claim is read-only. Observing a request does not deliver cancellation to
78/// workflow code. Canonical history and a committed delivery are still required.
79#[derive(Clone, Debug)]
80pub struct CooperativeWorkflowTask {
81    task: WorkflowTask,
82    cancellation_request: Option<CancellationRequest>,
83    sticky_snapshot: Option<sticky_worker::StickySnapshot>,
84}
85
86impl CooperativeWorkflowTask {
87    pub fn task(&self) -> &WorkflowTask {
88        &self.task
89    }
90
91    pub fn cancellation_request(&self) -> Option<&CancellationRequest> {
92        self.cancellation_request.as_ref()
93    }
94
95    /// Renew this exact claim and retain the first observation's identity/token.
96    ///
97    /// A refused or malformed renewal leaves the claim and observation intact.
98    pub async fn heartbeat(&mut self, client: &Client) -> Result<WorkflowTaskHeartbeat> {
99        let receipt = client
100            .heartbeat_workflow_task(&self.task, self.cancellation_request.as_ref())
101            .await?;
102        self.cancellation_request = receipt.cancellation_request.clone();
103        Ok(receipt)
104    }
105}
106
107/// An explicit cooperative poll, including ordinary idle and stop outcomes.
108#[derive(Clone, Debug)]
109pub struct CooperativeWorkflowTaskPoll {
110    pub task: Option<CooperativeWorkflowTask>,
111    pub outcome: WorkerPollOutcome,
112    pub protocol_version: Option<String>,
113    pub server_capabilities: Option<Value>,
114}
115
116/// Optional reason and runtime-owned cleanup limit for a cooperative request.
117#[derive(Clone, Debug, Default, Serialize)]
118pub struct CooperativeCancellationOptions {
119    #[serde(skip_serializing_if = "Option::is_none")]
120    pub reason: Option<String>,
121    #[serde(skip_serializing_if = "Option::is_none")]
122    pub cleanup_timeout_seconds: Option<u64>,
123}
124
125/// The Server's original request identity and immutable cleanup deadline.
126#[derive(Clone, Debug, PartialEq, Eq)]
127pub struct CancellationRequest {
128    pub request_id: String,
129    pub requested_at: String,
130    pub cleanup_deadline_at: String,
131    pub history_refresh_page_token: Option<String>,
132    pub context: Option<CancellationContext>,
133}
134
135/// Acknowledgment of a request targeting the current durable run.
136#[derive(Clone, Debug, PartialEq, Eq)]
137pub struct WorkflowCancellationRequest {
138    pub workflow_id: String,
139    pub run_id: String,
140    pub duplicate: bool,
141    pub cancellation_request: CancellationRequest,
142}
143
144/// Cancellation delivered at its committed authored call, with original identity.
145#[derive(Clone, Debug, Error, PartialEq, Eq)]
146#[error("workflow cancellation {request_id} was requested", request_id = .request.request_id)]
147pub struct CooperativeCancellationRequested {
148    pub request: CancellationRequest,
149    pub delivery: CancellationDelivery,
150}
151
152/// A workflow-local cleanup scope. Dropping it restores cancellation checks.
153///
154/// This does not renew or extend the Server's original cleanup deadline.
155#[derive(Debug)]
156pub struct CancellationShield {
157    state: Arc<Mutex<WorkflowState>>,
158}
159
160impl Drop for CancellationShield {
161    fn drop(&mut self) {
162        if let Ok(mut state) = self.state.lock() {
163            state.cancellation_shield_depth = state.cancellation_shield_depth.saturating_sub(1);
164        }
165    }
166}
167
168fn invalid(detail: impl Into<String>) -> Error {
169    Error::InvalidCooperativeCancellation(detail.into())
170}
171
172fn text<'a>(value: &'a Value, field: &str) -> Result<&'a str> {
173    value
174        .get(field)
175        .and_then(Value::as_str)
176        .filter(|text| !text.trim().is_empty())
177        .ok_or_else(|| invalid(format!("{field} must be a non-empty string")))
178}
179
180impl CancellationRequest {
181    pub(super) fn preserve_observation(&self, current: &Self) -> Result<Self> {
182        self.validate_observation()?;
183        current.validate_observation()?;
184        if self.request_id != current.request_id
185            || DateTime::parse_from_rfc3339(&self.requested_at).unwrap()
186                != DateTime::parse_from_rfc3339(&current.requested_at).unwrap()
187            || DateTime::parse_from_rfc3339(&self.cleanup_deadline_at).unwrap()
188                != DateTime::parse_from_rfc3339(&current.cleanup_deadline_at).unwrap()
189        {
190            return Err(invalid(
191                "observation changed the original request or cleanup deadline",
192            ));
193        }
194        Ok(self.clone())
195    }
196
197    pub(super) fn validate_observation(&self) -> Result<()> {
198        Self::from_observation(&json!({
199            "request_id":self.request_id, "requested_at":self.requested_at,
200            "cleanup_deadline_at":self.cleanup_deadline_at,
201            "history_refresh_page_token":self.history_refresh_page_token,
202        }))
203        .map(|_| ())
204    }
205
206    pub(crate) fn from_observation(value: &Value) -> Result<Self> {
207        let requested_at = text(value, "requested_at")?;
208        let cleanup_deadline_at = text(value, "cleanup_deadline_at")?;
209        let requested = DateTime::parse_from_rfc3339(requested_at)
210            .map_err(|_| invalid("requested_at must be a timestamp with a timezone"))?;
211        let deadline = DateTime::parse_from_rfc3339(cleanup_deadline_at)
212            .map_err(|_| invalid("cleanup_deadline_at must be a timestamp with a timezone"))?;
213        if deadline <= requested {
214            return Err(invalid("cleanup deadline must follow the original request"));
215        }
216        Ok(Self {
217            request_id: text(value, "request_id")?.to_owned(),
218            requested_at: requested_at.to_owned(),
219            cleanup_deadline_at: cleanup_deadline_at.to_owned(),
220            history_refresh_page_token: Some(text(value, "history_refresh_page_token")?.to_owned()),
221            context: None,
222        })
223    }
224}
225
226const REQUEST_EVENT: &str = "CooperativeCancellationRequested";
227const DELIVERY_EVENT: &str = "CooperativeCancellationDelivered";
228const MAX_SEQUENCE: u64 = i64::MAX as u64;
229
230/// The authored durable call where canonical cancellation is delivered.
231#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
232#[serde(rename_all = "snake_case")]
233pub enum CancellationCallKind {
234    Activity,
235    LocalActivity,
236    Timer,
237    Condition,
238    Signal,
239    Child,
240    Parallel,
241    SelectionHandle,
242}
243
244/// One committed delivery marker, bound to the original cancellation request.
245#[derive(Clone, Debug, PartialEq, Eq)]
246pub struct CancellationDelivery {
247    pub request_id: String,
248    pub sequence: u64,
249    pub call_kind: CancellationCallKind,
250    pub sequence_span: u64,
251    pub operation_sequence: Option<u64>,
252    pub operation_sequence_span: u64,
253}
254
255fn positive(value: &Value, field: &str, maximum: u64) -> Result<u64> {
256    value
257        .as_u64()
258        .filter(|number| (1..=maximum).contains(number))
259        .ok_or_else(|| {
260            invalid(format!(
261                "{field} must be a positive integer within {maximum}"
262            ))
263        })
264}
265
266impl CancellationDelivery {
267    pub(crate) fn from_payload(value: &Value) -> Result<Self> {
268        let sequence = positive(&value["sequence"], "sequence", MAX_SEQUENCE)?;
269        let call_kind: CancellationCallKind = serde_json::from_value(value["call_kind"].clone())
270            .map_err(|_| invalid("delivery must name a supported durable call kind"))?;
271        let span = positive(
272            value.get("sequence_span").unwrap_or(&json!(1)),
273            "sequence_span",
274            1000,
275        )?;
276        let operation_span = positive(
277            value.get("operation_sequence_span").unwrap_or(&json!(1)),
278            "operation_sequence_span",
279            1000,
280        )?;
281        let operation = value
282            .get("operation_sequence")
283            .filter(|value| !value.is_null());
284        if (call_kind != CancellationCallKind::Parallel && span != 1)
285            || sequence > MAX_SEQUENCE - span
286        {
287            return Err(invalid(
288                "delivery call span is invalid or overflows the portable sequence range",
289            ));
290        }
291        let operation_sequence = if call_kind == CancellationCallKind::SelectionHandle {
292            let base = positive(
293                operation.unwrap_or(&Value::Null),
294                "operation_sequence",
295                MAX_SEQUENCE,
296            )?;
297            if base >= sequence || operation_span > sequence - base {
298                return Err(invalid(
299                    "selection handle must name a complete earlier operation range",
300                ));
301            }
302            Some(base)
303        } else {
304            if operation.is_some() || operation_span != 1 {
305                return Err(invalid(
306                    "only selection handles may name an operation range",
307                ));
308            }
309            None
310        };
311        Ok(Self {
312            request_id: text(value, "workflow_command_id")?.to_owned(),
313            sequence,
314            call_kind,
315            sequence_span: span,
316            operation_sequence,
317            operation_sequence_span: operation_span,
318        })
319    }
320
321    fn range(&self) -> (u64, u64) {
322        self.operation_sequence
323            .map_or((self.sequence, self.sequence_span), |base| {
324                (base, self.operation_sequence_span)
325            })
326    }
327
328    /// Whether a sequence belongs to the canonical interrupted operation range.
329    pub fn interrupts(&self, sequence: u64) -> bool {
330        let (base, span) = self.range();
331        sequence >= base && sequence - base < span
332    }
333}
334
335/// Validated canonical cancellation state for one workflow run.
336///
337/// An observation identifies a request. Only its committed delivery marker
338/// authorizes interruption at an authored call. Earlier results retain priority.
339#[derive(Clone, Debug)]
340pub struct CancellationHistory {
341    pub request: Option<CancellationRequest>,
342    pub delivery: Option<CancellationDelivery>,
343    pub request_index: usize,
344    pub delivery_index: Option<usize>,
345    resolved_before_request: BTreeSet<u64>,
346    latest_condition_sequences_before_request: BTreeMap<u64, u64>,
347    failed_before_request: BTreeSet<u64>,
348    selected_before_request: BTreeSet<(u64, u64)>,
349}
350
351impl CancellationHistory {
352    pub(super) fn bind_commands(
353        &self,
354        mut commands: Vec<RecordedCommand>,
355    ) -> Result<Vec<RecordedCommand>> {
356        let Some(delivery) = &self.delivery else {
357            return Ok(commands);
358        };
359        if delivery.call_kind == CancellationCallKind::Parallel {
360            let index = commands.partition_point(|command| command.sequence() < delivery.sequence);
361            let end = commands.partition_point(|command| {
362                command.sequence() < delivery.sequence + delivery.sequence_span
363            });
364            if commands.get(index).map(RecordedCommand::sequence) != Some(delivery.sequence) {
365                let previous = index
366                    .checked_sub(1)
367                    .map_or(0, |index| commands[index].sequence());
368                if previous.checked_add(1) != Some(delivery.sequence) {
369                    return Err(invalid_recorded_history(
370                        "cooperative_cancellation_call_mismatch",
371                        delivery.sequence,
372                        "next authored parallel group",
373                        "missing earlier call",
374                        "parallel cancellation marker skips an unrecorded authored command",
375                    ));
376                }
377            }
378            let original = commands.drain(index..end).collect();
379            commands.insert(
380                index,
381                RecordedCommand::CancellationGroup {
382                    sequence: delivery.sequence,
383                    span: delivery.sequence_span,
384                    original,
385                },
386            );
387            // Physical condition reopens retain the group's authored identity
388            // but can lie beyond its original sequence span. Bind unresolved
389            // reopens as well, so replay consumes the latest wait at delivery.
390            for command in &mut commands[index + 1..] {
391                let RecordedCommand::ConditionWait {
392                    sequence,
393                    parallel_group_path: Some(path),
394                    ..
395                } = command
396                else {
397                    continue;
398                };
399                if self.resolved_before_request.contains(sequence)
400                    || !path.first().is_some_and(|group| {
401                        group.parallel_group_base_sequence == delivery.sequence
402                            && group.parallel_group_size as u64 == delivery.sequence_span
403                    })
404                {
405                    continue;
406                }
407                *command = RecordedCommand::CancellationBoundary {
408                    sequence: *sequence,
409                    call_kind: CancellationCallKind::Condition,
410                    original: Some(Box::new(command.clone())),
411                };
412            }
413            return Ok(commands);
414        }
415        if !matches!(
416            delivery.call_kind,
417            CancellationCallKind::Activity
418                | CancellationCallKind::Timer
419                | CancellationCallKind::Condition
420                | CancellationCallKind::Signal
421                | CancellationCallKind::Child
422                | CancellationCallKind::SelectionHandle
423        ) {
424            return Ok(commands);
425        }
426        let index = commands.partition_point(|command| command.sequence() < delivery.sequence);
427        let original = if commands
428            .get(index)
429            .is_some_and(|command| command.sequence() == delivery.sequence)
430        {
431            Some(Box::new(commands.remove(index)))
432        } else {
433            let previous = index
434                .checked_sub(1)
435                .map_or(0, |index| commands[index].sequence());
436            if previous.checked_add(1) != Some(delivery.sequence) {
437                return Err(invalid_recorded_history(
438                    "cooperative_cancellation_call_mismatch",
439                    delivery.sequence,
440                    "next authored durable call",
441                    "missing earlier call",
442                    "cancellation marker skips an unrecorded authored command",
443                ));
444            }
445            None
446        };
447        commands.insert(
448            index,
449            RecordedCommand::CancellationBoundary {
450                sequence: delivery.sequence,
451                call_kind: delivery.call_kind,
452                original,
453            },
454        );
455        Ok(commands)
456    }
457
458    /// Read canonical markers without replacing original request identity.
459    pub fn from_events(
460        events: &[HistoryEvent],
461        run_id: &str,
462        observation: Option<&CancellationRequest>,
463    ) -> Result<Self> {
464        Self::read(events, run_id, observation).map_err(|error| {
465            invalid_recorded_history(
466                "cooperative_cancellation_history_invalid",
467                0,
468                "one canonical request and matching authored delivery",
469                "invalid cancellation history",
470                &error.to_string(),
471            )
472        })
473    }
474
475    fn read(
476        events: &[HistoryEvent],
477        run_id: &str,
478        observation: Option<&CancellationRequest>,
479    ) -> Result<Self> {
480        if let Some(observed) = observation {
481            let requested = DateTime::parse_from_rfc3339(&observed.requested_at)
482                .map_err(|_| invalid("observed request timestamp is invalid"))?;
483            let deadline = DateTime::parse_from_rfc3339(&observed.cleanup_deadline_at)
484                .map_err(|_| invalid("observed cleanup deadline is invalid"))?;
485            if observed.request_id.trim().is_empty()
486                || deadline <= requested
487                || observed
488                    .history_refresh_page_token
489                    .as_ref()
490                    .is_some_and(|token| token.trim().is_empty())
491            {
492                return Err(invalid(
493                    "observed request identity, deadline or history token is invalid",
494                ));
495            }
496        }
497        let mut state = Self {
498            request: observation.cloned().map(|mut request| {
499                request.context = None;
500                request
501            }),
502            delivery: None,
503            request_index: events.len(),
504            delivery_index: None,
505            resolved_before_request: BTreeSet::new(),
506            latest_condition_sequences_before_request: BTreeMap::new(),
507            failed_before_request: BTreeSet::new(),
508            selected_before_request: BTreeSet::new(),
509        };
510        let mut saw_request = false;
511        for (index, event) in events.iter().enumerate() {
512            if !matches!(event.event_type.as_str(), REQUEST_EVENT | DELIVERY_EVENT) {
513                continue;
514            }
515            let request_id = text(&event.payload, "workflow_command_id")?;
516            let event_run = text(&event.payload, "workflow_run_id")?;
517            if event
518                .raw
519                .get("workflow_command_id")
520                .is_some_and(|value| value.as_str() != Some(request_id))
521                || (!run_id.is_empty() && event_run != run_id)
522            {
523                return Err(invalid(
524                    "canonical event does not match its request or workflow run",
525                ));
526            }
527            if event.event_type == REQUEST_EVENT {
528                if saw_request || state.delivery.is_some() {
529                    return Err(invalid("history must contain one request before delivery"));
530                }
531                let recorded_at = event
532                    .raw
533                    .get("recorded_at")
534                    .or_else(|| event.raw.get("timestamp"))
535                    .and_then(Value::as_str)
536                    .ok_or_else(|| invalid("canonical request lacks its recorded timestamp"))?;
537                let requested = DateTime::parse_from_rfc3339(recorded_at)
538                    .map_err(|_| invalid("canonical request timestamp is invalid"))?;
539                let deadline_text = text(&event.payload, "cleanup_deadline_at")?;
540                let deadline = DateTime::parse_from_rfc3339(deadline_text)
541                    .map_err(|_| invalid("canonical cleanup deadline is invalid"))?;
542                let context = event
543                    .payload
544                    .get("cancellation")
545                    .map(CancellationContext::from_value)
546                    .transpose()?;
547                if let Some(context) = &context {
548                    let local = context.lineage().last().unwrap();
549                    if context.request_id() != request_id
550                        || local.workflow_run_id() != event_run
551                        || event
552                            .payload
553                            .get("workflow_instance_id")
554                            .is_some_and(|instance| {
555                                instance.as_str() != Some(local.workflow_instance_id())
556                            })
557                        || context.deadline() != deadline
558                        || context.requested_at() > requested
559                        || event
560                            .payload
561                            .get("reason")
562                            .is_some_and(|reason| match reason {
563                                Value::Null => context.reason().is_some(),
564                                Value::String(reason) => Some(reason.as_str()) != context.reason(),
565                                _ => true,
566                            })
567                    {
568                        return Err(invalid(
569                            "canonical cancellation context does not match its request event",
570                        ));
571                    }
572                } else if deadline <= requested {
573                    return Err(invalid(
574                        "canonical cleanup deadline must follow the request",
575                    ));
576                }
577                if let Some(observed) = observation {
578                    let observed_deadline =
579                        DateTime::parse_from_rfc3339(&observed.cleanup_deadline_at)
580                            .map_err(|_| invalid("observed cleanup deadline is invalid"))?;
581                    if observed.request_id != request_id || observed_deadline != deadline {
582                        return Err(invalid(
583                            "observation changes the original request or cleanup deadline",
584                        ));
585                    }
586                }
587                if context.is_some() || observation.is_none() {
588                    state.request = Some(CancellationRequest {
589                        request_id: request_id.to_owned(),
590                        requested_at: context.as_ref().map_or_else(
591                            || recorded_at.to_owned(),
592                            |context| {
593                                context
594                                    .requested_at()
595                                    .to_rfc3339_opts(chrono::SecondsFormat::Micros, true)
596                            },
597                        ),
598                        cleanup_deadline_at: deadline_text.to_owned(),
599                        history_refresh_page_token: observation
600                            .and_then(|request| request.history_refresh_page_token.clone()),
601                        context,
602                    });
603                }
604                state.request_index = index;
605                saw_request = true;
606            } else {
607                if !saw_request || state.delivery.is_some() {
608                    return Err(invalid(
609                        "delivery requires one earlier request and one marker",
610                    ));
611                }
612                let delivery = CancellationDelivery::from_payload(&event.payload)?;
613                if state
614                    .request
615                    .as_ref()
616                    .map(|request| request.request_id.as_str())
617                    != Some(delivery.request_id.as_str())
618                {
619                    return Err(invalid("delivery names a different original request"));
620                }
621                if let Some(snapshot) = event.payload.get("cancellation") {
622                    let context = CancellationContext::from_value(snapshot)?;
623                    if state
624                        .request
625                        .as_ref()
626                        .and_then(|request| request.context.as_ref())
627                        != Some(&context)
628                    {
629                        return Err(invalid(
630                            "delivery changes the canonical cancellation context",
631                        ));
632                    }
633                }
634                state.delivery = Some(delivery);
635                state.delivery_index = Some(index);
636            }
637        }
638        if state.request.is_none() {
639            return Ok(state);
640        }
641        let mut condition_occurrences = BTreeMap::new();
642        for event in &events[..state.request_index] {
643            if matches!(
644                event.event_type.as_str(),
645                "SelectionResolved" | "SelectionOperationCancelled"
646            ) {
647                let (base_field, span_field) = if event.event_type == "SelectionResolved" {
648                    ("selection_group_base_sequence", "selection_group_size")
649                } else {
650                    ("member_base_sequence", "member_size")
651                };
652                if let (Ok(base), Ok(span)) = (
653                    positive(&event.payload[base_field], base_field, MAX_SEQUENCE),
654                    positive(&event.payload[span_field], span_field, 1000),
655                ) {
656                    if base <= MAX_SEQUENCE - span {
657                        if event.event_type == "SelectionResolved" {
658                            state.selected_before_request.insert((base, span));
659                        } else {
660                            state.resolved_before_request.extend(base..base + span);
661                        }
662                    }
663                }
664            }
665            let Ok(sequence) = positive(&event.payload["sequence"], "sequence", MAX_SEQUENCE)
666            else {
667                continue;
668            };
669            // Groups deliver at their original authored span. Scalar condition
670            // delivery already names the pending physical sequence directly.
671            if event.event_type == "ConditionWaitOpened"
672                && (event.payload["parallel_group_id"].is_string()
673                    || event.payload["parallel_group_path"]
674                        .as_array()
675                        .is_some_and(|path| !path.is_empty()))
676            {
677                if let Some(occurrence) = event.payload["condition_wait_occurrence_id"].as_str() {
678                    let authored = *condition_occurrences.entry(occurrence).or_insert(sequence);
679                    state
680                        .latest_condition_sequences_before_request
681                        .insert(authored, sequence);
682                }
683            }
684            if matches!(
685                event.event_type.as_str(),
686                "ActivityCompleted"
687                    | "ActivityFailed"
688                    | "ActivityCancelled"
689                    | "ActivityTimedOut"
690                    | "TimerFired"
691                    | "TimerCancelled"
692                    | "ConditionWaitSatisfied"
693                    | "ConditionWaitTimedOut"
694                    | "SignalApplied"
695                    | "ChildRunCompleted"
696                    | "ChildRunFailed"
697                    | "ChildRunCancelled"
698                    | "ChildRunTerminated"
699            ) {
700                state.resolved_before_request.insert(sequence);
701                if matches!(
702                    event.event_type.as_str(),
703                    "ActivityFailed"
704                        | "ActivityCancelled"
705                        | "ActivityTimedOut"
706                        | "ChildRunFailed"
707                        | "ChildRunCancelled"
708                        | "ChildRunTerminated"
709                ) {
710                    state.failed_before_request.insert(sequence);
711                }
712            }
713        }
714        if let Some(delivery) = &state.delivery {
715            let (base, span) = delivery.range();
716            if !state.range_eligible(base, span) {
717                return Err(invalid(
718                    "delivery cannot replace an earlier committed result",
719                ));
720            }
721        }
722        Ok(state)
723    }
724
725    fn range_eligible(&self, sequence: u64, span: u64) -> bool {
726        (1..=1000).contains(&span)
727            && sequence > 0
728            && sequence <= MAX_SEQUENCE - span
729            && !(sequence..sequence + span).all(|sequence| {
730                let latest = self
731                    .latest_condition_sequences_before_request
732                    .get(&sequence)
733                    .unwrap_or(&sequence);
734                self.resolved_before_request.contains(latest)
735            })
736            && !(sequence..sequence + span)
737                .any(|sequence| self.failed_before_request.contains(&sequence))
738            && !self.selected_before_request.contains(&(sequence, span))
739    }
740
741    /// Whether an unresolved authored call may deliver the pending request.
742    pub fn eligible(&self, sequence: u64, span: u64) -> bool {
743        self.request.is_some() && self.delivery.is_none() && self.range_eligible(sequence, span)
744    }
745}
746
747impl WorkflowState {
748    fn next_cancellation_sequence(&self) -> Result<u64> {
749        self.recorded_commands.get(self.command_cursor).map_or_else(
750            || {
751                self.recorded_commands
752                    .last()
753                    .map_or(0, RecordedCommand::sequence)
754                    .checked_add(self.commands.len() as u64)
755                    .and_then(|sequence| sequence.checked_add(1))
756                    .filter(|sequence| *sequence < MAX_SEQUENCE)
757                    .ok_or_else(|| invalid("authored cancellation sequence overflowed"))
758            },
759            |command| Ok(command.sequence()),
760        )
761    }
762
763    fn prepare_cancellation_delivery(&mut self, delivery: CancellationDelivery) -> Result<bool> {
764        if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 {
765            return Ok(false);
766        }
767        self.assert_root_cancellation_membership()?;
768        if self.cancellation_delivery_intent.is_some() {
769            self.matched_recorded_pending = true;
770            return Ok(true);
771        }
772        let (base, span) = delivery.range();
773        if !self.cancellation_history.eligible(base, span) {
774            return Ok(false);
775        }
776        let payload = json!({
777            "workflow_command_id":delivery.request_id, "sequence":delivery.sequence,
778            "call_kind":delivery.call_kind, "sequence_span":delivery.sequence_span,
779            "operation_sequence":delivery.operation_sequence,
780            "operation_sequence_span":delivery.operation_sequence_span,
781        });
782        CancellationDelivery::from_payload(&payload)?;
783        self.cancellation_delivery_command_count = self.commands.len();
784        self.cancellation_delivery_intent = Some(delivery);
785        self.matched_recorded_pending = true;
786        Ok(true)
787    }
788
789    fn assert_root_cancellation_membership(&self) -> Result<()> {
790        if self
791            .scope_delivery
792            .as_ref()
793            .is_some_and(|replay| replay.active_scope != "root")
794        {
795            return Err(Error::CancellationScopeExecutionUnavailable);
796        }
797        Ok(())
798    }
799
800    pub(super) fn prepare_scalar_cancellation(
801        &mut self,
802        index: usize,
803        kind: CancellationCallKind,
804        group_path: &[ParallelGroupMetadata],
805    ) -> Result<bool> {
806        if self.prepare_scalar_scope_cancellation(index, kind, group_path)? {
807            return Ok(true);
808        }
809        if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 {
810            return Ok(false);
811        }
812        // Group leaves validate their recorded calls before the enclosing
813        // parallel/select future suspends. They never choose scalar delivery.
814        if !group_path.is_empty() && self.cancellation_delivery_intent.is_none() {
815            return Ok(false);
816        }
817        let Some(request) = self.cancellation_history.request.as_ref() else {
818            return Ok(false);
819        };
820        let sequence = self.recorded_commands.get(index).map_or_else(
821            || self.next_cancellation_sequence(),
822            |command| Ok(command.sequence()),
823        )?;
824        let pending = self.prepare_cancellation_delivery(CancellationDelivery {
825            request_id: request.request_id.clone(),
826            sequence,
827            call_kind: kind,
828            sequence_span: 1,
829            operation_sequence: None,
830            operation_sequence_span: 1,
831        })?;
832        if pending && index < self.recorded_commands.len() {
833            self.command_cursor = index + 1;
834        }
835        Ok(pending)
836    }
837
838    pub(super) fn prepare_group_cancellation(
839        &mut self,
840        descriptors: &[ParallelDescriptor],
841    ) -> Result<()> {
842        let Some(request) = self.cancellation_history.request.as_ref() else {
843            return Ok(());
844        };
845        let Some(group) = descriptors.first().and_then(|leaf| leaf.group_path.first()) else {
846            return Ok(());
847        };
848        self.prepare_cancellation_delivery(CancellationDelivery {
849            request_id: request.request_id.clone(),
850            sequence: group.parallel_group_base_sequence,
851            call_kind: CancellationCallKind::Parallel,
852            sequence_span: descriptors.len() as u64,
853            operation_sequence: None,
854            operation_sequence_span: 1,
855        })?;
856        Ok(())
857    }
858
859    pub(super) fn prepare_selection_handle_cancellation(
860        &mut self,
861        handle: &DurableOperationHandle,
862    ) -> Result<bool> {
863        if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 {
864            return Ok(false);
865        }
866        validate_selection_delivery_handle(self, handle)?;
867        if !self
868            .cancellation_history
869            .eligible(handle.base_sequence, handle.size as u64)
870        {
871            return Ok(false);
872        }
873        let request_id = self
874            .cancellation_history
875            .request
876            .as_ref()
877            .expect("eligible request")
878            .request_id
879            .clone();
880        self.prepare_cancellation_delivery(CancellationDelivery {
881            request_id,
882            sequence: self.next_cancellation_sequence()?,
883            call_kind: CancellationCallKind::SelectionHandle,
884            sequence_span: 1,
885            operation_sequence: Some(handle.base_sequence),
886            operation_sequence_span: handle.size as u64,
887        })
888    }
889
890    pub(super) fn expand_cancellation_group(
891        &mut self,
892        descriptors: &[ParallelDescriptor],
893    ) -> Result<()> {
894        let Some(RecordedCommand::CancellationGroup {
895            sequence,
896            span,
897            original,
898        }) = self.recorded_commands.get(self.command_cursor).cloned()
899        else {
900            return Ok(());
901        };
902        self.validate_cancellation_call(
903            sequence,
904            CancellationCallKind::Parallel,
905            CancellationCallKind::Parallel,
906        )?;
907        if descriptors.len() as u64 != span {
908            return Err(invalid_recorded_history(
909                "cooperative_cancellation_call_mismatch",
910                sequence,
911                &format!("parallel group with {span} durable leaves"),
912                &format!("{} durable leaves", descriptors.len()),
913                "authored parallel group span differs from its committed cancellation",
914            ));
915        }
916        let mut original = original.into_iter().peekable();
917        let mut leaves = Vec::with_capacity(descriptors.len());
918        for descriptor in descriptors {
919            let leaf_sequence = sequence + descriptor.offset as u64;
920            let command = if original.peek().map(RecordedCommand::sequence) == Some(leaf_sequence) {
921                original.next()
922            } else {
923                None
924            };
925            if self
926                .cancellation_history
927                .resolved_before_request
928                .contains(&leaf_sequence)
929            {
930                let command = command.ok_or_else(|| {
931                    invalid_recorded_history(
932                        "cooperative_cancellation_call_mismatch",
933                        leaf_sequence,
934                        "recorded earlier result",
935                        "missing durable call",
936                        "parallel cancellation cannot discard an earlier committed result",
937                    )
938                })?;
939                leaves.push(command);
940                continue;
941            }
942            let kind = match &descriptor.operation {
943                ParallelOperation::Activity { .. } => CancellationCallKind::Activity,
944                ParallelOperation::ChildWorkflow { .. } => CancellationCallKind::Child,
945                ParallelOperation::Timer(_) => CancellationCallKind::Timer,
946                ParallelOperation::Signal(_) => CancellationCallKind::Signal,
947                ParallelOperation::Condition { .. } => CancellationCallKind::Condition,
948                ParallelOperation::Group(_) => unreachable!("descriptor is a durable leaf"),
949            };
950            leaves.push(RecordedCommand::CancellationBoundary {
951                sequence: leaf_sequence,
952                call_kind: kind,
953                original: command.map(Box::new),
954            });
955        }
956        self.recorded_commands
957            .splice(self.command_cursor..=self.command_cursor, leaves);
958        Ok(())
959    }
960
961    pub(super) fn cancellation_error(&mut self) -> Error {
962        if self.cancellation_consumed {
963            self.start_cancellation_clock();
964        }
965        match (
966            &self.cancellation_history.request,
967            &self.cancellation_history.delivery,
968        ) {
969            (Some(request), Some(delivery)) => {
970                let mut request = request.clone();
971                request.context = request.context.map(|context| {
972                    context.with_replay(cancellation_replay_clock::active_binding())
973                });
974                Error::CooperativeCancellationRequested(CooperativeCancellationRequested {
975                    request,
976                    delivery: delivery.clone(),
977                })
978            }
979            _ => Error::WorkflowCancellationRequested(WorkflowCancellationRequested),
980        }
981    }
982
983    fn validate_cancellation_call(
984        &self,
985        sequence: u64,
986        kind: CancellationCallKind,
987        recorded_kind: CancellationCallKind,
988    ) -> Result<()> {
989        if kind != recorded_kind || self.cancellation_shield_depth > 0 {
990            return Err(invalid_recorded_history(
991                "cooperative_cancellation_call_mismatch",
992                sequence,
993                "matching unshielded authored call",
994                &format!("{kind:?}"),
995                "committed cancellation call kind or cleanup scope changed",
996            ));
997        }
998        Ok(())
999    }
1000
1001    pub(super) fn cancellation_replay_command(
1002        &mut self,
1003        index: usize,
1004        kind: CancellationCallKind,
1005    ) -> Result<Option<RecordedCommand>> {
1006        match self.recorded_commands.get(index).cloned() {
1007            Some(RecordedCommand::CancellationBoundary {
1008                sequence,
1009                call_kind,
1010                original,
1011            }) => {
1012                if let Some(original) = original {
1013                    return Ok(Some(*original));
1014                }
1015                self.replay_scope_cancellation_at(index, kind)?;
1016                self.assert_root_cancellation_membership()?;
1017                self.validate_cancellation_call(sequence, kind, call_kind)?;
1018                self.cancellation_consumed = true;
1019                self.cancel_requested = true;
1020                self.command_cursor = index + 1;
1021                Err(self.cancellation_error())
1022            }
1023            command => Ok(command),
1024        }
1025    }
1026
1027    pub(super) fn replay_cancellation_at(
1028        &mut self,
1029        index: usize,
1030        kind: CancellationCallKind,
1031    ) -> Result<()> {
1032        self.replay_scope_cancellation_at(index, kind)?;
1033        if let Some(RecordedCommand::CancellationBoundary {
1034            sequence,
1035            call_kind,
1036            ..
1037        }) = self.recorded_commands.get(index)
1038        {
1039            self.assert_root_cancellation_membership()?;
1040            self.validate_cancellation_call(*sequence, kind, *call_kind)?;
1041            self.cancellation_consumed = true;
1042            self.cancel_requested = true;
1043            self.command_cursor = index + 1;
1044            return Err(self.cancellation_error());
1045        }
1046        Ok(())
1047    }
1048
1049    pub(super) fn replay_selection_handle_cancellation(
1050        &mut self,
1051        handle: &DurableOperationHandle,
1052    ) -> Result<()> {
1053        let Some(RecordedCommand::CancellationBoundary {
1054            sequence,
1055            call_kind,
1056            original,
1057        }) = self.recorded_commands.get(self.command_cursor)
1058        else {
1059            return Ok(());
1060        };
1061        self.validate_cancellation_call(
1062            *sequence,
1063            CancellationCallKind::SelectionHandle,
1064            *call_kind,
1065        )?;
1066        let delivery = self
1067            .cancellation_history
1068            .delivery
1069            .as_ref()
1070            .expect("bound canonical delivery");
1071        if original.is_some()
1072            || delivery.operation_sequence != Some(handle.base_sequence)
1073            || delivery.operation_sequence_span != handle.size as u64
1074        {
1075            return Err(invalid_recorded_history(
1076                "cooperative_cancellation_call_mismatch",
1077                *sequence,
1078                "selection handle for the committed operation range",
1079                &format!("{}:{}", handle.base_sequence, handle.size),
1080                "cancellation delivery targets a different authored selection member",
1081            ));
1082        }
1083        validate_selection_delivery_handle(self, handle)?;
1084        self.replay_cancellation_at(self.command_cursor, CancellationCallKind::SelectionHandle)
1085    }
1086}
1087
1088impl WorkflowContext {
1089    /// Original immutable metadata, visible only at committed delivery.
1090    pub fn cancellation_context(&self) -> Result<Option<CancellationContext>> {
1091        let state = self
1092            .state
1093            .lock()
1094            .map_err(|_| Error::WorkflowStatePoisoned)?;
1095        Ok(
1096            if state.cancellation_consumed && self.cancellation_scope_id == "root" {
1097                state
1098                    .cancellation_history
1099                    .request
1100                    .as_ref()
1101                    .and_then(|request| request.context.clone())
1102                    .map(|context| context.with_replay(Some(Arc::downgrade(&self.state))))
1103            } else {
1104                None
1105            },
1106        )
1107    }
1108
1109    /// Shield explicit cleanup from repeated cancellation checks.
1110    ///
1111    /// Keep the returned guard alive across cleanup awaits. Scopes can nest.
1112    /// The Server retains and enforces the original immutable cleanup deadline.
1113    pub fn cancellation_shield(&self) -> Result<CancellationShield> {
1114        let mut state = self
1115            .state
1116            .lock()
1117            .map_err(|_| Error::WorkflowStatePoisoned)?;
1118        state.cancellation_shield_depth = state
1119            .cancellation_shield_depth
1120            .checked_add(1)
1121            .ok_or_else(|| invalid("cancellation shield nesting overflowed"))?;
1122        Ok(CancellationShield {
1123            state: Arc::clone(&self.state),
1124        })
1125    }
1126}
1127
1128pub(super) fn supports_protocol(version: &str) -> bool {
1129    let Some((major, minor)) = version.split_once('.') else {
1130        return false;
1131    };
1132    major == "1"
1133        && !minor.is_empty()
1134        && minor.bytes().all(|byte| byte.is_ascii_digit())
1135        && minor.parse::<u64>().is_ok_and(|minor| minor >= 20)
1136}
1137
1138fn require_discovery(info: &Value) -> Result<()> {
1139    let protocol = &info["worker_protocol"];
1140    if protocol["server_capabilities"]["cooperative_cancellation"].as_bool() != Some(true)
1141        || !protocol["version"].as_str().is_some_and(supports_protocol)
1142    {
1143        return Err(Error::CooperativeCancellationUnavailable(
1144            "runtime discovery must explicitly advertise support and compatible protocol 1.20"
1145                .to_string(),
1146        ));
1147    }
1148    Ok(())
1149}
1150
1151fn acknowledgment(
1152    value: &Value,
1153    workflow_id: &str,
1154    selected_run: Option<&str>,
1155) -> Result<WorkflowCancellationRequest> {
1156    if value["accepted"].as_bool() != Some(true)
1157        || text(value, "workflow_id")? != workflow_id
1158        || selected_run.is_some_and(|run| value["run_id"].as_str() != Some(run))
1159    {
1160        return Err(invalid(
1161            "acknowledgment does not match the accepted workflow/run request",
1162        ));
1163    }
1164    Ok(WorkflowCancellationRequest {
1165        workflow_id: workflow_id.to_owned(),
1166        run_id: text(value, "run_id")?.to_owned(),
1167        duplicate: value["duplicate"]
1168            .as_bool()
1169            .ok_or_else(|| invalid("duplicate must be a boolean"))?,
1170        cancellation_request: CancellationRequest::from_observation(
1171            &value["cancellation_request"],
1172        )?,
1173    })
1174}
1175
1176impl Client {
1177    /// Report a stopped and dropped remote callback under its original claim.
1178    ///
1179    /// Call only after callback execution has ended. This explicit protocol 1.20
1180    /// operation preserves the original cancellation request and grants no lease
1181    /// or result publication authority. The request has one five-second budget.
1182    pub async fn acknowledge_activity_cancellation(
1183        &self,
1184        task_id: &str,
1185        activity_attempt_id: &str,
1186        lease_owner: &str,
1187        request_id: &str,
1188    ) -> Result<Value> {
1189        if [task_id, activity_attempt_id, lease_owner, request_id]
1190            .iter()
1191            .any(|value| value.trim().is_empty() || value.len() > 255)
1192        {
1193            return Err(invalid(
1194                "activity stop receipt requires bounded original claim and request identities",
1195            ));
1196        }
1197        tokio::time::timeout(CONTROL_BUDGET, async {
1198            let value: Value = activity_task_response(
1199                self.request_json(
1200                    reqwest::Method::POST,
1201                    &format!("/worker/activity-tasks/{}/acknowledge-cancellation", percent_encode_path_segment(task_id)),
1202                    RequestProtocol::Worker("1.20"),
1203                    Some(&json!({"activity_attempt_id":activity_attempt_id,"lease_owner":lease_owner,"request_id":request_id})),
1204                ).await,
1205                "acknowledge-cancellation", task_id, activity_attempt_id,
1206            )?;
1207            if value["task_id"].as_str() != Some(task_id)
1208                || value["activity_attempt_id"].as_str() != Some(activity_attempt_id)
1209                || value["lease_owner"].as_str() != Some(lease_owner)
1210                || value["request_id"].as_str() != Some(request_id)
1211                || value["acknowledged"].as_bool() != Some(true)
1212                || value["duplicate"].as_bool().is_none()
1213                || value.get("reason") != Some(&Value::Null)
1214                || value["heartbeat_recorded"].as_bool() != Some(false)
1215                || text(&value, "history_event_id").is_err()
1216            {
1217                return Err(invalid("activity callback-stop acknowledgment did not prove the original claim and receipt"));
1218            }
1219            Ok(value)
1220        }).await.map_err(|_| Error::Timeout)?
1221    }
1222
1223    /// Observe an exact activity attempt without renewing its lease or progress.
1224    ///
1225    /// This explicit worker-protocol 1.20 operation has one five-second budget.
1226    /// A pending workflow request alone does not cancel an activity. Its reply
1227    /// reports the runtime's current continuation authority after delivery.
1228    pub async fn activity_task_status(
1229        &self,
1230        task_id: &str,
1231        activity_attempt_id: &str,
1232        lease_owner: &str,
1233    ) -> Result<Value> {
1234        if [task_id, activity_attempt_id, lease_owner]
1235            .iter()
1236            .any(|value| value.trim().is_empty())
1237        {
1238            return Err(invalid("activity observation requires the actual claim"));
1239        }
1240        tokio::time::timeout(CONTROL_BUDGET, async {
1241            let value: Value = activity_task_response(
1242                self.request_json(
1243                    reqwest::Method::POST,
1244                    &format!(
1245                        "/worker/activity-tasks/{}/status",
1246                        percent_encode_path_segment(task_id)
1247                    ),
1248                    RequestProtocol::Worker("1.20"),
1249                    Some(&json!({
1250                        "activity_attempt_id":activity_attempt_id,"lease_owner":lease_owner
1251                    })),
1252                )
1253                .await,
1254                "status",
1255                task_id,
1256                activity_attempt_id,
1257            )?;
1258            if value["task_id"].as_str() != Some(task_id)
1259                || value["activity_attempt_id"].as_str() != Some(activity_attempt_id)
1260                || value["lease_owner"].as_str() != Some(lease_owner)
1261                || value["can_continue"].as_bool().is_none()
1262                || value["cancel_requested"].as_bool().is_none()
1263                || value["heartbeat_recorded"].as_bool() != Some(false)
1264            {
1265                return Err(invalid(
1266                    "activity observation did not acknowledge the exact claim",
1267                ));
1268            }
1269            Ok(value)
1270        })
1271        .await
1272        .map_err(|_| Error::Timeout)?
1273    }
1274
1275    /// Acquire a claim with protocol 1.20 and retain its pending observation.
1276    ///
1277    /// The Server must have admitted a capable worker registration. This method
1278    /// registers no capability and does not change ordinary [`Worker`] polling.
1279    /// It never substitutes a worker ID for a missing lease owner or invents a
1280    /// task attempt. Poll retries reuse one request ID. Poll and history loading
1281    /// share one long-poll budget plus five seconds, with at most 128 pages.
1282    pub async fn poll_cooperative_workflow_task(
1283        &self,
1284        worker_id: &str,
1285        task_queue: &str,
1286        timeout: Duration,
1287    ) -> Result<CooperativeWorkflowTaskPoll> {
1288        self.poll_cooperative_workflow_task_with_request_id(
1289            worker_id,
1290            task_queue,
1291            timeout,
1292            &unique_request_id("rust-workflow-poll"),
1293            1,
1294        )
1295        .await
1296    }
1297
1298    async fn poll_cooperative_workflow_task_with_request_id(
1299        &self,
1300        worker_id: &str,
1301        task_queue: &str,
1302        timeout: Duration,
1303        poll_request_id: &str,
1304        transport_retries: usize,
1305    ) -> Result<CooperativeWorkflowTaskPoll> {
1306        if worker_id.trim().is_empty()
1307            || task_queue.trim().is_empty()
1308            || poll_request_id.trim().is_empty()
1309        {
1310            return Err(invalid(
1311                "cooperative polling requires a worker and task queue",
1312            ));
1313        }
1314        let budget = timeout
1315            .checked_add(CONTROL_BUDGET)
1316            .ok_or_else(|| invalid("cooperative poll timeout exceeds its supported budget"))?;
1317        let body = json!({
1318            "worker_id":worker_id, "task_queue":task_queue,
1319            "poll_request_id":poll_request_id,
1320            "timeout_seconds":long_poll_timeout_seconds(timeout),
1321            "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE,
1322            "build_id":self.worker_build_id,
1323        });
1324        tokio::time::timeout(budget, async {
1325            let value: Value = self
1326                .poll_request_json(
1327                    "/worker/workflow-tasks/poll",
1328                    RequestProtocol::Worker("1.20"),
1329                    &body,
1330                    budget,
1331                    transport_retries,
1332                )
1333                .await?;
1334            let mut response: PollWorkflowTaskResponse = serde_json::from_value(value.clone())
1335                .map_err(|_| invalid("cooperative poll returned a malformed envelope or task"))?;
1336            let outcome = response.outcome();
1337            let task = if let Some(task) = response.task.take() {
1338                let (owner, _) = cancellation_claim(&task)?;
1339                if owner != worker_id
1340                    || value["task"]["workflow_task_attempt"].as_u64()
1341                        != Some(task.workflow_task_attempt)
1342                {
1343                    return Err(invalid(
1344                        "cooperative poll did not return the caller's actual owner and attempt",
1345                    ));
1346                }
1347                let cancellation_request = match value["task"].get("cancellation_request") {
1348                    None | Some(Value::Null) => None,
1349                    Some(observation) => Some(CancellationRequest::from_observation(observation)?),
1350                };
1351                let mut claim = CooperativeWorkflowTask {
1352                    task,
1353                    cancellation_request,
1354                    sticky_snapshot: None,
1355                };
1356                if self.sticky_cache.is_some() {
1357                    claim.sticky_snapshot = Some(
1358                        self.load_sticky_history(&mut claim.task, &value["task"], worker_id)
1359                            .await?,
1360                    );
1361                } else {
1362                    self.load_cooperative_claim_history(&mut claim).await?;
1363                }
1364                Some(claim)
1365            } else {
1366                None
1367            };
1368            Ok(CooperativeWorkflowTaskPoll {
1369                task,
1370                outcome,
1371                protocol_version: response.protocol_version,
1372                server_capabilities: response.server_capabilities,
1373            })
1374        })
1375        .await
1376        .map_err(|_| Error::Timeout)?
1377    }
1378
1379    async fn load_cooperative_claim_history(
1380        &self,
1381        claim: &mut CooperativeWorkflowTask,
1382    ) -> Result<()> {
1383        let (owner, _) = cancellation_claim(&claim.task)?;
1384        let owner = owner.to_owned();
1385        let mut token = claim.task.next_history_page_token.clone();
1386        let mut seen = BTreeSet::new();
1387        while let Some(current) = token.take() {
1388            if current.trim().is_empty()
1389                || seen.len() >= MAX_REFRESH_PAGES
1390                || !seen.insert(current.clone())
1391            {
1392                return Err(invalid(
1393                    "claim history exceeded its page bound or returned an invalid/repeated token",
1394                ));
1395            }
1396            let body = json!({
1397                "lease_owner":owner,"workflow_task_attempt":claim.task.workflow_task_attempt,
1398                "next_history_page_token":current,"history_page_size":WORKFLOW_HISTORY_PAGE_SIZE,
1399            });
1400            let value: Value = self
1401                .request_json(
1402                    reqwest::Method::POST,
1403                    &format!(
1404                        "/worker/workflow-tasks/{}/history",
1405                        percent_encode_path_segment(&claim.task.task_id)
1406                    ),
1407                    RequestProtocol::Worker("1.20"),
1408                    Some(&body),
1409                )
1410                .await?;
1411            if value["task_id"].as_str() != Some(claim.task.task_id.as_str())
1412                || value["workflow_task_attempt"].as_u64() != Some(claim.task.workflow_task_attempt)
1413            {
1414                return Err(invalid(
1415                    "claim history changed the selected task or attempt",
1416                ));
1417            }
1418            let events = value["history_events"]
1419                .as_array()
1420                .ok_or_else(|| invalid("claim history page must contain an event array"))?;
1421            if events.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize {
1422                return Err(invalid(
1423                    "claim history page exceeds its requested event limit",
1424                ));
1425            }
1426            token = match value.get("next_history_page_token") {
1427                Some(Value::Null) => None,
1428                Some(Value::String(next)) if !next.trim().is_empty() && !events.is_empty() => {
1429                    Some(next.clone())
1430                }
1431                _ => return Err(invalid("claim history page token or progress is invalid")),
1432            };
1433            let page: WorkflowTaskHistoryPage = serde_json::from_value(value)
1434                .map_err(|_| invalid("claim history page contains malformed fields or events"))?;
1435            if page
1436                .history_events
1437                .iter()
1438                .any(|event| event.event_type.trim().is_empty())
1439            {
1440                return Err(invalid("claim history event type must be non-empty"));
1441            }
1442            claim.task.append_history_page(page);
1443        }
1444        Ok(())
1445    }
1446
1447    /// Renew the exact selected workflow-task lease using worker protocol 1.20.
1448    ///
1449    /// The reply must acknowledge the actual task, owner and attempt. Renewal
1450    /// neither commits cancellation delivery nor extends its cleanup deadline.
1451    pub async fn heartbeat_workflow_task(
1452        &self,
1453        task: &WorkflowTask,
1454        original: Option<&CancellationRequest>,
1455    ) -> Result<WorkflowTaskHeartbeat> {
1456        self.heartbeat_workflow_task_with_protocol(task, original, "1.20")
1457            .await
1458    }
1459
1460    pub(super) async fn heartbeat_workflow_task_with_protocol(
1461        &self,
1462        task: &WorkflowTask,
1463        original: Option<&CancellationRequest>,
1464        protocol: &'static str,
1465    ) -> Result<WorkflowTaskHeartbeat> {
1466        let (owner, _) = cancellation_claim(task)?;
1467        if let Some(original) = original {
1468            original.validate_observation()?;
1469        }
1470        tokio::time::timeout(CONTROL_BUDGET, async {
1471            let response: Value = self.request_json(
1472                reqwest::Method::POST,
1473                &format!("/worker/workflow-tasks/{}/heartbeat", percent_encode_path_segment(&task.task_id)),
1474                RequestProtocol::Worker(protocol),
1475                Some(&json!({"lease_owner":owner,"workflow_task_attempt":task.workflow_task_attempt})),
1476            ).await?;
1477            if response["task_id"].as_str() != Some(task.task_id.as_str())
1478                || response["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt)
1479                || response["lease_owner"].as_str() != Some(owner)
1480                || response["renewed"].as_bool() != Some(true)
1481                || response.get("reason") != Some(&Value::Null)
1482                || response["task_status"].as_str() != Some("leased")
1483            {
1484                return Err(invalid("workflow heartbeat did not acknowledge the exact leased claim"));
1485            }
1486            let expires = text(&response, "lease_expires_at")?;
1487            DateTime::parse_from_rfc3339(expires).map_err(|_| invalid("workflow heartbeat lease expiry must include a timezone"))?;
1488            let run_status = text(&response, "run_status")?;
1489            if !matches!(run_status, "pending" | "running" | "waiting") {
1490                return Err(invalid("workflow heartbeat did not acknowledge an active run"));
1491            }
1492            let cancellation_request = match response.get("cancellation_request") {
1493                None | Some(Value::Null) if original.is_none() => None,
1494                None | Some(Value::Null) => return Err(invalid("workflow heartbeat omitted its original pending request")),
1495                Some(observation) => {
1496                    let current = CancellationRequest::from_observation(observation)?;
1497                    Some(match original {
1498                        Some(original) => original.preserve_observation(&current)?,
1499                        None => current,
1500                    })
1501                },
1502            };
1503            Ok(WorkflowTaskHeartbeat {
1504                task_id:task.task_id.clone(), workflow_task_attempt:task.workflow_task_attempt,
1505                lease_owner:owner.to_owned(), lease_expires_at:expires.to_owned(),
1506                run_status:run_status.to_owned(), cancellation_request,
1507            })
1508        }).await.map_err(|_| Error::Timeout)?
1509    }
1510
1511    /// Commit cooperative delivery on the exact selected workflow-task claim.
1512    ///
1513    /// This explicit worker-protocol 1.20 operation renews no lease and advertises
1514    /// no worker capability. The Server requires a previously admitted capable
1515    /// claim. Reload canonical history before exposing cancellation to workflow
1516    /// code, including when an acknowledgment is lost. A cancellation wait explicitly
1517    /// releases the claim and must return the worker to polling.
1518    pub async fn deliver_workflow_cancellation(
1519        &self,
1520        task: &WorkflowTask,
1521        delivery: &CancellationDelivery,
1522    ) -> Result<CancellationDeliveryReply> {
1523        let (owner, run_id) = cancellation_claim(task)?;
1524        let body = json!({
1525            "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt,
1526            "request_id":delivery.request_id, "sequence":delivery.sequence,
1527            "call_kind":delivery.call_kind, "sequence_span":delivery.sequence_span,
1528            "operation_sequence":delivery.operation_sequence,
1529            "operation_sequence_span":delivery.operation_sequence_span
1530        });
1531        let mut payload = body.clone();
1532        payload["workflow_command_id"] = json!(delivery.request_id);
1533        if CancellationDelivery::from_payload(&payload)? != *delivery {
1534            return Err(invalid(
1535                "delivery must name a valid authored operation range",
1536            ));
1537        }
1538        tokio::time::timeout(CONTROL_BUDGET, async {
1539            let path = format!(
1540                "/worker/workflow-tasks/{}/deliver-cancellation",
1541                percent_encode_path_segment(&task.task_id)
1542            );
1543            let response: Value = self
1544                .request_json(
1545                    reqwest::Method::POST,
1546                    &path,
1547                    RequestProtocol::Worker("1.20"),
1548                    Some(&body),
1549                )
1550                .await?;
1551            if response["delivered"].as_bool() == Some(false)
1552                && match response["reason"].as_str() {
1553                    Some("cancellation_waiting_for_child") => matches!(
1554                        delivery.call_kind,
1555                        CancellationCallKind::Child
1556                            | CancellationCallKind::Parallel
1557                            | CancellationCallKind::SelectionHandle
1558                    ),
1559                    Some("cancellation_waiting_for_activity") => matches!(
1560                        delivery.call_kind,
1561                        CancellationCallKind::Activity
1562                            | CancellationCallKind::LocalActivity
1563                            | CancellationCallKind::Parallel
1564                            | CancellationCallKind::SelectionHandle
1565                    ),
1566                    _ => false,
1567                }
1568                && response["claim_released"].as_bool() == Some(true)
1569                && response["task_id"].as_str() == Some(task.task_id.as_str())
1570                && response["workflow_run_id"].as_str() == Some(run_id)
1571                && [
1572                    "request_id",
1573                    "sequence",
1574                    "call_kind",
1575                    "sequence_span",
1576                    "operation_sequence",
1577                    "operation_sequence_span",
1578                ]
1579                .iter()
1580                .all(|field| response.get(*field).is_none_or(Value::is_null))
1581            {
1582                return Ok(CancellationDeliveryReply::ClaimReleased {
1583                    task_id: task.task_id.clone(),
1584                    run_id: run_id.to_owned(),
1585                });
1586            }
1587            if response["delivered"].as_bool() != Some(true)
1588                || response["task_id"].as_str() != Some(task.task_id.as_str())
1589                || response["workflow_run_id"].as_str() != Some(run_id)
1590                || response.get("reason") != Some(&Value::Null)
1591                || [
1592                    "sequence_span",
1593                    "operation_sequence",
1594                    "operation_sequence_span",
1595                ]
1596                .iter()
1597                .any(|field| response.get(*field).is_none())
1598            {
1599                return Err(invalid(
1600                    "delivery acknowledgment does not match the selected task/run",
1601                ));
1602            }
1603            let mut recorded = response.clone();
1604            recorded["workflow_command_id"] = response["request_id"].clone();
1605            if CancellationDelivery::from_payload(&recorded)? != *delivery {
1606                return Err(invalid(
1607                    "delivery acknowledgment changed its authored operation range",
1608                ));
1609            }
1610            Ok(CancellationDeliveryReply::Delivered(
1611                CancellationDeliveryReceipt {
1612                    task_id: task.task_id.clone(),
1613                    run_id: run_id.to_owned(),
1614                    delivery: delivery.clone(),
1615                },
1616            ))
1617        })
1618        .await
1619        .map_err(|_| Error::Timeout)?
1620    }
1621
1622    /// Reload canonical history with the Server-issued token and exact claim.
1623    ///
1624    /// The refresh has one five-second budget, at most 128 pages, and the existing
1625    /// SDK page-size limit. Invalid, repeated or non-progressing tokens and pages
1626    /// fail closed. This returns fresh history without modifying the caller's
1627    /// previous task snapshot or its original request identity.
1628    pub async fn refresh_workflow_cancellation_history(
1629        &self,
1630        task: &WorkflowTask,
1631        observation: &CancellationRequest,
1632    ) -> Result<Vec<HistoryEvent>> {
1633        let (owner, run_id) = cancellation_claim(task)?;
1634        observation.validate_observation()?;
1635        let first_token = observation
1636            .history_refresh_page_token
1637            .clone()
1638            .expect("validated history token");
1639        tokio::time::timeout(CONTROL_BUDGET, async {
1640            let path = format!("/worker/workflow-tasks/{}/history", percent_encode_path_segment(&task.task_id));
1641            let mut token = Some(first_token);
1642            let mut seen = BTreeSet::new();
1643            let mut history = Vec::new();
1644            while let Some(current) = token.take() {
1645                if seen.len() >= MAX_REFRESH_PAGES || !seen.insert(current.clone()) {
1646                    return Err(invalid("canonical history refresh exceeded its page bound or repeated a token"));
1647                }
1648                let body = json!({
1649                    "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt,
1650                    "next_history_page_token":current, "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE,
1651                });
1652                let page: Value = self.request_json(
1653                    reqwest::Method::POST, &path, RequestProtocol::Worker("1.20"), Some(&body),
1654                ).await?;
1655                if page["task_id"].as_str() != Some(task.task_id.as_str())
1656                    || page["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt)
1657                {
1658                    return Err(invalid("canonical history page changed the selected task/attempt"));
1659                }
1660                let events = page["history_events"].as_array().ok_or_else(|| invalid("canonical history page must contain an event array"))?;
1661                if events.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize {
1662                    return Err(invalid("canonical history page exceeds its requested event limit"));
1663                }
1664                token = match page.get("next_history_page_token") {
1665                    Some(Value::Null) => None,
1666                    Some(Value::String(next)) if !next.is_empty() && !events.is_empty() => Some(next.clone()),
1667                    _ => return Err(invalid("canonical history page token or progress is invalid")),
1668                };
1669                for event in events {
1670                    let event: HistoryEvent = serde_json::from_value(event.clone()).map_err(|_| invalid("canonical history page contains a malformed event"))?;
1671                    if event.event_type.trim().is_empty() {
1672                        return Err(invalid("canonical history event type must be non-empty"));
1673                    }
1674                    history.push(event);
1675                }
1676            }
1677            let canonical = CancellationHistory::from_events(&history, run_id, Some(observation))?;
1678            if canonical.request_index >= history.len() {
1679                return Err(invalid("canonical refresh omitted the original request event"));
1680            }
1681            Ok(history)
1682        }).await.map_err(|_| Error::Timeout)?
1683    }
1684
1685    /// Request bounded workflow-authored cleanup on a capable runtime.
1686    ///
1687    /// This is separate from terminal cancellation. Discovery must advertise
1688    /// cooperative support and compatible protocol 1.20. The Server also
1689    /// refuses an active workflow claim that cannot deliver cooperation.
1690    /// Repeated requests retain its original request identity and deadline.
1691    pub async fn request_workflow_cancellation(
1692        &self,
1693        workflow_id: &str,
1694        options: CooperativeCancellationOptions,
1695    ) -> Result<WorkflowCancellationRequest> {
1696        self.request_workflow_cancellation_target(workflow_id, None, options)
1697            .await
1698    }
1699
1700    /// Request cleanup only if the selected run remains current.
1701    pub async fn request_workflow_run_cancellation(
1702        &self,
1703        workflow_id: &str,
1704        run_id: &str,
1705        options: CooperativeCancellationOptions,
1706    ) -> Result<WorkflowCancellationRequest> {
1707        self.request_workflow_cancellation_target(workflow_id, Some(run_id), options)
1708            .await
1709    }
1710
1711    async fn request_workflow_cancellation_target(
1712        &self,
1713        workflow_id: &str,
1714        run_id: Option<&str>,
1715        options: CooperativeCancellationOptions,
1716    ) -> Result<WorkflowCancellationRequest> {
1717        if workflow_id.trim().is_empty() || run_id.is_some_and(|id| id.trim().is_empty()) {
1718            return Err(invalid(
1719                "workflow and selected run identities must be non-empty",
1720            ));
1721        }
1722        if options
1723            .cleanup_timeout_seconds
1724            .is_some_and(|seconds| !(1..=3600).contains(&seconds))
1725            || options
1726                .reason
1727                .as_ref()
1728                .is_some_and(|reason| reason.chars().count() > 1000)
1729        {
1730            return Err(invalid(
1731                "cleanup timeout must be 1..3600 seconds and reason at most 1000 characters",
1732            ));
1733        }
1734
1735        // Bound discovery, mutation and any storage retry by one total budget.
1736        tokio::time::timeout(CONTROL_BUDGET, async {
1737            let info: Value = self
1738                .request_json(
1739                    reqwest::Method::GET,
1740                    "/cluster/info",
1741                    RequestProtocol::ControlPlane,
1742                    Option::<&Value>::None,
1743                )
1744                .await?;
1745            require_discovery(&info)?;
1746            let mut path = format!("/workflows/{}", percent_encode_path_segment(workflow_id));
1747            if let Some(run_id) = run_id {
1748                path.push_str(&format!("/runs/{}", percent_encode_path_segment(run_id)));
1749            }
1750            path.push_str("/request-cancellation");
1751            let response: Value = self
1752                .request_json(
1753                    reqwest::Method::POST,
1754                    &path,
1755                    RequestProtocol::ControlPlane,
1756                    Some(&options),
1757                )
1758                .await?;
1759            acknowledgment(&response, workflow_id, run_id)
1760        })
1761        .await
1762        .map_err(|_| Error::Timeout)?
1763    }
1764}
1765
1766impl Worker {
1767    pub(super) async fn poll_cooperative_activity_once(&self) -> Result<ManagedPollOutcome> {
1768        if self.client.worker_sessions_enabled {
1769            self.require_session_registration()?;
1770        }
1771        let poll_request_id = unique_request_id("rust-activity-poll");
1772        let response = self
1773            .retry_worker_operation(|| {
1774                self.client.poll_activity_task_response_with_request_id(
1775                    &self.worker_id,
1776                    &self.task_queue,
1777                    self.poll_timeout,
1778                    &poll_request_id,
1779                    0,
1780                )
1781            })
1782            .await;
1783        let Some(response) = self.settle_worker_poll_response(response).await? else {
1784            return Ok(ManagedPollOutcome::Idle);
1785        };
1786        if response.outcome().should_stop() {
1787            return Ok(ManagedPollOutcome::Stop);
1788        }
1789        let Some(task) = response.task else {
1790            return Ok(ManagedPollOutcome::Idle);
1791        };
1792        let session = self.track_session_task(task.worker_session.as_ref())?;
1793        let task = task.task;
1794        let guard = ActivityClaimGuard::new(&self.client, &task, &self.worker_id)?;
1795        let _abandon_on_drop = AbandonActivityOnDrop(guard.clone());
1796        if guard.observe().await.is_err() {
1797            return Ok(ManagedPollOutcome::Handled);
1798        }
1799        let mut callback_started = false;
1800        let result = {
1801            let invocation = self.execute_cooperative_activity_task(
1802                &task,
1803                &guard,
1804                &mut callback_started,
1805                session,
1806            );
1807            tokio::pin!(invocation);
1808            loop {
1809                tokio::select! {
1810                    biased;
1811                    _ = guard.wait_for_shutdown() => {
1812                        guard.abandon();
1813                        break None;
1814                    }
1815                    result = &mut invocation => break Some(result),
1816                    _ = tokio::time::sleep(Duration::from_secs(1)) => {
1817                        if guard.observe().await.is_err() {
1818                            break None;
1819                        }
1820                    }
1821                }
1822            }
1823        };
1824        // The invocation and its callback future are dropped before any receipt.
1825        let Some(result) = result else {
1826            if callback_started {
1827                guard.acknowledge_stopped().await?;
1828            }
1829            return Ok(ManagedPollOutcome::Handled);
1830        };
1831        // Both genuine application failures and successful results need current
1832        // authority before result upload or completion/failure publication.
1833        if matches!(result, Err(Error::ActivityExecutionAbandoned(_)))
1834            || guard.observe().await.is_err()
1835        {
1836            if callback_started {
1837                guard.acknowledge_stopped().await?;
1838            }
1839            return Ok(ManagedPollOutcome::Handled);
1840        }
1841        let settlement = match result {
1842            Ok(value) => {
1843                self.client
1844                    .complete_activity_task(
1845                        &guard.task_id,
1846                        &guard.attempt_id,
1847                        &guard.owner,
1848                        value,
1849                        &task.payload_codec,
1850                    )
1851                    .await
1852            }
1853            Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
1854            Err(error) => {
1855                self.client
1856                    .fail_activity_task(
1857                        &guard.task_id,
1858                        &guard.attempt_id,
1859                        &guard.owner,
1860                        error.to_string(),
1861                        false,
1862                    )
1863                    .await
1864            }
1865        };
1866        if let Err(error) = settlement {
1867            if !activity_task_rejection_is_final(&error) {
1868                return Err(error);
1869            }
1870        }
1871        Ok(ManagedPollOutcome::Handled)
1872    }
1873
1874    async fn execute_cooperative_activity_task(
1875        &self,
1876        task: &ActivityTask,
1877        guard: &ActivityClaimGuard,
1878        callback_started: &mut bool,
1879        worker_session: Option<crate::WorkerSession>,
1880    ) -> Result<AvroValue> {
1881        validate_activity_task_payloads(task)?;
1882        let handler = self
1883            .activities
1884            .get(&task.activity_type)
1885            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
1886        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
1887        let context = ActivityContext {
1888            client: self.client.clone(),
1889            task_id: guard.task_id.clone(),
1890            activity_attempt_id: guard.attempt_id.clone(),
1891            lease_owner: guard.owner.clone(),
1892            activity_type: task.activity_type.clone(),
1893            attempt_number: task.attempt_number,
1894            task_queue: self.task_queue.clone(),
1895            worker_id: self.worker_id.clone(),
1896            claim_guard: Some(guard.clone()),
1897            local_heartbeats: None,
1898            worker_session,
1899        };
1900        guard.boundary()?;
1901        *callback_started = true;
1902        handler(context, args).await
1903    }
1904
1905    pub(super) async fn poll_cooperative_workflow_once(&self) -> Result<ManagedPollOutcome> {
1906        let poll_request_id = unique_request_id("rust-workflow-poll");
1907        let response = self
1908            .retry_worker_operation(|| {
1909                self.client.poll_cooperative_workflow_task_with_request_id(
1910                    &self.worker_id,
1911                    &self.task_queue,
1912                    self.poll_timeout,
1913                    &poll_request_id,
1914                    0,
1915                )
1916            })
1917            .await;
1918        let Some(response) = self.settle_worker_poll_response(response).await? else {
1919            return Ok(ManagedPollOutcome::Idle);
1920        };
1921        if response.outcome.should_stop() {
1922            return Ok(ManagedPollOutcome::Stop);
1923        }
1924        let memo_updates_supported =
1925            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
1926        let Some(claim) = response.task else {
1927            return Ok(ManagedPollOutcome::Idle);
1928        };
1929        // An uncertain observation, delivery or canonical refresh cannot become
1930        // an application failure or be published as a workflow-task decision.
1931        let Some(decision) = self.replay_cooperative_workflow_claim(&claim).await? else {
1932            return Ok(ManagedPollOutcome::Handled);
1933        };
1934        let (owner, _) = cancellation_claim(&claim.task)?;
1935        self.settle_workflow_task_decision(
1936            &claim.task.task_id,
1937            owner,
1938            claim.task.workflow_task_attempt,
1939            claim.task.run_id.as_deref(),
1940            Ok(decision),
1941            memo_updates_supported,
1942            claim.sticky_snapshot.clone(),
1943        )
1944        .await
1945    }
1946
1947    async fn replay_cooperative_workflow_claim(
1948        &self,
1949        claim: &CooperativeWorkflowTask,
1950    ) -> Result<Option<WorkflowTaskDecision>> {
1951        let (owner, run_id) = cancellation_claim(&claim.task)?;
1952        if owner != self.worker_id {
1953            return Err(invalid(
1954                "cooperative replay requires this worker's actual claim",
1955            ));
1956        }
1957        let mut task = claim.task.clone();
1958        let Some(observation) = claim.cancellation_request.as_ref() else {
1959            let canonical = CancellationHistory::from_events(&task.history_events, run_id, None)?;
1960            if canonical.request.is_some() {
1961                return Err(invalid(
1962                    "cooperative replay omitted its original pending observation",
1963                ));
1964            }
1965            for _ in 0..1000 {
1966                let mut decision = self.execute_workflow_task_decision(task.clone())?;
1967                if let Some(intent) = decision.cancellation_scope_delivery.as_ref() {
1968                    if !decision.commands.is_empty() {
1969                        decision.cancellation_scope_delivery = None;
1970                        return Ok(Some(decision));
1971                    }
1972                    let mut budget = CancellationScopeDeliveryBudget::new();
1973                    budget.restrict(intent.context.deadline())?;
1974                    let prepared = self
1975                        .client
1976                        .prepare_cancellation_scope_on_claim(
1977                            &task,
1978                            &intent.context,
1979                            &intent.boundary,
1980                            &budget,
1981                        )
1982                        .await?;
1983                    budget.restrict(prepared.authority_deadline())?;
1984                    let delivered = self
1985                        .client
1986                        .deliver_cancellation_scope_on_claim(&task, &prepared, &budget)
1987                        .await?;
1988                    task.history_events = delivered.history().to_vec();
1989                    task.total_history_events = None;
1990                    task.history_size_bytes = None;
1991                    continue;
1992                }
1993                let Some(opening) = decision.cancellation_scope_opening.as_ref() else {
1994                    return Ok(Some(decision));
1995                };
1996                if !decision.commands.is_empty() {
1997                    decision.cancellation_scope_opening = None;
1998                    return Ok(Some(decision));
1999                }
2000                let receipt = self
2001                    .client
2002                    .open_cancellation_scope_on_claim(
2003                        &task,
2004                        opening.sequence,
2005                        &opening.parent_scope_id,
2006                        opening.shield_parent,
2007                    )
2008                    .await?;
2009                task.history_events = receipt.history().to_vec();
2010                task.total_history_events = None;
2011                task.history_size_bytes = None;
2012            }
2013            return Err(invalid(
2014                "workflow exceeded the canonical scope opening replay limit",
2015            ));
2016        };
2017        task.history_events = self
2018            .client
2019            .refresh_workflow_cancellation_history(&claim.task, observation)
2020            .await?;
2021        for _ in 0..1000 {
2022            // Count the actual refreshed snapshot. Its byte budget was not
2023            // remeasured, so do not expose the old snapshot's size as current.
2024            task.total_history_events = None;
2025            task.history_size_bytes = None;
2026            let mut decision = self.execute_workflow_task_decision_with_cancellation(
2027                task.clone(),
2028                Some(observation),
2029            )?;
2030            if let Some(intent) = decision.cancellation_scope_delivery.as_ref() {
2031                if !decision.commands.is_empty() {
2032                    decision.cancellation_scope_delivery = None;
2033                    return Ok(Some(decision));
2034                }
2035                let mut budget = CancellationScopeDeliveryBudget::new();
2036                budget.restrict(intent.context.deadline())?;
2037                let prepared = self
2038                    .client
2039                    .prepare_cancellation_scope_on_claim(
2040                        &task,
2041                        &intent.context,
2042                        &intent.boundary,
2043                        &budget,
2044                    )
2045                    .await?;
2046                budget.restrict(prepared.authority_deadline())?;
2047                let delivered = self
2048                    .client
2049                    .deliver_cancellation_scope_on_claim(&task, &prepared, &budget)
2050                    .await?;
2051                task.history_events = delivered.history().to_vec();
2052                continue;
2053            }
2054            let Some(intent) = decision.cancellation_delivery.as_ref() else {
2055                return Ok(Some(decision));
2056            };
2057            if !decision.commands.is_empty() {
2058                // Native permits delivery only at/before its next durable call.
2059                // Commit earlier commands first. Completion releases this claim
2060                // and its successor replays their durable results before delivery.
2061                decision.cancellation_delivery = None;
2062                return Ok(Some(decision));
2063            }
2064            let delivery_error = match self
2065                .client
2066                .deliver_workflow_cancellation(&claim.task, intent)
2067                .await
2068            {
2069                Ok(CancellationDeliveryReply::ClaimReleased { .. }) => return Ok(None),
2070                Ok(CancellationDeliveryReply::Delivered(_)) => None,
2071                Err(error) => Some(error),
2072            };
2073            task.history_events = self
2074                .client
2075                .refresh_workflow_cancellation_history(&claim.task, observation)
2076                .await?;
2077            let canonical =
2078                CancellationHistory::from_events(&task.history_events, run_id, Some(observation))?;
2079            if canonical.delivery.as_ref() != Some(intent) {
2080                return Err(delivery_error.unwrap_or_else(|| {
2081                    invalid(
2082                        "canonical history did not prove the exact intended cancellation delivery",
2083                    )
2084                }));
2085            }
2086        }
2087        Err(invalid(
2088            "workflow replay did not converge on its canonical cancellation delivery",
2089        ))
2090    }
2091}
2092
2093#[derive(Clone, Debug)]
2094pub(super) struct ActivityClaimGuard {
2095    client: Client,
2096    task_id: String,
2097    attempt_id: String,
2098    owner: String,
2099    active: Arc<AtomicBool>,
2100    cancellation_receipt: Arc<Mutex<Option<Value>>>,
2101    stop: Option<Arc<AtomicBool>>,
2102}
2103
2104struct AbandonActivityOnDrop(ActivityClaimGuard);
2105
2106impl Drop for AbandonActivityOnDrop {
2107    fn drop(&mut self) {
2108        self.0.abandon();
2109    }
2110}
2111
2112impl ActivityClaimGuard {
2113    fn new(client: &Client, task: &ActivityTask, worker_id: &str) -> Result<Self> {
2114        let owner = task.lease_owner.as_deref().unwrap_or_default();
2115        let attempt = task
2116            .activity_attempt_id
2117            .as_deref()
2118            .or(task.attempt_id.as_deref())
2119            .unwrap_or_default();
2120        if task.task_id.trim().is_empty()
2121            || attempt.trim().is_empty()
2122            || owner.trim().is_empty()
2123            || owner != worker_id
2124            || matches!((&task.activity_attempt_id, &task.attempt_id), (Some(left), Some(right)) if left != right)
2125        {
2126            return Err(invalid(
2127                "activity execution requires the worker's actual immutable claim",
2128            ));
2129        }
2130        Ok(Self {
2131            client: client.clone(),
2132            task_id: task.task_id.clone(),
2133            attempt_id: attempt.to_owned(),
2134            owner: owner.to_owned(),
2135            active: Arc::new(AtomicBool::new(true)),
2136            cancellation_receipt: Arc::new(Mutex::new(None)),
2137            stop: client
2138                .worker_storage_admission
2139                .as_ref()
2140                .map(|admission| Arc::clone(&admission.stop)),
2141        })
2142    }
2143
2144    fn abandon(&self) {
2145        self.active.store(false, Ordering::SeqCst);
2146    }
2147
2148    /// Caller has already dropped the callback future. Never restores authority.
2149    async fn acknowledge_stopped(&self) -> Result<()> {
2150        self.abandon();
2151        let receipt = self
2152            .cancellation_receipt
2153            .lock()
2154            .map_err(|_| invalid("activity cancellation receipt lock was poisoned"))?
2155            .clone();
2156        if let Some(receipt) = receipt {
2157            self.client
2158                .acknowledge_activity_cancellation(
2159                    &self.task_id,
2160                    &self.attempt_id,
2161                    &self.owner,
2162                    text(&receipt, "request_id")?,
2163                )
2164                .await?;
2165        }
2166        Ok(())
2167    }
2168
2169    fn retain_cancellation_receipt(&self, value: &Value) -> Result<()> {
2170        if value["can_continue"].as_bool() != Some(false)
2171            || value["cancel_requested"].as_bool() != Some(true)
2172        {
2173            return Ok(());
2174        }
2175        let Some(receipt) = value
2176            .get("cancellation_acknowledgement")
2177            .filter(|receipt| receipt.is_object())
2178        else {
2179            return Ok(());
2180        };
2181        if !matches!(
2182            receipt["callback_state"].as_str(),
2183            Some("unknown" | "stopped")
2184        ) || [
2185            "request_id",
2186            "root_request_id",
2187            "cleanup_deadline_at",
2188            "cancellation_history_event_id",
2189        ]
2190        .iter()
2191        .any(|field| text(receipt, field).is_err())
2192            || DateTime::parse_from_rfc3339(text(receipt, "cleanup_deadline_at")?).is_err()
2193        {
2194            return Ok(());
2195        }
2196        let mut retained = self
2197            .cancellation_receipt
2198            .lock()
2199            .map_err(|_| invalid("activity cancellation receipt lock was poisoned"))?;
2200        if let Some(original) = retained.as_ref() {
2201            if [
2202                "request_id",
2203                "root_request_id",
2204                "cleanup_deadline_at",
2205                "cancellation_history_event_id",
2206            ]
2207            .iter()
2208            .any(|field| original[*field] != receipt[*field])
2209            {
2210                return Err(invalid(
2211                    "activity cancellation observation changed its original identity or deadline",
2212                ));
2213            }
2214        } else {
2215            *retained = Some(receipt.clone());
2216        }
2217        Ok(())
2218    }
2219
2220    fn boundary(&self) -> Result<()> {
2221        if !self.active.load(Ordering::SeqCst)
2222            || self
2223                .stop
2224                .as_ref()
2225                .is_some_and(|stop| stop.load(Ordering::SeqCst))
2226        {
2227            self.abandon();
2228            return Err(Error::ActivityExecutionAbandoned(
2229                "callback completed, was abandoned, or its worker is stopping".into(),
2230            ));
2231        }
2232        Ok(())
2233    }
2234
2235    async fn wait_for_shutdown(&self) {
2236        if let Some(stop) = &self.stop {
2237            wait_for_worker_stop(stop).await;
2238        } else {
2239            std::future::pending::<()>().await;
2240        }
2241    }
2242
2243    async fn observe(&self) -> Result<()> {
2244        self.boundary()?;
2245        let result = self
2246            .client
2247            .activity_task_status(&self.task_id, &self.attempt_id, &self.owner)
2248            .await
2249            .and_then(|value| {
2250                self.retain_cancellation_receipt(&value)?;
2251                if value["can_continue"].as_bool() != Some(true)
2252                    || value["cancel_requested"].as_bool() != Some(false)
2253                    || value.get("reason") != Some(&Value::Null)
2254                    || value["task_status"].as_str() != Some("leased")
2255                    || value["attempt_status"].as_str() != Some("running")
2256                    || value["activity_status"].as_str() != Some("running")
2257                {
2258                    return Err(invalid("activity observation refused continuation"));
2259                }
2260                let mut bounds = vec![text(&value, "lease_expires_at")?];
2261                if let Some(deadlines) = value.get("deadlines").filter(|v| !v.is_null()) {
2262                    if !deadlines.is_object() {
2263                        return Err(invalid("activity execution deadlines must be an object"));
2264                    }
2265                    for kind in ["heartbeat", "start_to_close", "schedule_to_close"] {
2266                        if let Some(deadline) = deadlines.get(kind).filter(|v| !v.is_null()) {
2267                            bounds.push(
2268                                deadline
2269                                    .as_str()
2270                                    .ok_or_else(|| invalid("invalid activity deadline"))?,
2271                            );
2272                        }
2273                    }
2274                }
2275                if let Some(session) = value.get("worker_session").filter(|v| !v.is_null()) {
2276                    if session["status"].as_str() != Some("active")
2277                        || session["lease_owner"].as_str() != Some(self.owner.as_str())
2278                    {
2279                        return Err(invalid(
2280                            "activity no longer owns its required worker session",
2281                        ));
2282                    }
2283                    bounds.push(text(session, "lease_expires_at")?);
2284                    bounds.push(text(session, "ttl_expires_at")?);
2285                }
2286                for bound in bounds {
2287                    let deadline = DateTime::parse_from_rfc3339(bound)
2288                        .map_err(|_| invalid("activity deadline must include a timezone"))?;
2289                    let now = SystemTime::now()
2290                        .duration_since(UNIX_EPOCH)
2291                        .map_err(|_| invalid("activity observation clock precedes the epoch"))?;
2292                    if !u128::try_from(deadline.timestamp_millis())
2293                        .is_ok_and(|millis| millis > now.as_millis())
2294                    {
2295                        return Err(invalid("activity ownership or execution deadline elapsed"));
2296                    }
2297                }
2298                self.boundary()
2299            });
2300        result.map_err(|error| {
2301            self.abandon();
2302            Error::ActivityExecutionAbandoned(error.to_string())
2303        })
2304    }
2305
2306    pub(super) async fn heartbeat<T: Serialize>(
2307        &self,
2308        context: &ActivityContext,
2309        details: T,
2310    ) -> Result<ActivityHeartbeatResponse> {
2311        if context.task_id != self.task_id
2312            || context.activity_attempt_id != self.attempt_id
2313            || context.lease_owner != self.owner
2314            || context.worker_id != self.owner
2315        {
2316            self.abandon();
2317            return Err(Error::ActivityExecutionAbandoned(
2318                "activity context changed its original claim".into(),
2319            ));
2320        }
2321        self.observe().await?;
2322        let result = tokio::time::timeout(CONTROL_BUDGET, async {
2323            let details = AvroValue::from_serialize(&details)?.into_json()?;
2324            self.boundary()?;
2325            let value: Value = self.client.request_json(
2326                reqwest::Method::POST,
2327                &format!("/worker/activity-tasks/{}/heartbeat", percent_encode_path_segment(&self.task_id)),
2328                RequestProtocol::Worker("1.20"),
2329                Some(&json!({"activity_attempt_id":self.attempt_id,"lease_owner":self.owner,"details":details})),
2330            ).await?;
2331            if value["task_id"].as_str() != Some(self.task_id.as_str())
2332                || value["activity_attempt_id"].as_str() != Some(self.attempt_id.as_str())
2333                || value["lease_owner"].as_str() != Some(self.owner.as_str())
2334                || value["can_continue"].as_bool() != Some(true)
2335                || value["cancel_requested"].as_bool() != Some(false)
2336                || value["heartbeat_recorded"].as_bool() != Some(true)
2337            {
2338                return Err(invalid("activity heartbeat lost its original claim"));
2339            }
2340            if let Some(session) = context.worker_session() {
2341                session.track(&value["worker_session"])?;
2342            }
2343            serde_json::from_value(value).map_err(Error::from)
2344        }).await.map_err(|_| Error::Timeout).and_then(|result| result);
2345        let response = result.map_err(|error| {
2346            self.abandon();
2347            Error::ActivityExecutionAbandoned(error.to_string())
2348        })?;
2349        self.observe().await?;
2350        Ok(response)
2351    }
2352}
2353
2354pub(super) fn cancellation_claim(task: &WorkflowTask) -> Result<(&str, &str)> {
2355    let owner = task
2356        .lease_owner
2357        .as_deref()
2358        .filter(|owner| !owner.trim().is_empty());
2359    let run = task.run_id.as_deref().filter(|run| !run.trim().is_empty());
2360    if task.task_id.trim().is_empty()
2361        || task.workflow_task_attempt == 0
2362        || task.workflow_task_attempt > MAX_SEQUENCE
2363    {
2364        return Err(invalid(
2365            "cancellation transport requires a valid task and attempt",
2366        ));
2367    }
2368    Ok((
2369        owner.ok_or_else(|| invalid("cancellation transport requires the actual lease owner"))?,
2370        run.ok_or_else(|| invalid("cancellation transport requires the selected durable run"))?,
2371    ))
2372}
2373
2374impl WorkflowHandle {
2375    /// Request bounded cleanup for whichever run is current.
2376    pub async fn request_cancellation(
2377        &self,
2378        options: CooperativeCancellationOptions,
2379    ) -> Result<WorkflowCancellationRequest> {
2380        self.client
2381            .request_workflow_cancellation(&self.workflow_id, options)
2382            .await
2383    }
2384
2385    /// Request bounded cleanup only while this handle's selected run is current.
2386    pub async fn request_selected_run_cancellation(
2387        &self,
2388        options: CooperativeCancellationOptions,
2389    ) -> Result<WorkflowCancellationRequest> {
2390        let run_id = self
2391            .run_id
2392            .as_deref()
2393            .ok_or_else(|| invalid("selected run_id is required"))?;
2394        self.client
2395            .request_workflow_run_cancellation(&self.workflow_id, run_id, options)
2396            .await
2397    }
2398}