1use super::*;
2use std::collections::BTreeSet;
3
4const CONTROL_BUDGET: Duration = Duration::from_secs(5);
5const MAX_REFRESH_PAGES: usize = 128;
6
7pub(super) fn assert_cancellation_scope_replay_supported(
8 events: &[HistoryEvent],
9 allow_authoring: bool,
10 allow_delivery: bool,
11) -> Result<()> {
12 if allow_delivery && !allow_authoring {
13 return Err(Error::CancellationScopeExecutionUnavailable);
14 }
15 for event in events {
16 let scope_marker = matches!(
17 event.event_type.as_str(),
18 "CancellationScopeRequested"
19 | "CancellationScopeDeliveryPrepared"
20 | "CancellationScopeDelivered"
21 | "CancellationScopeRequestConflicted"
22 );
23 let scoped_membership = std::iter::once(&event.payload)
24 .chain(
25 ["activity", "timer", "child_workflow"]
26 .iter()
27 .filter_map(|name| event.payload.get(name)),
28 )
29 .any(|payload| {
30 payload
31 .get("cancellation_scope_id")
32 .is_some_and(|scope| scope.as_str() != Some("root"))
33 });
34 if (scope_marker && !allow_delivery)
35 || (!allow_authoring
36 && (event.event_type == "CancellationScopeOpened" || scoped_membership))
37 {
38 return Err(Error::CancellationScopeExecutionUnavailable);
39 }
40 }
41 Ok(())
42}
43
44#[derive(Clone, Debug, PartialEq, Eq)]
46pub struct CancellationDeliveryReceipt {
47 pub task_id: String,
48 pub run_id: String,
49 pub delivery: CancellationDelivery,
50}
51
52#[derive(Clone, Debug, PartialEq, Eq)]
54pub enum CancellationDeliveryReply {
55 Delivered(CancellationDeliveryReceipt),
56 ClaimReleased {
59 task_id: String,
60 run_id: String,
61 },
62}
63
64#[derive(Clone, Debug, PartialEq, Eq)]
66pub struct WorkflowTaskHeartbeat {
67 pub task_id: String,
68 pub workflow_task_attempt: u64,
69 pub lease_owner: String,
70 pub lease_expires_at: String,
71 pub run_status: String,
72 pub cancellation_request: Option<CancellationRequest>,
73}
74
75#[derive(Clone, Debug)]
80pub struct CooperativeWorkflowTask {
81 task: WorkflowTask,
82 cancellation_request: Option<CancellationRequest>,
83 sticky_snapshot: Option<sticky_worker::StickySnapshot>,
84}
85
86impl CooperativeWorkflowTask {
87 pub fn task(&self) -> &WorkflowTask {
88 &self.task
89 }
90
91 pub fn cancellation_request(&self) -> Option<&CancellationRequest> {
92 self.cancellation_request.as_ref()
93 }
94
95 pub async fn heartbeat(&mut self, client: &Client) -> Result<WorkflowTaskHeartbeat> {
99 let receipt = client
100 .heartbeat_workflow_task(&self.task, self.cancellation_request.as_ref())
101 .await?;
102 self.cancellation_request = receipt.cancellation_request.clone();
103 Ok(receipt)
104 }
105}
106
107#[derive(Clone, Debug)]
109pub struct CooperativeWorkflowTaskPoll {
110 pub task: Option<CooperativeWorkflowTask>,
111 pub outcome: WorkerPollOutcome,
112 pub protocol_version: Option<String>,
113 pub server_capabilities: Option<Value>,
114}
115
116#[derive(Clone, Debug, Default, Serialize)]
118pub struct CooperativeCancellationOptions {
119 #[serde(skip_serializing_if = "Option::is_none")]
120 pub reason: Option<String>,
121 #[serde(skip_serializing_if = "Option::is_none")]
122 pub cleanup_timeout_seconds: Option<u64>,
123}
124
125#[derive(Clone, Debug, PartialEq, Eq)]
127pub struct CancellationRequest {
128 pub request_id: String,
129 pub requested_at: String,
130 pub cleanup_deadline_at: String,
131 pub history_refresh_page_token: Option<String>,
132 pub context: Option<CancellationContext>,
133}
134
135#[derive(Clone, Debug, PartialEq, Eq)]
137pub struct WorkflowCancellationRequest {
138 pub workflow_id: String,
139 pub run_id: String,
140 pub duplicate: bool,
141 pub cancellation_request: CancellationRequest,
142}
143
144#[derive(Clone, Debug, Error, PartialEq, Eq)]
146#[error("workflow cancellation {request_id} was requested", request_id = .request.request_id)]
147pub struct CooperativeCancellationRequested {
148 pub request: CancellationRequest,
149 pub delivery: CancellationDelivery,
150}
151
152#[derive(Debug)]
156pub struct CancellationShield {
157 state: Arc<Mutex<WorkflowState>>,
158}
159
160impl Drop for CancellationShield {
161 fn drop(&mut self) {
162 if let Ok(mut state) = self.state.lock() {
163 state.cancellation_shield_depth = state.cancellation_shield_depth.saturating_sub(1);
164 }
165 }
166}
167
168fn invalid(detail: impl Into<String>) -> Error {
169 Error::InvalidCooperativeCancellation(detail.into())
170}
171
172fn text<'a>(value: &'a Value, field: &str) -> Result<&'a str> {
173 value
174 .get(field)
175 .and_then(Value::as_str)
176 .filter(|text| !text.trim().is_empty())
177 .ok_or_else(|| invalid(format!("{field} must be a non-empty string")))
178}
179
180impl CancellationRequest {
181 pub(super) fn preserve_observation(&self, current: &Self) -> Result<Self> {
182 self.validate_observation()?;
183 current.validate_observation()?;
184 if self.request_id != current.request_id
185 || DateTime::parse_from_rfc3339(&self.requested_at).unwrap()
186 != DateTime::parse_from_rfc3339(¤t.requested_at).unwrap()
187 || DateTime::parse_from_rfc3339(&self.cleanup_deadline_at).unwrap()
188 != DateTime::parse_from_rfc3339(¤t.cleanup_deadline_at).unwrap()
189 {
190 return Err(invalid(
191 "observation changed the original request or cleanup deadline",
192 ));
193 }
194 Ok(self.clone())
195 }
196
197 pub(super) fn validate_observation(&self) -> Result<()> {
198 Self::from_observation(&json!({
199 "request_id":self.request_id, "requested_at":self.requested_at,
200 "cleanup_deadline_at":self.cleanup_deadline_at,
201 "history_refresh_page_token":self.history_refresh_page_token,
202 }))
203 .map(|_| ())
204 }
205
206 pub(crate) fn from_observation(value: &Value) -> Result<Self> {
207 let requested_at = text(value, "requested_at")?;
208 let cleanup_deadline_at = text(value, "cleanup_deadline_at")?;
209 let requested = DateTime::parse_from_rfc3339(requested_at)
210 .map_err(|_| invalid("requested_at must be a timestamp with a timezone"))?;
211 let deadline = DateTime::parse_from_rfc3339(cleanup_deadline_at)
212 .map_err(|_| invalid("cleanup_deadline_at must be a timestamp with a timezone"))?;
213 if deadline <= requested {
214 return Err(invalid("cleanup deadline must follow the original request"));
215 }
216 Ok(Self {
217 request_id: text(value, "request_id")?.to_owned(),
218 requested_at: requested_at.to_owned(),
219 cleanup_deadline_at: cleanup_deadline_at.to_owned(),
220 history_refresh_page_token: Some(text(value, "history_refresh_page_token")?.to_owned()),
221 context: None,
222 })
223 }
224}
225
226const REQUEST_EVENT: &str = "CooperativeCancellationRequested";
227const DELIVERY_EVENT: &str = "CooperativeCancellationDelivered";
228const MAX_SEQUENCE: u64 = i64::MAX as u64;
229
230#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
232#[serde(rename_all = "snake_case")]
233pub enum CancellationCallKind {
234 Activity,
235 LocalActivity,
236 Timer,
237 Condition,
238 Signal,
239 Child,
240 Parallel,
241 SelectionHandle,
242}
243
244#[derive(Clone, Debug, PartialEq, Eq)]
246pub struct CancellationDelivery {
247 pub request_id: String,
248 pub sequence: u64,
249 pub call_kind: CancellationCallKind,
250 pub sequence_span: u64,
251 pub operation_sequence: Option<u64>,
252 pub operation_sequence_span: u64,
253}
254
255fn positive(value: &Value, field: &str, maximum: u64) -> Result<u64> {
256 value
257 .as_u64()
258 .filter(|number| (1..=maximum).contains(number))
259 .ok_or_else(|| {
260 invalid(format!(
261 "{field} must be a positive integer within {maximum}"
262 ))
263 })
264}
265
266impl CancellationDelivery {
267 pub(crate) fn from_payload(value: &Value) -> Result<Self> {
268 let sequence = positive(&value["sequence"], "sequence", MAX_SEQUENCE)?;
269 let call_kind: CancellationCallKind = serde_json::from_value(value["call_kind"].clone())
270 .map_err(|_| invalid("delivery must name a supported durable call kind"))?;
271 let span = positive(
272 value.get("sequence_span").unwrap_or(&json!(1)),
273 "sequence_span",
274 1000,
275 )?;
276 let operation_span = positive(
277 value.get("operation_sequence_span").unwrap_or(&json!(1)),
278 "operation_sequence_span",
279 1000,
280 )?;
281 let operation = value
282 .get("operation_sequence")
283 .filter(|value| !value.is_null());
284 if (call_kind != CancellationCallKind::Parallel && span != 1)
285 || sequence > MAX_SEQUENCE - span
286 {
287 return Err(invalid(
288 "delivery call span is invalid or overflows the portable sequence range",
289 ));
290 }
291 let operation_sequence = if call_kind == CancellationCallKind::SelectionHandle {
292 let base = positive(
293 operation.unwrap_or(&Value::Null),
294 "operation_sequence",
295 MAX_SEQUENCE,
296 )?;
297 if base >= sequence || operation_span > sequence - base {
298 return Err(invalid(
299 "selection handle must name a complete earlier operation range",
300 ));
301 }
302 Some(base)
303 } else {
304 if operation.is_some() || operation_span != 1 {
305 return Err(invalid(
306 "only selection handles may name an operation range",
307 ));
308 }
309 None
310 };
311 Ok(Self {
312 request_id: text(value, "workflow_command_id")?.to_owned(),
313 sequence,
314 call_kind,
315 sequence_span: span,
316 operation_sequence,
317 operation_sequence_span: operation_span,
318 })
319 }
320
321 fn range(&self) -> (u64, u64) {
322 self.operation_sequence
323 .map_or((self.sequence, self.sequence_span), |base| {
324 (base, self.operation_sequence_span)
325 })
326 }
327
328 pub fn interrupts(&self, sequence: u64) -> bool {
330 let (base, span) = self.range();
331 sequence >= base && sequence - base < span
332 }
333}
334
335#[derive(Clone, Debug)]
340pub struct CancellationHistory {
341 pub request: Option<CancellationRequest>,
342 pub delivery: Option<CancellationDelivery>,
343 pub request_index: usize,
344 pub delivery_index: Option<usize>,
345 resolved_before_request: BTreeSet<u64>,
346 latest_condition_sequences_before_request: BTreeMap<u64, u64>,
347 failed_before_request: BTreeSet<u64>,
348 selected_before_request: BTreeSet<(u64, u64)>,
349}
350
351impl CancellationHistory {
352 pub(super) fn bind_commands(
353 &self,
354 mut commands: Vec<RecordedCommand>,
355 ) -> Result<Vec<RecordedCommand>> {
356 let Some(delivery) = &self.delivery else {
357 return Ok(commands);
358 };
359 if delivery.call_kind == CancellationCallKind::Parallel {
360 let index = commands.partition_point(|command| command.sequence() < delivery.sequence);
361 let end = commands.partition_point(|command| {
362 command.sequence() < delivery.sequence + delivery.sequence_span
363 });
364 if commands.get(index).map(RecordedCommand::sequence) != Some(delivery.sequence) {
365 let previous = index
366 .checked_sub(1)
367 .map_or(0, |index| commands[index].sequence());
368 if previous.checked_add(1) != Some(delivery.sequence) {
369 return Err(invalid_recorded_history(
370 "cooperative_cancellation_call_mismatch",
371 delivery.sequence,
372 "next authored parallel group",
373 "missing earlier call",
374 "parallel cancellation marker skips an unrecorded authored command",
375 ));
376 }
377 }
378 let original = commands.drain(index..end).collect();
379 commands.insert(
380 index,
381 RecordedCommand::CancellationGroup {
382 sequence: delivery.sequence,
383 span: delivery.sequence_span,
384 original,
385 },
386 );
387 for command in &mut commands[index + 1..] {
391 let RecordedCommand::ConditionWait {
392 sequence,
393 parallel_group_path: Some(path),
394 ..
395 } = command
396 else {
397 continue;
398 };
399 if self.resolved_before_request.contains(sequence)
400 || !path.first().is_some_and(|group| {
401 group.parallel_group_base_sequence == delivery.sequence
402 && group.parallel_group_size as u64 == delivery.sequence_span
403 })
404 {
405 continue;
406 }
407 *command = RecordedCommand::CancellationBoundary {
408 sequence: *sequence,
409 call_kind: CancellationCallKind::Condition,
410 original: Some(Box::new(command.clone())),
411 };
412 }
413 return Ok(commands);
414 }
415 if !matches!(
416 delivery.call_kind,
417 CancellationCallKind::Activity
418 | CancellationCallKind::Timer
419 | CancellationCallKind::Condition
420 | CancellationCallKind::Signal
421 | CancellationCallKind::Child
422 | CancellationCallKind::SelectionHandle
423 ) {
424 return Ok(commands);
425 }
426 let index = commands.partition_point(|command| command.sequence() < delivery.sequence);
427 let original = if commands
428 .get(index)
429 .is_some_and(|command| command.sequence() == delivery.sequence)
430 {
431 Some(Box::new(commands.remove(index)))
432 } else {
433 let previous = index
434 .checked_sub(1)
435 .map_or(0, |index| commands[index].sequence());
436 if previous.checked_add(1) != Some(delivery.sequence) {
437 return Err(invalid_recorded_history(
438 "cooperative_cancellation_call_mismatch",
439 delivery.sequence,
440 "next authored durable call",
441 "missing earlier call",
442 "cancellation marker skips an unrecorded authored command",
443 ));
444 }
445 None
446 };
447 commands.insert(
448 index,
449 RecordedCommand::CancellationBoundary {
450 sequence: delivery.sequence,
451 call_kind: delivery.call_kind,
452 original,
453 },
454 );
455 Ok(commands)
456 }
457
458 pub fn from_events(
460 events: &[HistoryEvent],
461 run_id: &str,
462 observation: Option<&CancellationRequest>,
463 ) -> Result<Self> {
464 Self::read(events, run_id, observation).map_err(|error| {
465 invalid_recorded_history(
466 "cooperative_cancellation_history_invalid",
467 0,
468 "one canonical request and matching authored delivery",
469 "invalid cancellation history",
470 &error.to_string(),
471 )
472 })
473 }
474
475 fn read(
476 events: &[HistoryEvent],
477 run_id: &str,
478 observation: Option<&CancellationRequest>,
479 ) -> Result<Self> {
480 if let Some(observed) = observation {
481 let requested = DateTime::parse_from_rfc3339(&observed.requested_at)
482 .map_err(|_| invalid("observed request timestamp is invalid"))?;
483 let deadline = DateTime::parse_from_rfc3339(&observed.cleanup_deadline_at)
484 .map_err(|_| invalid("observed cleanup deadline is invalid"))?;
485 if observed.request_id.trim().is_empty()
486 || deadline <= requested
487 || observed
488 .history_refresh_page_token
489 .as_ref()
490 .is_some_and(|token| token.trim().is_empty())
491 {
492 return Err(invalid(
493 "observed request identity, deadline or history token is invalid",
494 ));
495 }
496 }
497 let mut state = Self {
498 request: observation.cloned().map(|mut request| {
499 request.context = None;
500 request
501 }),
502 delivery: None,
503 request_index: events.len(),
504 delivery_index: None,
505 resolved_before_request: BTreeSet::new(),
506 latest_condition_sequences_before_request: BTreeMap::new(),
507 failed_before_request: BTreeSet::new(),
508 selected_before_request: BTreeSet::new(),
509 };
510 let mut saw_request = false;
511 for (index, event) in events.iter().enumerate() {
512 if !matches!(event.event_type.as_str(), REQUEST_EVENT | DELIVERY_EVENT) {
513 continue;
514 }
515 let request_id = text(&event.payload, "workflow_command_id")?;
516 let event_run = text(&event.payload, "workflow_run_id")?;
517 if event
518 .raw
519 .get("workflow_command_id")
520 .is_some_and(|value| value.as_str() != Some(request_id))
521 || (!run_id.is_empty() && event_run != run_id)
522 {
523 return Err(invalid(
524 "canonical event does not match its request or workflow run",
525 ));
526 }
527 if event.event_type == REQUEST_EVENT {
528 if saw_request || state.delivery.is_some() {
529 return Err(invalid("history must contain one request before delivery"));
530 }
531 let recorded_at = event
532 .raw
533 .get("recorded_at")
534 .or_else(|| event.raw.get("timestamp"))
535 .and_then(Value::as_str)
536 .ok_or_else(|| invalid("canonical request lacks its recorded timestamp"))?;
537 let requested = DateTime::parse_from_rfc3339(recorded_at)
538 .map_err(|_| invalid("canonical request timestamp is invalid"))?;
539 let deadline_text = text(&event.payload, "cleanup_deadline_at")?;
540 let deadline = DateTime::parse_from_rfc3339(deadline_text)
541 .map_err(|_| invalid("canonical cleanup deadline is invalid"))?;
542 let context = event
543 .payload
544 .get("cancellation")
545 .map(CancellationContext::from_value)
546 .transpose()?;
547 if let Some(context) = &context {
548 let local = context.lineage().last().unwrap();
549 if context.request_id() != request_id
550 || local.workflow_run_id() != event_run
551 || event
552 .payload
553 .get("workflow_instance_id")
554 .is_some_and(|instance| {
555 instance.as_str() != Some(local.workflow_instance_id())
556 })
557 || context.deadline() != deadline
558 || context.requested_at() > requested
559 || event
560 .payload
561 .get("reason")
562 .is_some_and(|reason| match reason {
563 Value::Null => context.reason().is_some(),
564 Value::String(reason) => Some(reason.as_str()) != context.reason(),
565 _ => true,
566 })
567 {
568 return Err(invalid(
569 "canonical cancellation context does not match its request event",
570 ));
571 }
572 } else if deadline <= requested {
573 return Err(invalid(
574 "canonical cleanup deadline must follow the request",
575 ));
576 }
577 if let Some(observed) = observation {
578 let observed_deadline =
579 DateTime::parse_from_rfc3339(&observed.cleanup_deadline_at)
580 .map_err(|_| invalid("observed cleanup deadline is invalid"))?;
581 if observed.request_id != request_id || observed_deadline != deadline {
582 return Err(invalid(
583 "observation changes the original request or cleanup deadline",
584 ));
585 }
586 }
587 if context.is_some() || observation.is_none() {
588 state.request = Some(CancellationRequest {
589 request_id: request_id.to_owned(),
590 requested_at: context.as_ref().map_or_else(
591 || recorded_at.to_owned(),
592 |context| {
593 context
594 .requested_at()
595 .to_rfc3339_opts(chrono::SecondsFormat::Micros, true)
596 },
597 ),
598 cleanup_deadline_at: deadline_text.to_owned(),
599 history_refresh_page_token: observation
600 .and_then(|request| request.history_refresh_page_token.clone()),
601 context,
602 });
603 }
604 state.request_index = index;
605 saw_request = true;
606 } else {
607 if !saw_request || state.delivery.is_some() {
608 return Err(invalid(
609 "delivery requires one earlier request and one marker",
610 ));
611 }
612 let delivery = CancellationDelivery::from_payload(&event.payload)?;
613 if state
614 .request
615 .as_ref()
616 .map(|request| request.request_id.as_str())
617 != Some(delivery.request_id.as_str())
618 {
619 return Err(invalid("delivery names a different original request"));
620 }
621 if let Some(snapshot) = event.payload.get("cancellation") {
622 let context = CancellationContext::from_value(snapshot)?;
623 if state
624 .request
625 .as_ref()
626 .and_then(|request| request.context.as_ref())
627 != Some(&context)
628 {
629 return Err(invalid(
630 "delivery changes the canonical cancellation context",
631 ));
632 }
633 }
634 state.delivery = Some(delivery);
635 state.delivery_index = Some(index);
636 }
637 }
638 if state.request.is_none() {
639 return Ok(state);
640 }
641 let mut condition_occurrences = BTreeMap::new();
642 for event in &events[..state.request_index] {
643 if matches!(
644 event.event_type.as_str(),
645 "SelectionResolved" | "SelectionOperationCancelled"
646 ) {
647 let (base_field, span_field) = if event.event_type == "SelectionResolved" {
648 ("selection_group_base_sequence", "selection_group_size")
649 } else {
650 ("member_base_sequence", "member_size")
651 };
652 if let (Ok(base), Ok(span)) = (
653 positive(&event.payload[base_field], base_field, MAX_SEQUENCE),
654 positive(&event.payload[span_field], span_field, 1000),
655 ) {
656 if base <= MAX_SEQUENCE - span {
657 if event.event_type == "SelectionResolved" {
658 state.selected_before_request.insert((base, span));
659 } else {
660 state.resolved_before_request.extend(base..base + span);
661 }
662 }
663 }
664 }
665 let Ok(sequence) = positive(&event.payload["sequence"], "sequence", MAX_SEQUENCE)
666 else {
667 continue;
668 };
669 if event.event_type == "ConditionWaitOpened"
672 && (event.payload["parallel_group_id"].is_string()
673 || event.payload["parallel_group_path"]
674 .as_array()
675 .is_some_and(|path| !path.is_empty()))
676 {
677 if let Some(occurrence) = event.payload["condition_wait_occurrence_id"].as_str() {
678 let authored = *condition_occurrences.entry(occurrence).or_insert(sequence);
679 state
680 .latest_condition_sequences_before_request
681 .insert(authored, sequence);
682 }
683 }
684 if matches!(
685 event.event_type.as_str(),
686 "ActivityCompleted"
687 | "ActivityFailed"
688 | "ActivityCancelled"
689 | "ActivityTimedOut"
690 | "TimerFired"
691 | "TimerCancelled"
692 | "ConditionWaitSatisfied"
693 | "ConditionWaitTimedOut"
694 | "SignalApplied"
695 | "ChildRunCompleted"
696 | "ChildRunFailed"
697 | "ChildRunCancelled"
698 | "ChildRunTerminated"
699 ) {
700 state.resolved_before_request.insert(sequence);
701 if matches!(
702 event.event_type.as_str(),
703 "ActivityFailed"
704 | "ActivityCancelled"
705 | "ActivityTimedOut"
706 | "ChildRunFailed"
707 | "ChildRunCancelled"
708 | "ChildRunTerminated"
709 ) {
710 state.failed_before_request.insert(sequence);
711 }
712 }
713 }
714 if let Some(delivery) = &state.delivery {
715 let (base, span) = delivery.range();
716 if !state.range_eligible(base, span) {
717 return Err(invalid(
718 "delivery cannot replace an earlier committed result",
719 ));
720 }
721 }
722 Ok(state)
723 }
724
725 fn range_eligible(&self, sequence: u64, span: u64) -> bool {
726 (1..=1000).contains(&span)
727 && sequence > 0
728 && sequence <= MAX_SEQUENCE - span
729 && !(sequence..sequence + span).all(|sequence| {
730 let latest = self
731 .latest_condition_sequences_before_request
732 .get(&sequence)
733 .unwrap_or(&sequence);
734 self.resolved_before_request.contains(latest)
735 })
736 && !(sequence..sequence + span)
737 .any(|sequence| self.failed_before_request.contains(&sequence))
738 && !self.selected_before_request.contains(&(sequence, span))
739 }
740
741 pub fn eligible(&self, sequence: u64, span: u64) -> bool {
743 self.request.is_some() && self.delivery.is_none() && self.range_eligible(sequence, span)
744 }
745}
746
747impl WorkflowState {
748 fn next_cancellation_sequence(&self) -> Result<u64> {
749 self.recorded_commands.get(self.command_cursor).map_or_else(
750 || {
751 self.recorded_commands
752 .last()
753 .map_or(0, RecordedCommand::sequence)
754 .checked_add(self.commands.len() as u64)
755 .and_then(|sequence| sequence.checked_add(1))
756 .filter(|sequence| *sequence < MAX_SEQUENCE)
757 .ok_or_else(|| invalid("authored cancellation sequence overflowed"))
758 },
759 |command| Ok(command.sequence()),
760 )
761 }
762
763 fn prepare_cancellation_delivery(&mut self, delivery: CancellationDelivery) -> Result<bool> {
764 if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 {
765 return Ok(false);
766 }
767 self.assert_root_cancellation_membership()?;
768 if self.cancellation_delivery_intent.is_some() {
769 self.matched_recorded_pending = true;
770 return Ok(true);
771 }
772 let (base, span) = delivery.range();
773 if !self.cancellation_history.eligible(base, span) {
774 return Ok(false);
775 }
776 let payload = json!({
777 "workflow_command_id":delivery.request_id, "sequence":delivery.sequence,
778 "call_kind":delivery.call_kind, "sequence_span":delivery.sequence_span,
779 "operation_sequence":delivery.operation_sequence,
780 "operation_sequence_span":delivery.operation_sequence_span,
781 });
782 CancellationDelivery::from_payload(&payload)?;
783 self.cancellation_delivery_command_count = self.commands.len();
784 self.cancellation_delivery_intent = Some(delivery);
785 self.matched_recorded_pending = true;
786 Ok(true)
787 }
788
789 fn assert_root_cancellation_membership(&self) -> Result<()> {
790 if self
791 .scope_delivery
792 .as_ref()
793 .is_some_and(|replay| replay.active_scope != "root")
794 {
795 return Err(Error::CancellationScopeExecutionUnavailable);
796 }
797 Ok(())
798 }
799
800 pub(super) fn prepare_scalar_cancellation(
801 &mut self,
802 index: usize,
803 kind: CancellationCallKind,
804 group_path: &[ParallelGroupMetadata],
805 ) -> Result<bool> {
806 if self.prepare_scalar_scope_cancellation(index, kind, group_path)? {
807 return Ok(true);
808 }
809 if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 {
810 return Ok(false);
811 }
812 if !group_path.is_empty() && self.cancellation_delivery_intent.is_none() {
815 return Ok(false);
816 }
817 let Some(request) = self.cancellation_history.request.as_ref() else {
818 return Ok(false);
819 };
820 let sequence = self.recorded_commands.get(index).map_or_else(
821 || self.next_cancellation_sequence(),
822 |command| Ok(command.sequence()),
823 )?;
824 let pending = self.prepare_cancellation_delivery(CancellationDelivery {
825 request_id: request.request_id.clone(),
826 sequence,
827 call_kind: kind,
828 sequence_span: 1,
829 operation_sequence: None,
830 operation_sequence_span: 1,
831 })?;
832 if pending && index < self.recorded_commands.len() {
833 self.command_cursor = index + 1;
834 }
835 Ok(pending)
836 }
837
838 pub(super) fn prepare_group_cancellation(
839 &mut self,
840 descriptors: &[ParallelDescriptor],
841 ) -> Result<()> {
842 let Some(request) = self.cancellation_history.request.as_ref() else {
843 return Ok(());
844 };
845 let Some(group) = descriptors.first().and_then(|leaf| leaf.group_path.first()) else {
846 return Ok(());
847 };
848 self.prepare_cancellation_delivery(CancellationDelivery {
849 request_id: request.request_id.clone(),
850 sequence: group.parallel_group_base_sequence,
851 call_kind: CancellationCallKind::Parallel,
852 sequence_span: descriptors.len() as u64,
853 operation_sequence: None,
854 operation_sequence_span: 1,
855 })?;
856 Ok(())
857 }
858
859 pub(super) fn prepare_selection_handle_cancellation(
860 &mut self,
861 handle: &DurableOperationHandle,
862 ) -> Result<bool> {
863 if !self.cancellation_delivery_enabled || self.cancellation_shield_depth > 0 {
864 return Ok(false);
865 }
866 validate_selection_delivery_handle(self, handle)?;
867 if !self
868 .cancellation_history
869 .eligible(handle.base_sequence, handle.size as u64)
870 {
871 return Ok(false);
872 }
873 let request_id = self
874 .cancellation_history
875 .request
876 .as_ref()
877 .expect("eligible request")
878 .request_id
879 .clone();
880 self.prepare_cancellation_delivery(CancellationDelivery {
881 request_id,
882 sequence: self.next_cancellation_sequence()?,
883 call_kind: CancellationCallKind::SelectionHandle,
884 sequence_span: 1,
885 operation_sequence: Some(handle.base_sequence),
886 operation_sequence_span: handle.size as u64,
887 })
888 }
889
890 pub(super) fn expand_cancellation_group(
891 &mut self,
892 descriptors: &[ParallelDescriptor],
893 ) -> Result<()> {
894 let Some(RecordedCommand::CancellationGroup {
895 sequence,
896 span,
897 original,
898 }) = self.recorded_commands.get(self.command_cursor).cloned()
899 else {
900 return Ok(());
901 };
902 self.validate_cancellation_call(
903 sequence,
904 CancellationCallKind::Parallel,
905 CancellationCallKind::Parallel,
906 )?;
907 if descriptors.len() as u64 != span {
908 return Err(invalid_recorded_history(
909 "cooperative_cancellation_call_mismatch",
910 sequence,
911 &format!("parallel group with {span} durable leaves"),
912 &format!("{} durable leaves", descriptors.len()),
913 "authored parallel group span differs from its committed cancellation",
914 ));
915 }
916 let mut original = original.into_iter().peekable();
917 let mut leaves = Vec::with_capacity(descriptors.len());
918 for descriptor in descriptors {
919 let leaf_sequence = sequence + descriptor.offset as u64;
920 let command = if original.peek().map(RecordedCommand::sequence) == Some(leaf_sequence) {
921 original.next()
922 } else {
923 None
924 };
925 if self
926 .cancellation_history
927 .resolved_before_request
928 .contains(&leaf_sequence)
929 {
930 let command = command.ok_or_else(|| {
931 invalid_recorded_history(
932 "cooperative_cancellation_call_mismatch",
933 leaf_sequence,
934 "recorded earlier result",
935 "missing durable call",
936 "parallel cancellation cannot discard an earlier committed result",
937 )
938 })?;
939 leaves.push(command);
940 continue;
941 }
942 let kind = match &descriptor.operation {
943 ParallelOperation::Activity { .. } => CancellationCallKind::Activity,
944 ParallelOperation::ChildWorkflow { .. } => CancellationCallKind::Child,
945 ParallelOperation::Timer(_) => CancellationCallKind::Timer,
946 ParallelOperation::Signal(_) => CancellationCallKind::Signal,
947 ParallelOperation::Condition { .. } => CancellationCallKind::Condition,
948 ParallelOperation::Group(_) => unreachable!("descriptor is a durable leaf"),
949 };
950 leaves.push(RecordedCommand::CancellationBoundary {
951 sequence: leaf_sequence,
952 call_kind: kind,
953 original: command.map(Box::new),
954 });
955 }
956 self.recorded_commands
957 .splice(self.command_cursor..=self.command_cursor, leaves);
958 Ok(())
959 }
960
961 pub(super) fn cancellation_error(&mut self) -> Error {
962 if self.cancellation_consumed {
963 self.start_cancellation_clock();
964 }
965 match (
966 &self.cancellation_history.request,
967 &self.cancellation_history.delivery,
968 ) {
969 (Some(request), Some(delivery)) => {
970 let mut request = request.clone();
971 request.context = request.context.map(|context| {
972 context.with_replay(cancellation_replay_clock::active_binding())
973 });
974 Error::CooperativeCancellationRequested(CooperativeCancellationRequested {
975 request,
976 delivery: delivery.clone(),
977 })
978 }
979 _ => Error::WorkflowCancellationRequested(WorkflowCancellationRequested),
980 }
981 }
982
983 fn validate_cancellation_call(
984 &self,
985 sequence: u64,
986 kind: CancellationCallKind,
987 recorded_kind: CancellationCallKind,
988 ) -> Result<()> {
989 if kind != recorded_kind || self.cancellation_shield_depth > 0 {
990 return Err(invalid_recorded_history(
991 "cooperative_cancellation_call_mismatch",
992 sequence,
993 "matching unshielded authored call",
994 &format!("{kind:?}"),
995 "committed cancellation call kind or cleanup scope changed",
996 ));
997 }
998 Ok(())
999 }
1000
1001 pub(super) fn cancellation_replay_command(
1002 &mut self,
1003 index: usize,
1004 kind: CancellationCallKind,
1005 ) -> Result<Option<RecordedCommand>> {
1006 match self.recorded_commands.get(index).cloned() {
1007 Some(RecordedCommand::CancellationBoundary {
1008 sequence,
1009 call_kind,
1010 original,
1011 }) => {
1012 if let Some(original) = original {
1013 return Ok(Some(*original));
1014 }
1015 self.replay_scope_cancellation_at(index, kind)?;
1016 self.assert_root_cancellation_membership()?;
1017 self.validate_cancellation_call(sequence, kind, call_kind)?;
1018 self.cancellation_consumed = true;
1019 self.cancel_requested = true;
1020 self.command_cursor = index + 1;
1021 Err(self.cancellation_error())
1022 }
1023 command => Ok(command),
1024 }
1025 }
1026
1027 pub(super) fn replay_cancellation_at(
1028 &mut self,
1029 index: usize,
1030 kind: CancellationCallKind,
1031 ) -> Result<()> {
1032 self.replay_scope_cancellation_at(index, kind)?;
1033 if let Some(RecordedCommand::CancellationBoundary {
1034 sequence,
1035 call_kind,
1036 ..
1037 }) = self.recorded_commands.get(index)
1038 {
1039 self.assert_root_cancellation_membership()?;
1040 self.validate_cancellation_call(*sequence, kind, *call_kind)?;
1041 self.cancellation_consumed = true;
1042 self.cancel_requested = true;
1043 self.command_cursor = index + 1;
1044 return Err(self.cancellation_error());
1045 }
1046 Ok(())
1047 }
1048
1049 pub(super) fn replay_selection_handle_cancellation(
1050 &mut self,
1051 handle: &DurableOperationHandle,
1052 ) -> Result<()> {
1053 let Some(RecordedCommand::CancellationBoundary {
1054 sequence,
1055 call_kind,
1056 original,
1057 }) = self.recorded_commands.get(self.command_cursor)
1058 else {
1059 return Ok(());
1060 };
1061 self.validate_cancellation_call(
1062 *sequence,
1063 CancellationCallKind::SelectionHandle,
1064 *call_kind,
1065 )?;
1066 let delivery = self
1067 .cancellation_history
1068 .delivery
1069 .as_ref()
1070 .expect("bound canonical delivery");
1071 if original.is_some()
1072 || delivery.operation_sequence != Some(handle.base_sequence)
1073 || delivery.operation_sequence_span != handle.size as u64
1074 {
1075 return Err(invalid_recorded_history(
1076 "cooperative_cancellation_call_mismatch",
1077 *sequence,
1078 "selection handle for the committed operation range",
1079 &format!("{}:{}", handle.base_sequence, handle.size),
1080 "cancellation delivery targets a different authored selection member",
1081 ));
1082 }
1083 validate_selection_delivery_handle(self, handle)?;
1084 self.replay_cancellation_at(self.command_cursor, CancellationCallKind::SelectionHandle)
1085 }
1086}
1087
1088impl WorkflowContext {
1089 pub fn cancellation_context(&self) -> Result<Option<CancellationContext>> {
1091 let state = self
1092 .state
1093 .lock()
1094 .map_err(|_| Error::WorkflowStatePoisoned)?;
1095 Ok(
1096 if state.cancellation_consumed && self.cancellation_scope_id == "root" {
1097 state
1098 .cancellation_history
1099 .request
1100 .as_ref()
1101 .and_then(|request| request.context.clone())
1102 .map(|context| context.with_replay(Some(Arc::downgrade(&self.state))))
1103 } else {
1104 None
1105 },
1106 )
1107 }
1108
1109 pub fn cancellation_shield(&self) -> Result<CancellationShield> {
1114 let mut state = self
1115 .state
1116 .lock()
1117 .map_err(|_| Error::WorkflowStatePoisoned)?;
1118 state.cancellation_shield_depth = state
1119 .cancellation_shield_depth
1120 .checked_add(1)
1121 .ok_or_else(|| invalid("cancellation shield nesting overflowed"))?;
1122 Ok(CancellationShield {
1123 state: Arc::clone(&self.state),
1124 })
1125 }
1126}
1127
1128pub(super) fn supports_protocol(version: &str) -> bool {
1129 let Some((major, minor)) = version.split_once('.') else {
1130 return false;
1131 };
1132 major == "1"
1133 && !minor.is_empty()
1134 && minor.bytes().all(|byte| byte.is_ascii_digit())
1135 && minor.parse::<u64>().is_ok_and(|minor| minor >= 20)
1136}
1137
1138fn require_discovery(info: &Value) -> Result<()> {
1139 let protocol = &info["worker_protocol"];
1140 if protocol["server_capabilities"]["cooperative_cancellation"].as_bool() != Some(true)
1141 || !protocol["version"].as_str().is_some_and(supports_protocol)
1142 {
1143 return Err(Error::CooperativeCancellationUnavailable(
1144 "runtime discovery must explicitly advertise support and compatible protocol 1.20"
1145 .to_string(),
1146 ));
1147 }
1148 Ok(())
1149}
1150
1151fn acknowledgment(
1152 value: &Value,
1153 workflow_id: &str,
1154 selected_run: Option<&str>,
1155) -> Result<WorkflowCancellationRequest> {
1156 if value["accepted"].as_bool() != Some(true)
1157 || text(value, "workflow_id")? != workflow_id
1158 || selected_run.is_some_and(|run| value["run_id"].as_str() != Some(run))
1159 {
1160 return Err(invalid(
1161 "acknowledgment does not match the accepted workflow/run request",
1162 ));
1163 }
1164 Ok(WorkflowCancellationRequest {
1165 workflow_id: workflow_id.to_owned(),
1166 run_id: text(value, "run_id")?.to_owned(),
1167 duplicate: value["duplicate"]
1168 .as_bool()
1169 .ok_or_else(|| invalid("duplicate must be a boolean"))?,
1170 cancellation_request: CancellationRequest::from_observation(
1171 &value["cancellation_request"],
1172 )?,
1173 })
1174}
1175
1176impl Client {
1177 pub async fn acknowledge_activity_cancellation(
1183 &self,
1184 task_id: &str,
1185 activity_attempt_id: &str,
1186 lease_owner: &str,
1187 request_id: &str,
1188 ) -> Result<Value> {
1189 if [task_id, activity_attempt_id, lease_owner, request_id]
1190 .iter()
1191 .any(|value| value.trim().is_empty() || value.len() > 255)
1192 {
1193 return Err(invalid(
1194 "activity stop receipt requires bounded original claim and request identities",
1195 ));
1196 }
1197 tokio::time::timeout(CONTROL_BUDGET, async {
1198 let value: Value = activity_task_response(
1199 self.request_json(
1200 reqwest::Method::POST,
1201 &format!("/worker/activity-tasks/{}/acknowledge-cancellation", percent_encode_path_segment(task_id)),
1202 RequestProtocol::Worker("1.20"),
1203 Some(&json!({"activity_attempt_id":activity_attempt_id,"lease_owner":lease_owner,"request_id":request_id})),
1204 ).await,
1205 "acknowledge-cancellation", task_id, activity_attempt_id,
1206 )?;
1207 if value["task_id"].as_str() != Some(task_id)
1208 || value["activity_attempt_id"].as_str() != Some(activity_attempt_id)
1209 || value["lease_owner"].as_str() != Some(lease_owner)
1210 || value["request_id"].as_str() != Some(request_id)
1211 || value["acknowledged"].as_bool() != Some(true)
1212 || value["duplicate"].as_bool().is_none()
1213 || value.get("reason") != Some(&Value::Null)
1214 || value["heartbeat_recorded"].as_bool() != Some(false)
1215 || text(&value, "history_event_id").is_err()
1216 {
1217 return Err(invalid("activity callback-stop acknowledgment did not prove the original claim and receipt"));
1218 }
1219 Ok(value)
1220 }).await.map_err(|_| Error::Timeout)?
1221 }
1222
1223 pub async fn activity_task_status(
1229 &self,
1230 task_id: &str,
1231 activity_attempt_id: &str,
1232 lease_owner: &str,
1233 ) -> Result<Value> {
1234 if [task_id, activity_attempt_id, lease_owner]
1235 .iter()
1236 .any(|value| value.trim().is_empty())
1237 {
1238 return Err(invalid("activity observation requires the actual claim"));
1239 }
1240 tokio::time::timeout(CONTROL_BUDGET, async {
1241 let value: Value = activity_task_response(
1242 self.request_json(
1243 reqwest::Method::POST,
1244 &format!(
1245 "/worker/activity-tasks/{}/status",
1246 percent_encode_path_segment(task_id)
1247 ),
1248 RequestProtocol::Worker("1.20"),
1249 Some(&json!({
1250 "activity_attempt_id":activity_attempt_id,"lease_owner":lease_owner
1251 })),
1252 )
1253 .await,
1254 "status",
1255 task_id,
1256 activity_attempt_id,
1257 )?;
1258 if value["task_id"].as_str() != Some(task_id)
1259 || value["activity_attempt_id"].as_str() != Some(activity_attempt_id)
1260 || value["lease_owner"].as_str() != Some(lease_owner)
1261 || value["can_continue"].as_bool().is_none()
1262 || value["cancel_requested"].as_bool().is_none()
1263 || value["heartbeat_recorded"].as_bool() != Some(false)
1264 {
1265 return Err(invalid(
1266 "activity observation did not acknowledge the exact claim",
1267 ));
1268 }
1269 Ok(value)
1270 })
1271 .await
1272 .map_err(|_| Error::Timeout)?
1273 }
1274
1275 pub async fn poll_cooperative_workflow_task(
1283 &self,
1284 worker_id: &str,
1285 task_queue: &str,
1286 timeout: Duration,
1287 ) -> Result<CooperativeWorkflowTaskPoll> {
1288 self.poll_cooperative_workflow_task_with_request_id(
1289 worker_id,
1290 task_queue,
1291 timeout,
1292 &unique_request_id("rust-workflow-poll"),
1293 1,
1294 )
1295 .await
1296 }
1297
1298 async fn poll_cooperative_workflow_task_with_request_id(
1299 &self,
1300 worker_id: &str,
1301 task_queue: &str,
1302 timeout: Duration,
1303 poll_request_id: &str,
1304 transport_retries: usize,
1305 ) -> Result<CooperativeWorkflowTaskPoll> {
1306 if worker_id.trim().is_empty()
1307 || task_queue.trim().is_empty()
1308 || poll_request_id.trim().is_empty()
1309 {
1310 return Err(invalid(
1311 "cooperative polling requires a worker and task queue",
1312 ));
1313 }
1314 let budget = timeout
1315 .checked_add(CONTROL_BUDGET)
1316 .ok_or_else(|| invalid("cooperative poll timeout exceeds its supported budget"))?;
1317 let body = json!({
1318 "worker_id":worker_id, "task_queue":task_queue,
1319 "poll_request_id":poll_request_id,
1320 "timeout_seconds":long_poll_timeout_seconds(timeout),
1321 "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE,
1322 "build_id":self.worker_build_id,
1323 });
1324 tokio::time::timeout(budget, async {
1325 let value: Value = self
1326 .poll_request_json(
1327 "/worker/workflow-tasks/poll",
1328 RequestProtocol::Worker("1.20"),
1329 &body,
1330 budget,
1331 transport_retries,
1332 )
1333 .await?;
1334 let mut response: PollWorkflowTaskResponse = serde_json::from_value(value.clone())
1335 .map_err(|_| invalid("cooperative poll returned a malformed envelope or task"))?;
1336 let outcome = response.outcome();
1337 let task = if let Some(task) = response.task.take() {
1338 let (owner, _) = cancellation_claim(&task)?;
1339 if owner != worker_id
1340 || value["task"]["workflow_task_attempt"].as_u64()
1341 != Some(task.workflow_task_attempt)
1342 {
1343 return Err(invalid(
1344 "cooperative poll did not return the caller's actual owner and attempt",
1345 ));
1346 }
1347 let cancellation_request = match value["task"].get("cancellation_request") {
1348 None | Some(Value::Null) => None,
1349 Some(observation) => Some(CancellationRequest::from_observation(observation)?),
1350 };
1351 let mut claim = CooperativeWorkflowTask {
1352 task,
1353 cancellation_request,
1354 sticky_snapshot: None,
1355 };
1356 if self.sticky_cache.is_some() {
1357 claim.sticky_snapshot = Some(
1358 self.load_sticky_history(&mut claim.task, &value["task"], worker_id)
1359 .await?,
1360 );
1361 } else {
1362 self.load_cooperative_claim_history(&mut claim).await?;
1363 }
1364 Some(claim)
1365 } else {
1366 None
1367 };
1368 Ok(CooperativeWorkflowTaskPoll {
1369 task,
1370 outcome,
1371 protocol_version: response.protocol_version,
1372 server_capabilities: response.server_capabilities,
1373 })
1374 })
1375 .await
1376 .map_err(|_| Error::Timeout)?
1377 }
1378
1379 async fn load_cooperative_claim_history(
1380 &self,
1381 claim: &mut CooperativeWorkflowTask,
1382 ) -> Result<()> {
1383 let (owner, _) = cancellation_claim(&claim.task)?;
1384 let owner = owner.to_owned();
1385 let mut token = claim.task.next_history_page_token.clone();
1386 let mut seen = BTreeSet::new();
1387 while let Some(current) = token.take() {
1388 if current.trim().is_empty()
1389 || seen.len() >= MAX_REFRESH_PAGES
1390 || !seen.insert(current.clone())
1391 {
1392 return Err(invalid(
1393 "claim history exceeded its page bound or returned an invalid/repeated token",
1394 ));
1395 }
1396 let body = json!({
1397 "lease_owner":owner,"workflow_task_attempt":claim.task.workflow_task_attempt,
1398 "next_history_page_token":current,"history_page_size":WORKFLOW_HISTORY_PAGE_SIZE,
1399 });
1400 let value: Value = self
1401 .request_json(
1402 reqwest::Method::POST,
1403 &format!(
1404 "/worker/workflow-tasks/{}/history",
1405 percent_encode_path_segment(&claim.task.task_id)
1406 ),
1407 RequestProtocol::Worker("1.20"),
1408 Some(&body),
1409 )
1410 .await?;
1411 if value["task_id"].as_str() != Some(claim.task.task_id.as_str())
1412 || value["workflow_task_attempt"].as_u64() != Some(claim.task.workflow_task_attempt)
1413 {
1414 return Err(invalid(
1415 "claim history changed the selected task or attempt",
1416 ));
1417 }
1418 let events = value["history_events"]
1419 .as_array()
1420 .ok_or_else(|| invalid("claim history page must contain an event array"))?;
1421 if events.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize {
1422 return Err(invalid(
1423 "claim history page exceeds its requested event limit",
1424 ));
1425 }
1426 token = match value.get("next_history_page_token") {
1427 Some(Value::Null) => None,
1428 Some(Value::String(next)) if !next.trim().is_empty() && !events.is_empty() => {
1429 Some(next.clone())
1430 }
1431 _ => return Err(invalid("claim history page token or progress is invalid")),
1432 };
1433 let page: WorkflowTaskHistoryPage = serde_json::from_value(value)
1434 .map_err(|_| invalid("claim history page contains malformed fields or events"))?;
1435 if page
1436 .history_events
1437 .iter()
1438 .any(|event| event.event_type.trim().is_empty())
1439 {
1440 return Err(invalid("claim history event type must be non-empty"));
1441 }
1442 claim.task.append_history_page(page);
1443 }
1444 Ok(())
1445 }
1446
1447 pub async fn heartbeat_workflow_task(
1452 &self,
1453 task: &WorkflowTask,
1454 original: Option<&CancellationRequest>,
1455 ) -> Result<WorkflowTaskHeartbeat> {
1456 self.heartbeat_workflow_task_with_protocol(task, original, "1.20")
1457 .await
1458 }
1459
1460 pub(super) async fn heartbeat_workflow_task_with_protocol(
1461 &self,
1462 task: &WorkflowTask,
1463 original: Option<&CancellationRequest>,
1464 protocol: &'static str,
1465 ) -> Result<WorkflowTaskHeartbeat> {
1466 let (owner, _) = cancellation_claim(task)?;
1467 if let Some(original) = original {
1468 original.validate_observation()?;
1469 }
1470 tokio::time::timeout(CONTROL_BUDGET, async {
1471 let response: Value = self.request_json(
1472 reqwest::Method::POST,
1473 &format!("/worker/workflow-tasks/{}/heartbeat", percent_encode_path_segment(&task.task_id)),
1474 RequestProtocol::Worker(protocol),
1475 Some(&json!({"lease_owner":owner,"workflow_task_attempt":task.workflow_task_attempt})),
1476 ).await?;
1477 if response["task_id"].as_str() != Some(task.task_id.as_str())
1478 || response["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt)
1479 || response["lease_owner"].as_str() != Some(owner)
1480 || response["renewed"].as_bool() != Some(true)
1481 || response.get("reason") != Some(&Value::Null)
1482 || response["task_status"].as_str() != Some("leased")
1483 {
1484 return Err(invalid("workflow heartbeat did not acknowledge the exact leased claim"));
1485 }
1486 let expires = text(&response, "lease_expires_at")?;
1487 DateTime::parse_from_rfc3339(expires).map_err(|_| invalid("workflow heartbeat lease expiry must include a timezone"))?;
1488 let run_status = text(&response, "run_status")?;
1489 if !matches!(run_status, "pending" | "running" | "waiting") {
1490 return Err(invalid("workflow heartbeat did not acknowledge an active run"));
1491 }
1492 let cancellation_request = match response.get("cancellation_request") {
1493 None | Some(Value::Null) if original.is_none() => None,
1494 None | Some(Value::Null) => return Err(invalid("workflow heartbeat omitted its original pending request")),
1495 Some(observation) => {
1496 let current = CancellationRequest::from_observation(observation)?;
1497 Some(match original {
1498 Some(original) => original.preserve_observation(¤t)?,
1499 None => current,
1500 })
1501 },
1502 };
1503 Ok(WorkflowTaskHeartbeat {
1504 task_id:task.task_id.clone(), workflow_task_attempt:task.workflow_task_attempt,
1505 lease_owner:owner.to_owned(), lease_expires_at:expires.to_owned(),
1506 run_status:run_status.to_owned(), cancellation_request,
1507 })
1508 }).await.map_err(|_| Error::Timeout)?
1509 }
1510
1511 pub async fn deliver_workflow_cancellation(
1519 &self,
1520 task: &WorkflowTask,
1521 delivery: &CancellationDelivery,
1522 ) -> Result<CancellationDeliveryReply> {
1523 let (owner, run_id) = cancellation_claim(task)?;
1524 let body = json!({
1525 "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt,
1526 "request_id":delivery.request_id, "sequence":delivery.sequence,
1527 "call_kind":delivery.call_kind, "sequence_span":delivery.sequence_span,
1528 "operation_sequence":delivery.operation_sequence,
1529 "operation_sequence_span":delivery.operation_sequence_span
1530 });
1531 let mut payload = body.clone();
1532 payload["workflow_command_id"] = json!(delivery.request_id);
1533 if CancellationDelivery::from_payload(&payload)? != *delivery {
1534 return Err(invalid(
1535 "delivery must name a valid authored operation range",
1536 ));
1537 }
1538 tokio::time::timeout(CONTROL_BUDGET, async {
1539 let path = format!(
1540 "/worker/workflow-tasks/{}/deliver-cancellation",
1541 percent_encode_path_segment(&task.task_id)
1542 );
1543 let response: Value = self
1544 .request_json(
1545 reqwest::Method::POST,
1546 &path,
1547 RequestProtocol::Worker("1.20"),
1548 Some(&body),
1549 )
1550 .await?;
1551 if response["delivered"].as_bool() == Some(false)
1552 && match response["reason"].as_str() {
1553 Some("cancellation_waiting_for_child") => matches!(
1554 delivery.call_kind,
1555 CancellationCallKind::Child
1556 | CancellationCallKind::Parallel
1557 | CancellationCallKind::SelectionHandle
1558 ),
1559 Some("cancellation_waiting_for_activity") => matches!(
1560 delivery.call_kind,
1561 CancellationCallKind::Activity
1562 | CancellationCallKind::LocalActivity
1563 | CancellationCallKind::Parallel
1564 | CancellationCallKind::SelectionHandle
1565 ),
1566 _ => false,
1567 }
1568 && response["claim_released"].as_bool() == Some(true)
1569 && response["task_id"].as_str() == Some(task.task_id.as_str())
1570 && response["workflow_run_id"].as_str() == Some(run_id)
1571 && [
1572 "request_id",
1573 "sequence",
1574 "call_kind",
1575 "sequence_span",
1576 "operation_sequence",
1577 "operation_sequence_span",
1578 ]
1579 .iter()
1580 .all(|field| response.get(*field).is_none_or(Value::is_null))
1581 {
1582 return Ok(CancellationDeliveryReply::ClaimReleased {
1583 task_id: task.task_id.clone(),
1584 run_id: run_id.to_owned(),
1585 });
1586 }
1587 if response["delivered"].as_bool() != Some(true)
1588 || response["task_id"].as_str() != Some(task.task_id.as_str())
1589 || response["workflow_run_id"].as_str() != Some(run_id)
1590 || response.get("reason") != Some(&Value::Null)
1591 || [
1592 "sequence_span",
1593 "operation_sequence",
1594 "operation_sequence_span",
1595 ]
1596 .iter()
1597 .any(|field| response.get(*field).is_none())
1598 {
1599 return Err(invalid(
1600 "delivery acknowledgment does not match the selected task/run",
1601 ));
1602 }
1603 let mut recorded = response.clone();
1604 recorded["workflow_command_id"] = response["request_id"].clone();
1605 if CancellationDelivery::from_payload(&recorded)? != *delivery {
1606 return Err(invalid(
1607 "delivery acknowledgment changed its authored operation range",
1608 ));
1609 }
1610 Ok(CancellationDeliveryReply::Delivered(
1611 CancellationDeliveryReceipt {
1612 task_id: task.task_id.clone(),
1613 run_id: run_id.to_owned(),
1614 delivery: delivery.clone(),
1615 },
1616 ))
1617 })
1618 .await
1619 .map_err(|_| Error::Timeout)?
1620 }
1621
1622 pub async fn refresh_workflow_cancellation_history(
1629 &self,
1630 task: &WorkflowTask,
1631 observation: &CancellationRequest,
1632 ) -> Result<Vec<HistoryEvent>> {
1633 let (owner, run_id) = cancellation_claim(task)?;
1634 observation.validate_observation()?;
1635 let first_token = observation
1636 .history_refresh_page_token
1637 .clone()
1638 .expect("validated history token");
1639 tokio::time::timeout(CONTROL_BUDGET, async {
1640 let path = format!("/worker/workflow-tasks/{}/history", percent_encode_path_segment(&task.task_id));
1641 let mut token = Some(first_token);
1642 let mut seen = BTreeSet::new();
1643 let mut history = Vec::new();
1644 while let Some(current) = token.take() {
1645 if seen.len() >= MAX_REFRESH_PAGES || !seen.insert(current.clone()) {
1646 return Err(invalid("canonical history refresh exceeded its page bound or repeated a token"));
1647 }
1648 let body = json!({
1649 "lease_owner":owner, "workflow_task_attempt":task.workflow_task_attempt,
1650 "next_history_page_token":current, "history_page_size":WORKFLOW_HISTORY_PAGE_SIZE,
1651 });
1652 let page: Value = self.request_json(
1653 reqwest::Method::POST, &path, RequestProtocol::Worker("1.20"), Some(&body),
1654 ).await?;
1655 if page["task_id"].as_str() != Some(task.task_id.as_str())
1656 || page["workflow_task_attempt"].as_u64() != Some(task.workflow_task_attempt)
1657 {
1658 return Err(invalid("canonical history page changed the selected task/attempt"));
1659 }
1660 let events = page["history_events"].as_array().ok_or_else(|| invalid("canonical history page must contain an event array"))?;
1661 if events.len() > WORKFLOW_HISTORY_PAGE_SIZE as usize {
1662 return Err(invalid("canonical history page exceeds its requested event limit"));
1663 }
1664 token = match page.get("next_history_page_token") {
1665 Some(Value::Null) => None,
1666 Some(Value::String(next)) if !next.is_empty() && !events.is_empty() => Some(next.clone()),
1667 _ => return Err(invalid("canonical history page token or progress is invalid")),
1668 };
1669 for event in events {
1670 let event: HistoryEvent = serde_json::from_value(event.clone()).map_err(|_| invalid("canonical history page contains a malformed event"))?;
1671 if event.event_type.trim().is_empty() {
1672 return Err(invalid("canonical history event type must be non-empty"));
1673 }
1674 history.push(event);
1675 }
1676 }
1677 let canonical = CancellationHistory::from_events(&history, run_id, Some(observation))?;
1678 if canonical.request_index >= history.len() {
1679 return Err(invalid("canonical refresh omitted the original request event"));
1680 }
1681 Ok(history)
1682 }).await.map_err(|_| Error::Timeout)?
1683 }
1684
1685 pub async fn request_workflow_cancellation(
1692 &self,
1693 workflow_id: &str,
1694 options: CooperativeCancellationOptions,
1695 ) -> Result<WorkflowCancellationRequest> {
1696 self.request_workflow_cancellation_target(workflow_id, None, options)
1697 .await
1698 }
1699
1700 pub async fn request_workflow_run_cancellation(
1702 &self,
1703 workflow_id: &str,
1704 run_id: &str,
1705 options: CooperativeCancellationOptions,
1706 ) -> Result<WorkflowCancellationRequest> {
1707 self.request_workflow_cancellation_target(workflow_id, Some(run_id), options)
1708 .await
1709 }
1710
1711 async fn request_workflow_cancellation_target(
1712 &self,
1713 workflow_id: &str,
1714 run_id: Option<&str>,
1715 options: CooperativeCancellationOptions,
1716 ) -> Result<WorkflowCancellationRequest> {
1717 if workflow_id.trim().is_empty() || run_id.is_some_and(|id| id.trim().is_empty()) {
1718 return Err(invalid(
1719 "workflow and selected run identities must be non-empty",
1720 ));
1721 }
1722 if options
1723 .cleanup_timeout_seconds
1724 .is_some_and(|seconds| !(1..=3600).contains(&seconds))
1725 || options
1726 .reason
1727 .as_ref()
1728 .is_some_and(|reason| reason.chars().count() > 1000)
1729 {
1730 return Err(invalid(
1731 "cleanup timeout must be 1..3600 seconds and reason at most 1000 characters",
1732 ));
1733 }
1734
1735 tokio::time::timeout(CONTROL_BUDGET, async {
1737 let info: Value = self
1738 .request_json(
1739 reqwest::Method::GET,
1740 "/cluster/info",
1741 RequestProtocol::ControlPlane,
1742 Option::<&Value>::None,
1743 )
1744 .await?;
1745 require_discovery(&info)?;
1746 let mut path = format!("/workflows/{}", percent_encode_path_segment(workflow_id));
1747 if let Some(run_id) = run_id {
1748 path.push_str(&format!("/runs/{}", percent_encode_path_segment(run_id)));
1749 }
1750 path.push_str("/request-cancellation");
1751 let response: Value = self
1752 .request_json(
1753 reqwest::Method::POST,
1754 &path,
1755 RequestProtocol::ControlPlane,
1756 Some(&options),
1757 )
1758 .await?;
1759 acknowledgment(&response, workflow_id, run_id)
1760 })
1761 .await
1762 .map_err(|_| Error::Timeout)?
1763 }
1764}
1765
1766impl Worker {
1767 pub(super) async fn poll_cooperative_activity_once(&self) -> Result<ManagedPollOutcome> {
1768 if self.client.worker_sessions_enabled {
1769 self.require_session_registration()?;
1770 }
1771 let poll_request_id = unique_request_id("rust-activity-poll");
1772 let response = self
1773 .retry_worker_operation(|| {
1774 self.client.poll_activity_task_response_with_request_id(
1775 &self.worker_id,
1776 &self.task_queue,
1777 self.poll_timeout,
1778 &poll_request_id,
1779 0,
1780 )
1781 })
1782 .await;
1783 let Some(response) = self.settle_worker_poll_response(response).await? else {
1784 return Ok(ManagedPollOutcome::Idle);
1785 };
1786 if response.outcome().should_stop() {
1787 return Ok(ManagedPollOutcome::Stop);
1788 }
1789 let Some(task) = response.task else {
1790 return Ok(ManagedPollOutcome::Idle);
1791 };
1792 let session = self.track_session_task(task.worker_session.as_ref())?;
1793 let task = task.task;
1794 let guard = ActivityClaimGuard::new(&self.client, &task, &self.worker_id)?;
1795 let _abandon_on_drop = AbandonActivityOnDrop(guard.clone());
1796 if guard.observe().await.is_err() {
1797 return Ok(ManagedPollOutcome::Handled);
1798 }
1799 let mut callback_started = false;
1800 let result = {
1801 let invocation = self.execute_cooperative_activity_task(
1802 &task,
1803 &guard,
1804 &mut callback_started,
1805 session,
1806 );
1807 tokio::pin!(invocation);
1808 loop {
1809 tokio::select! {
1810 biased;
1811 _ = guard.wait_for_shutdown() => {
1812 guard.abandon();
1813 break None;
1814 }
1815 result = &mut invocation => break Some(result),
1816 _ = tokio::time::sleep(Duration::from_secs(1)) => {
1817 if guard.observe().await.is_err() {
1818 break None;
1819 }
1820 }
1821 }
1822 }
1823 };
1824 let Some(result) = result else {
1826 if callback_started {
1827 guard.acknowledge_stopped().await?;
1828 }
1829 return Ok(ManagedPollOutcome::Handled);
1830 };
1831 if matches!(result, Err(Error::ActivityExecutionAbandoned(_)))
1834 || guard.observe().await.is_err()
1835 {
1836 if callback_started {
1837 guard.acknowledge_stopped().await?;
1838 }
1839 return Ok(ManagedPollOutcome::Handled);
1840 }
1841 let settlement = match result {
1842 Ok(value) => {
1843 self.client
1844 .complete_activity_task(
1845 &guard.task_id,
1846 &guard.attempt_id,
1847 &guard.owner,
1848 value,
1849 &task.payload_codec,
1850 )
1851 .await
1852 }
1853 Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
1854 Err(error) => {
1855 self.client
1856 .fail_activity_task(
1857 &guard.task_id,
1858 &guard.attempt_id,
1859 &guard.owner,
1860 error.to_string(),
1861 false,
1862 )
1863 .await
1864 }
1865 };
1866 if let Err(error) = settlement {
1867 if !activity_task_rejection_is_final(&error) {
1868 return Err(error);
1869 }
1870 }
1871 Ok(ManagedPollOutcome::Handled)
1872 }
1873
1874 async fn execute_cooperative_activity_task(
1875 &self,
1876 task: &ActivityTask,
1877 guard: &ActivityClaimGuard,
1878 callback_started: &mut bool,
1879 worker_session: Option<crate::WorkerSession>,
1880 ) -> Result<AvroValue> {
1881 validate_activity_task_payloads(task)?;
1882 let handler = self
1883 .activities
1884 .get(&task.activity_type)
1885 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
1886 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
1887 let context = ActivityContext {
1888 client: self.client.clone(),
1889 task_id: guard.task_id.clone(),
1890 activity_attempt_id: guard.attempt_id.clone(),
1891 lease_owner: guard.owner.clone(),
1892 activity_type: task.activity_type.clone(),
1893 attempt_number: task.attempt_number,
1894 task_queue: self.task_queue.clone(),
1895 worker_id: self.worker_id.clone(),
1896 claim_guard: Some(guard.clone()),
1897 local_heartbeats: None,
1898 worker_session,
1899 };
1900 guard.boundary()?;
1901 *callback_started = true;
1902 handler(context, args).await
1903 }
1904
1905 pub(super) async fn poll_cooperative_workflow_once(&self) -> Result<ManagedPollOutcome> {
1906 let poll_request_id = unique_request_id("rust-workflow-poll");
1907 let response = self
1908 .retry_worker_operation(|| {
1909 self.client.poll_cooperative_workflow_task_with_request_id(
1910 &self.worker_id,
1911 &self.task_queue,
1912 self.poll_timeout,
1913 &poll_request_id,
1914 0,
1915 )
1916 })
1917 .await;
1918 let Some(response) = self.settle_worker_poll_response(response).await? else {
1919 return Ok(ManagedPollOutcome::Idle);
1920 };
1921 if response.outcome.should_stop() {
1922 return Ok(ManagedPollOutcome::Stop);
1923 }
1924 let memo_updates_supported =
1925 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
1926 let Some(claim) = response.task else {
1927 return Ok(ManagedPollOutcome::Idle);
1928 };
1929 let Some(decision) = self.replay_cooperative_workflow_claim(&claim).await? else {
1932 return Ok(ManagedPollOutcome::Handled);
1933 };
1934 let (owner, _) = cancellation_claim(&claim.task)?;
1935 self.settle_workflow_task_decision(
1936 &claim.task.task_id,
1937 owner,
1938 claim.task.workflow_task_attempt,
1939 claim.task.run_id.as_deref(),
1940 Ok(decision),
1941 memo_updates_supported,
1942 claim.sticky_snapshot.clone(),
1943 )
1944 .await
1945 }
1946
1947 async fn replay_cooperative_workflow_claim(
1948 &self,
1949 claim: &CooperativeWorkflowTask,
1950 ) -> Result<Option<WorkflowTaskDecision>> {
1951 let (owner, run_id) = cancellation_claim(&claim.task)?;
1952 if owner != self.worker_id {
1953 return Err(invalid(
1954 "cooperative replay requires this worker's actual claim",
1955 ));
1956 }
1957 let mut task = claim.task.clone();
1958 let Some(observation) = claim.cancellation_request.as_ref() else {
1959 let canonical = CancellationHistory::from_events(&task.history_events, run_id, None)?;
1960 if canonical.request.is_some() {
1961 return Err(invalid(
1962 "cooperative replay omitted its original pending observation",
1963 ));
1964 }
1965 for _ in 0..1000 {
1966 let mut decision = self.execute_workflow_task_decision(task.clone())?;
1967 if let Some(intent) = decision.cancellation_scope_delivery.as_ref() {
1968 if !decision.commands.is_empty() {
1969 decision.cancellation_scope_delivery = None;
1970 return Ok(Some(decision));
1971 }
1972 let mut budget = CancellationScopeDeliveryBudget::new();
1973 budget.restrict(intent.context.deadline())?;
1974 let prepared = self
1975 .client
1976 .prepare_cancellation_scope_on_claim(
1977 &task,
1978 &intent.context,
1979 &intent.boundary,
1980 &budget,
1981 )
1982 .await?;
1983 budget.restrict(prepared.authority_deadline())?;
1984 let delivered = self
1985 .client
1986 .deliver_cancellation_scope_on_claim(&task, &prepared, &budget)
1987 .await?;
1988 task.history_events = delivered.history().to_vec();
1989 task.total_history_events = None;
1990 task.history_size_bytes = None;
1991 continue;
1992 }
1993 let Some(opening) = decision.cancellation_scope_opening.as_ref() else {
1994 return Ok(Some(decision));
1995 };
1996 if !decision.commands.is_empty() {
1997 decision.cancellation_scope_opening = None;
1998 return Ok(Some(decision));
1999 }
2000 let receipt = self
2001 .client
2002 .open_cancellation_scope_on_claim(
2003 &task,
2004 opening.sequence,
2005 &opening.parent_scope_id,
2006 opening.shield_parent,
2007 )
2008 .await?;
2009 task.history_events = receipt.history().to_vec();
2010 task.total_history_events = None;
2011 task.history_size_bytes = None;
2012 }
2013 return Err(invalid(
2014 "workflow exceeded the canonical scope opening replay limit",
2015 ));
2016 };
2017 task.history_events = self
2018 .client
2019 .refresh_workflow_cancellation_history(&claim.task, observation)
2020 .await?;
2021 for _ in 0..1000 {
2022 task.total_history_events = None;
2025 task.history_size_bytes = None;
2026 let mut decision = self.execute_workflow_task_decision_with_cancellation(
2027 task.clone(),
2028 Some(observation),
2029 )?;
2030 if let Some(intent) = decision.cancellation_scope_delivery.as_ref() {
2031 if !decision.commands.is_empty() {
2032 decision.cancellation_scope_delivery = None;
2033 return Ok(Some(decision));
2034 }
2035 let mut budget = CancellationScopeDeliveryBudget::new();
2036 budget.restrict(intent.context.deadline())?;
2037 let prepared = self
2038 .client
2039 .prepare_cancellation_scope_on_claim(
2040 &task,
2041 &intent.context,
2042 &intent.boundary,
2043 &budget,
2044 )
2045 .await?;
2046 budget.restrict(prepared.authority_deadline())?;
2047 let delivered = self
2048 .client
2049 .deliver_cancellation_scope_on_claim(&task, &prepared, &budget)
2050 .await?;
2051 task.history_events = delivered.history().to_vec();
2052 continue;
2053 }
2054 let Some(intent) = decision.cancellation_delivery.as_ref() else {
2055 return Ok(Some(decision));
2056 };
2057 if !decision.commands.is_empty() {
2058 decision.cancellation_delivery = None;
2062 return Ok(Some(decision));
2063 }
2064 let delivery_error = match self
2065 .client
2066 .deliver_workflow_cancellation(&claim.task, intent)
2067 .await
2068 {
2069 Ok(CancellationDeliveryReply::ClaimReleased { .. }) => return Ok(None),
2070 Ok(CancellationDeliveryReply::Delivered(_)) => None,
2071 Err(error) => Some(error),
2072 };
2073 task.history_events = self
2074 .client
2075 .refresh_workflow_cancellation_history(&claim.task, observation)
2076 .await?;
2077 let canonical =
2078 CancellationHistory::from_events(&task.history_events, run_id, Some(observation))?;
2079 if canonical.delivery.as_ref() != Some(intent) {
2080 return Err(delivery_error.unwrap_or_else(|| {
2081 invalid(
2082 "canonical history did not prove the exact intended cancellation delivery",
2083 )
2084 }));
2085 }
2086 }
2087 Err(invalid(
2088 "workflow replay did not converge on its canonical cancellation delivery",
2089 ))
2090 }
2091}
2092
2093#[derive(Clone, Debug)]
2094pub(super) struct ActivityClaimGuard {
2095 client: Client,
2096 task_id: String,
2097 attempt_id: String,
2098 owner: String,
2099 active: Arc<AtomicBool>,
2100 cancellation_receipt: Arc<Mutex<Option<Value>>>,
2101 stop: Option<Arc<AtomicBool>>,
2102}
2103
2104struct AbandonActivityOnDrop(ActivityClaimGuard);
2105
2106impl Drop for AbandonActivityOnDrop {
2107 fn drop(&mut self) {
2108 self.0.abandon();
2109 }
2110}
2111
2112impl ActivityClaimGuard {
2113 fn new(client: &Client, task: &ActivityTask, worker_id: &str) -> Result<Self> {
2114 let owner = task.lease_owner.as_deref().unwrap_or_default();
2115 let attempt = task
2116 .activity_attempt_id
2117 .as_deref()
2118 .or(task.attempt_id.as_deref())
2119 .unwrap_or_default();
2120 if task.task_id.trim().is_empty()
2121 || attempt.trim().is_empty()
2122 || owner.trim().is_empty()
2123 || owner != worker_id
2124 || matches!((&task.activity_attempt_id, &task.attempt_id), (Some(left), Some(right)) if left != right)
2125 {
2126 return Err(invalid(
2127 "activity execution requires the worker's actual immutable claim",
2128 ));
2129 }
2130 Ok(Self {
2131 client: client.clone(),
2132 task_id: task.task_id.clone(),
2133 attempt_id: attempt.to_owned(),
2134 owner: owner.to_owned(),
2135 active: Arc::new(AtomicBool::new(true)),
2136 cancellation_receipt: Arc::new(Mutex::new(None)),
2137 stop: client
2138 .worker_storage_admission
2139 .as_ref()
2140 .map(|admission| Arc::clone(&admission.stop)),
2141 })
2142 }
2143
2144 fn abandon(&self) {
2145 self.active.store(false, Ordering::SeqCst);
2146 }
2147
2148 async fn acknowledge_stopped(&self) -> Result<()> {
2150 self.abandon();
2151 let receipt = self
2152 .cancellation_receipt
2153 .lock()
2154 .map_err(|_| invalid("activity cancellation receipt lock was poisoned"))?
2155 .clone();
2156 if let Some(receipt) = receipt {
2157 self.client
2158 .acknowledge_activity_cancellation(
2159 &self.task_id,
2160 &self.attempt_id,
2161 &self.owner,
2162 text(&receipt, "request_id")?,
2163 )
2164 .await?;
2165 }
2166 Ok(())
2167 }
2168
2169 fn retain_cancellation_receipt(&self, value: &Value) -> Result<()> {
2170 if value["can_continue"].as_bool() != Some(false)
2171 || value["cancel_requested"].as_bool() != Some(true)
2172 {
2173 return Ok(());
2174 }
2175 let Some(receipt) = value
2176 .get("cancellation_acknowledgement")
2177 .filter(|receipt| receipt.is_object())
2178 else {
2179 return Ok(());
2180 };
2181 if !matches!(
2182 receipt["callback_state"].as_str(),
2183 Some("unknown" | "stopped")
2184 ) || [
2185 "request_id",
2186 "root_request_id",
2187 "cleanup_deadline_at",
2188 "cancellation_history_event_id",
2189 ]
2190 .iter()
2191 .any(|field| text(receipt, field).is_err())
2192 || DateTime::parse_from_rfc3339(text(receipt, "cleanup_deadline_at")?).is_err()
2193 {
2194 return Ok(());
2195 }
2196 let mut retained = self
2197 .cancellation_receipt
2198 .lock()
2199 .map_err(|_| invalid("activity cancellation receipt lock was poisoned"))?;
2200 if let Some(original) = retained.as_ref() {
2201 if [
2202 "request_id",
2203 "root_request_id",
2204 "cleanup_deadline_at",
2205 "cancellation_history_event_id",
2206 ]
2207 .iter()
2208 .any(|field| original[*field] != receipt[*field])
2209 {
2210 return Err(invalid(
2211 "activity cancellation observation changed its original identity or deadline",
2212 ));
2213 }
2214 } else {
2215 *retained = Some(receipt.clone());
2216 }
2217 Ok(())
2218 }
2219
2220 fn boundary(&self) -> Result<()> {
2221 if !self.active.load(Ordering::SeqCst)
2222 || self
2223 .stop
2224 .as_ref()
2225 .is_some_and(|stop| stop.load(Ordering::SeqCst))
2226 {
2227 self.abandon();
2228 return Err(Error::ActivityExecutionAbandoned(
2229 "callback completed, was abandoned, or its worker is stopping".into(),
2230 ));
2231 }
2232 Ok(())
2233 }
2234
2235 async fn wait_for_shutdown(&self) {
2236 if let Some(stop) = &self.stop {
2237 wait_for_worker_stop(stop).await;
2238 } else {
2239 std::future::pending::<()>().await;
2240 }
2241 }
2242
2243 async fn observe(&self) -> Result<()> {
2244 self.boundary()?;
2245 let result = self
2246 .client
2247 .activity_task_status(&self.task_id, &self.attempt_id, &self.owner)
2248 .await
2249 .and_then(|value| {
2250 self.retain_cancellation_receipt(&value)?;
2251 if value["can_continue"].as_bool() != Some(true)
2252 || value["cancel_requested"].as_bool() != Some(false)
2253 || value.get("reason") != Some(&Value::Null)
2254 || value["task_status"].as_str() != Some("leased")
2255 || value["attempt_status"].as_str() != Some("running")
2256 || value["activity_status"].as_str() != Some("running")
2257 {
2258 return Err(invalid("activity observation refused continuation"));
2259 }
2260 let mut bounds = vec![text(&value, "lease_expires_at")?];
2261 if let Some(deadlines) = value.get("deadlines").filter(|v| !v.is_null()) {
2262 if !deadlines.is_object() {
2263 return Err(invalid("activity execution deadlines must be an object"));
2264 }
2265 for kind in ["heartbeat", "start_to_close", "schedule_to_close"] {
2266 if let Some(deadline) = deadlines.get(kind).filter(|v| !v.is_null()) {
2267 bounds.push(
2268 deadline
2269 .as_str()
2270 .ok_or_else(|| invalid("invalid activity deadline"))?,
2271 );
2272 }
2273 }
2274 }
2275 if let Some(session) = value.get("worker_session").filter(|v| !v.is_null()) {
2276 if session["status"].as_str() != Some("active")
2277 || session["lease_owner"].as_str() != Some(self.owner.as_str())
2278 {
2279 return Err(invalid(
2280 "activity no longer owns its required worker session",
2281 ));
2282 }
2283 bounds.push(text(session, "lease_expires_at")?);
2284 bounds.push(text(session, "ttl_expires_at")?);
2285 }
2286 for bound in bounds {
2287 let deadline = DateTime::parse_from_rfc3339(bound)
2288 .map_err(|_| invalid("activity deadline must include a timezone"))?;
2289 let now = SystemTime::now()
2290 .duration_since(UNIX_EPOCH)
2291 .map_err(|_| invalid("activity observation clock precedes the epoch"))?;
2292 if !u128::try_from(deadline.timestamp_millis())
2293 .is_ok_and(|millis| millis > now.as_millis())
2294 {
2295 return Err(invalid("activity ownership or execution deadline elapsed"));
2296 }
2297 }
2298 self.boundary()
2299 });
2300 result.map_err(|error| {
2301 self.abandon();
2302 Error::ActivityExecutionAbandoned(error.to_string())
2303 })
2304 }
2305
2306 pub(super) async fn heartbeat<T: Serialize>(
2307 &self,
2308 context: &ActivityContext,
2309 details: T,
2310 ) -> Result<ActivityHeartbeatResponse> {
2311 if context.task_id != self.task_id
2312 || context.activity_attempt_id != self.attempt_id
2313 || context.lease_owner != self.owner
2314 || context.worker_id != self.owner
2315 {
2316 self.abandon();
2317 return Err(Error::ActivityExecutionAbandoned(
2318 "activity context changed its original claim".into(),
2319 ));
2320 }
2321 self.observe().await?;
2322 let result = tokio::time::timeout(CONTROL_BUDGET, async {
2323 let details = AvroValue::from_serialize(&details)?.into_json()?;
2324 self.boundary()?;
2325 let value: Value = self.client.request_json(
2326 reqwest::Method::POST,
2327 &format!("/worker/activity-tasks/{}/heartbeat", percent_encode_path_segment(&self.task_id)),
2328 RequestProtocol::Worker("1.20"),
2329 Some(&json!({"activity_attempt_id":self.attempt_id,"lease_owner":self.owner,"details":details})),
2330 ).await?;
2331 if value["task_id"].as_str() != Some(self.task_id.as_str())
2332 || value["activity_attempt_id"].as_str() != Some(self.attempt_id.as_str())
2333 || value["lease_owner"].as_str() != Some(self.owner.as_str())
2334 || value["can_continue"].as_bool() != Some(true)
2335 || value["cancel_requested"].as_bool() != Some(false)
2336 || value["heartbeat_recorded"].as_bool() != Some(true)
2337 {
2338 return Err(invalid("activity heartbeat lost its original claim"));
2339 }
2340 if let Some(session) = context.worker_session() {
2341 session.track(&value["worker_session"])?;
2342 }
2343 serde_json::from_value(value).map_err(Error::from)
2344 }).await.map_err(|_| Error::Timeout).and_then(|result| result);
2345 let response = result.map_err(|error| {
2346 self.abandon();
2347 Error::ActivityExecutionAbandoned(error.to_string())
2348 })?;
2349 self.observe().await?;
2350 Ok(response)
2351 }
2352}
2353
2354pub(super) fn cancellation_claim(task: &WorkflowTask) -> Result<(&str, &str)> {
2355 let owner = task
2356 .lease_owner
2357 .as_deref()
2358 .filter(|owner| !owner.trim().is_empty());
2359 let run = task.run_id.as_deref().filter(|run| !run.trim().is_empty());
2360 if task.task_id.trim().is_empty()
2361 || task.workflow_task_attempt == 0
2362 || task.workflow_task_attempt > MAX_SEQUENCE
2363 {
2364 return Err(invalid(
2365 "cancellation transport requires a valid task and attempt",
2366 ));
2367 }
2368 Ok((
2369 owner.ok_or_else(|| invalid("cancellation transport requires the actual lease owner"))?,
2370 run.ok_or_else(|| invalid("cancellation transport requires the selected durable run"))?,
2371 ))
2372}
2373
2374impl WorkflowHandle {
2375 pub async fn request_cancellation(
2377 &self,
2378 options: CooperativeCancellationOptions,
2379 ) -> Result<WorkflowCancellationRequest> {
2380 self.client
2381 .request_workflow_cancellation(&self.workflow_id, options)
2382 .await
2383 }
2384
2385 pub async fn request_selected_run_cancellation(
2387 &self,
2388 options: CooperativeCancellationOptions,
2389 ) -> Result<WorkflowCancellationRequest> {
2390 let run_id = self
2391 .run_id
2392 .as_deref()
2393 .ok_or_else(|| invalid("selected run_id is required"))?;
2394 self.client
2395 .request_workflow_run_cancellation(&self.workflow_id, run_id, options)
2396 .await
2397 }
2398}