Skip to content
Durable Workflowrust

durable_workflow/
local_activity.rs

1use super::*;
2
3const MAX_ATTEMPTS: u64 = 100;
4const MAX_HEARTBEATS: usize = 1000;
5const LEASE_RENEWAL_INTERVAL: Duration = Duration::from_secs(1);
6
7/// Retry and timeout settings for an activity executed by the workflow worker.
8///
9/// Local activities bypass the activity queue. They do not accept routing,
10/// schedule-to-start timeouts, or remote activity cancellation policies.
11#[derive(Clone, Debug, Default, PartialEq, Eq)]
12pub struct LocalActivityOptions {
13    pub retry_policy: Option<ActivityRetryPolicy>,
14    pub start_to_close_timeout: Option<Duration>,
15    pub schedule_to_close_timeout: Option<Duration>,
16    pub heartbeat_timeout: Option<Duration>,
17}
18
19impl LocalActivityOptions {
20    pub fn new() -> Self {
21        Self::default()
22    }
23    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
24        self.retry_policy = Some(policy);
25        self
26    }
27    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
28        self.start_to_close_timeout = Some(timeout);
29        self
30    }
31    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
32        self.schedule_to_close_timeout = Some(timeout);
33        self
34    }
35    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
36        self.heartbeat_timeout = Some(timeout);
37        self
38    }
39    fn into_activity_options(self) -> ActivityOptions {
40        ActivityOptions {
41            retry_policy: self.retry_policy,
42            start_to_close_timeout: self.start_to_close_timeout,
43            schedule_to_close_timeout: self.schedule_to_close_timeout,
44            heartbeat_timeout: self.heartbeat_timeout,
45            ..ActivityOptions::default()
46        }
47    }
48}
49
50pub(super) fn validate(options: &ValidatedActivityOptions) -> Result<()> {
51    let max_attempts = options
52        .retry_policy
53        .as_ref()
54        .and_then(|policy| policy["max_attempts"].as_u64())
55        .unwrap_or(1);
56    if max_attempts > MAX_ATTEMPTS {
57        return Err(Error::InvalidActivityOptions(ActivityOptionsError::new(
58            ActivityOptionsErrorKind::InvalidMaxAttempts,
59            Some("max_attempts"),
60            "local activities support at most 100 attempts",
61        )));
62    }
63    for seconds in [
64        options.start_to_close_timeout,
65        options.schedule_to_close_timeout,
66        options.heartbeat_timeout,
67    ]
68    .into_iter()
69    .flatten()
70    .chain(options.retry_policy.iter().flat_map(|p| {
71        p["backoff_seconds"]
72            .as_array()
73            .into_iter()
74            .flatten()
75            .filter_map(Value::as_u64)
76    })) {
77        if Instant::now()
78            .checked_add(Duration::from_secs(seconds))
79            .is_none()
80        {
81            return Err(Error::InvalidActivityOptions(ActivityOptionsError::new(
82                ActivityOptionsErrorKind::TimeoutOverflow,
83                Some("local_activity"),
84                "local timing exceeds the monotonic clock range",
85            )));
86        }
87    }
88    Ok(())
89}
90
91impl WorkflowContext {
92    /// Execute a registered activity in this workflow worker and record its outcome.
93    ///
94    /// Committed results are replayed without executing the callback. A worker
95    /// lost before acknowledgment can execute the callback again, so side effects
96    /// must be idempotent. Callbacks must yield to Tokio and avoid blocking work.
97    pub fn local_activity<T: Serialize>(
98        &self,
99        activity_type: impl Into<String>,
100        args: T,
101    ) -> ActivityCall {
102        self.local_activity_with_options(activity_type, LocalActivityOptions::new(), args)
103    }
104
105    pub fn local_activity_with_options<T: Serialize>(
106        &self,
107        activity_type: impl Into<String>,
108        options: LocalActivityOptions,
109        args: T,
110    ) -> ActivityCall {
111        let mut call =
112            self.activity_with_options(activity_type, options.into_activity_options(), args);
113        call.local = true;
114        call
115    }
116
117    /// Lossless Avro result, including bytes and large integers.
118    pub async fn local_activity_avro_value<T: Serialize>(
119        &self,
120        activity_type: impl Into<String>,
121        args: T,
122    ) -> Result<AvroValue> {
123        self.local_activity_avro_value_with_options(
124            activity_type,
125            LocalActivityOptions::new(),
126            args,
127        )
128        .await
129    }
130
131    pub async fn local_activity_avro_value_with_options<T: Serialize>(
132        &self,
133        activity_type: impl Into<String>,
134        options: LocalActivityOptions,
135        args: T,
136    ) -> Result<AvroValue> {
137        let mut call = self.local_activity_with_options(activity_type, options, args);
138        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
139    }
140
141    pub async fn local_activity_typed<I: Serialize, O: DeserializeOwned>(
142        &self,
143        activity_type: impl Into<String>,
144        args: I,
145    ) -> Result<O> {
146        self.local_activity_typed_with_options(activity_type, LocalActivityOptions::new(), args)
147            .await
148    }
149
150    pub async fn local_activity_typed_with_options<I: Serialize, O: DeserializeOwned>(
151        &self,
152        activity_type: impl Into<String>,
153        options: LocalActivityOptions,
154        args: I,
155    ) -> Result<O> {
156        let activity_type = activity_type.into();
157        let result = self
158            .local_activity_avro_value_with_options(activity_type.clone(), options, args)
159            .await?;
160        decode_handler_result(result, HandlerKind::Activity, &activity_type)
161    }
162}
163
164#[derive(Debug)]
165pub(super) struct Request {
166    pub command_index: usize,
167    pub options: ValidatedActivityOptions,
168    pub arguments: AvroValue,
169    pub result: Arc<Mutex<Option<ActivityOutcome>>>,
170}
171
172#[derive(Debug)]
173pub(super) struct Heartbeats {
174    active: bool,
175    started: Instant,
176    last: Instant,
177    reports: Vec<Value>,
178    capacity: usize,
179    overflowed: bool,
180}
181
182impl Heartbeats {
183    pub(super) fn captured_attempt() -> (Arc<Mutex<Self>>, impl Drop) {
184        let now = Instant::now();
185        let reports = Arc::new(Mutex::new(Self {
186            active: true,
187            started: now,
188            last: now,
189            reports: Vec::new(),
190            capacity: MAX_HEARTBEATS,
191            overflowed: false,
192        }));
193        let close = CloseAttempt(reports.clone());
194        (reports, close)
195    }
196
197    pub fn record<T: Serialize>(&mut self, details: T) -> Result<ActivityHeartbeatResponse> {
198        if !self.active {
199            return Ok(heartbeat_response(false));
200        }
201        if self.reports.len() >= self.capacity {
202            self.overflowed = true;
203            return Err(Error::WorkerLoop(
204                "local_activity_heartbeat_limit_exceeded: at most 1000 reports per local command"
205                    .into(),
206            ));
207        }
208        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
209        let now = Instant::now();
210        self.reports.push(
211            json!({"elapsed_ms": millis(now.duration_since(self.started)), "details": details}),
212        );
213        self.last = now;
214        Ok(heartbeat_response(true))
215    }
216}
217
218fn heartbeat_response(active: bool) -> ActivityHeartbeatResponse {
219    ActivityHeartbeatResponse {
220        cancel_requested: false,
221        heartbeat_recorded: active,
222        can_continue: Some(active),
223        reason: (!active).then(|| "local_activity_attempt_closed".into()),
224        run_closed_reason: None,
225        run_closed_at: None,
226        lease_expires_at: None,
227        last_heartbeat_at: None,
228    }
229}
230
231struct CloseAttempt(Arc<Mutex<Heartbeats>>);
232impl Drop for CloseAttempt {
233    fn drop(&mut self) {
234        if let Ok(mut heartbeats) = self.0.lock() {
235            heartbeats.active = false;
236        }
237    }
238}
239
240fn millis(duration: Duration) -> u64 {
241    u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
242}
243
244impl Worker {
245    pub(super) async fn execute_workflow_with_local_activities(
246        &self,
247        task: WorkflowTask,
248    ) -> Result<Option<WorkflowTaskDecision>> {
249        #[cfg(feature = "telemetry")]
250        if let Some(telemetry) = &self.telemetry {
251            return telemetry
252                .run_observed(
253                    "durable.workflow.task",
254                    self.workflow_telemetry_fields(&task),
255                    false,
256                    self.workflow_observations(&task),
257                    async {
258                        let mut decision = self
259                            .execute_workflow_with_local_activities_inner(task)
260                            .await?;
261                        if let Some(decision) = &mut decision {
262                            self.stamp_observability_commands(&mut decision.commands);
263                        }
264                        Ok(decision)
265                    },
266                )
267                .await;
268        }
269        self.execute_workflow_with_local_activities_inner(task)
270            .await
271    }
272
273    async fn execute_workflow_with_local_activities_inner(
274        &self,
275        task: WorkflowTask,
276    ) -> Result<Option<WorkflowTaskDecision>> {
277        let (task, context, mut future) = match self.prepare_workflow_task_execution(task, None)? {
278            PreparedWorkflowTask::Decision(decision) => return Ok(Some(decision)),
279            PreparedWorkflowTask::Execution {
280                task,
281                context,
282                future,
283            } => (task, context, future),
284        };
285        loop {
286            let outcome = Self::poll_workflow_future(&context, &mut future)?;
287            let requests = {
288                let mut state = context
289                    .state
290                    .lock()
291                    .map_err(|_| Error::WorkflowStatePoisoned)?;
292                std::mem::take(&mut state.local_activity_requests)
293            };
294            if requests.is_empty() {
295                return self
296                    .finish_workflow_task_execution(&task, &context, outcome)
297                    .map(Some);
298            }
299            if !self.client.local_activities_enabled {
300                return Err(Error::WorkerLoop("local_activities_not_enabled: opt in with Worker::local_activities(true) before executing inline local work".into()));
301            }
302            // A custom future cannot publish a terminal decision while ignoring
303            // an unresolved local call. Resolve only genuinely suspended work.
304            if !outcome.is_pending() {
305                return Err(Error::WorkflowYieldedWithoutCommand);
306            }
307            for request in requests {
308                let Some((wire, result)) = self
309                    .execute_local_activity(&task, &context, &request)
310                    .await?
311                else {
312                    // Do not fail, complete, or continue the workflow after a
313                    // refused or uncertain lease. Reclaim owns subsequent work.
314                    return Ok(None);
315                };
316                let mut state = context
317                    .state
318                    .lock()
319                    .map_err(|_| Error::WorkflowStatePoisoned)?;
320                state.commands[request.command_index] = wire;
321                *request
322                    .result
323                    .lock()
324                    .map_err(|_| Error::WorkflowStatePoisoned)? = Some(result);
325            }
326        }
327    }
328
329    async fn local_claim_is_active(&self, task: &WorkflowTask) -> bool {
330        tokio::select! {
331            biased;
332            _ = self.local_worker_stopped() => false,
333            receipt = self.client.heartbeat_workflow_task_with_protocol(task, None, MINIMUM_WORKER_PROTOCOL_VERSION) => {
334                receipt.is_ok_and(|receipt| receipt.cancellation_request.is_none())
335            }
336        }
337    }
338
339    async fn local_worker_stopped(&self) {
340        match &self.client.worker_storage_admission {
341            Some(admission) => wait_for_worker_stop(&admission.stop).await,
342            None => std::future::pending::<()>().await,
343        }
344    }
345
346    async fn externalize_local_command(
347        &self,
348        task: &WorkflowTask,
349        context: &WorkflowContext,
350        index: usize,
351        wire: &mut Value,
352        messages: &[(usize, crate::failure_reporting::LocalText)],
353    ) -> Result<()> {
354        let mut commands = context
355            .state
356            .lock()
357            .map_err(|_| Error::WorkflowStatePoisoned)?
358            .commands
359            .clone();
360        commands[index] = wire.clone();
361        let mut body = json!({"lease_owner": task.lease_owner, "workflow_task_attempt": task.workflow_task_attempt,
362            "commands": commands});
363        let path = format!("/worker/workflow-tasks/{}/complete", task.task_id);
364        let limit = self
365            .client
366            .known_failure_request_limit(RequestProtocol::DefaultWorker)?;
367        crate::failure_reporting::fit_local(&mut body, index, messages, limit)?;
368        self.client
369            .externalize_runtime_payloads(&mut body, &path, RequestProtocol::DefaultWorker)
370            .await?;
371        let limit = self
372            .client
373            .known_failure_request_limit(RequestProtocol::DefaultWorker)?;
374        crate::failure_reporting::fit_local(&mut body, index, messages, limit)?;
375        *wire = body["commands"][index].take();
376        Ok(())
377    }
378
379    async fn admit_local_command(
380        &self,
381        task: &WorkflowTask,
382        context: &WorkflowContext,
383        index: usize,
384        wire: &mut Value,
385        messages: &[(usize, crate::failure_reporting::LocalText)],
386    ) -> Result<bool> {
387        match self
388            .externalize_local_command(task, context, index, wire, messages)
389            .await
390        {
391            Ok(()) => Ok(true),
392            Err(error @ Error::FailureReporting(_)) => Err(error),
393            Err(_) => Ok(false),
394        }
395    }
396
397    async fn execute_local_activity(
398        &self,
399        task: &WorkflowTask,
400        context: &WorkflowContext,
401        request: &Request,
402    ) -> Result<Option<(Value, ActivityOutcome)>> {
403        let mut wire = context
404            .state
405            .lock()
406            .map_err(|_| Error::WorkflowStatePoisoned)?
407            .commands[request.command_index]
408            .clone();
409        // Serialize and admit input before the callback can produce side effects.
410        if !self
411            .admit_local_command(task, context, request.command_index, &mut wire, &[])
412            .await?
413        {
414            return Ok(None);
415        }
416        let activity_type = wire["activity_type"]
417            .as_str()
418            .unwrap_or_default()
419            .to_owned();
420        let policy = request.options.retry_policy.as_ref();
421        let max_attempts = policy
422            .and_then(|policy| policy["max_attempts"].as_u64())
423            .unwrap_or(1);
424        let started = Instant::now();
425        let total_deadline = request
426            .options
427            .schedule_to_close_timeout
428            .map(|seconds| started + Duration::from_secs(seconds));
429        let mut attempts = Vec::new();
430        let mut messages = Vec::new();
431        let mut heartbeat_count = 0;
432        for number in 1..=max_attempts {
433            if !self.local_claim_is_active(task).await {
434                return Ok(None);
435            }
436            let attempt_started = Instant::now();
437            let reports = Arc::new(Mutex::new(Heartbeats {
438                active: true,
439                started: attempt_started,
440                last: attempt_started,
441                reports: Vec::new(),
442                capacity: MAX_HEARTBEATS - heartbeat_count,
443                overflowed: false,
444            }));
445            let close = CloseAttempt(reports.clone());
446            let attempt_id = format!(
447                "{:x}",
448                Sha256::digest(
449                    format!(
450                        "{}\0{}\0{}\0{}",
451                        task.task_id, task.workflow_task_attempt, request.command_index, number
452                    )
453                    .as_bytes()
454                )
455            );
456            let ctx = ActivityContext {
457                client: self.client.clone(),
458                task_id: task.task_id.clone(),
459                activity_attempt_id: attempt_id.clone(),
460                lease_owner: task.lease_owner.clone().unwrap_or_default(),
461                activity_type: activity_type.clone(),
462                attempt_number: number,
463                retry_info: crate::ActivityRetryInfo::new(number, None),
464                activity_execution_id: None,
465                workflow_id: task.workflow_id.clone(),
466                run_id: task.run_id.clone(),
467                task_queue: self.task_queue.clone(),
468                worker_id: self.worker_id.clone(),
469                claim_guard: None,
470                local_heartbeats: Some(reports.clone()),
471                worker_session: None,
472            };
473            let callback = self
474                .activities
475                .get(&activity_type)
476                .filter(|_| !total_deadline.is_some_and(|deadline| Instant::now() >= deadline));
477            let mut future: ActivityFuture = match callback {
478                Some(callback) => {
479                    self.activity_callback(callback, ctx, request.arguments.clone(), true)
480                }
481                None if total_deadline.is_some_and(|deadline| Instant::now() >= deadline) => {
482                    Box::pin(std::future::pending())
483                }
484                None => {
485                    let unknown = activity_type.clone();
486                    Box::pin(async move { Err(Error::ActivityNotRegistered(unknown)) })
487                }
488            };
489            let attempt_deadline = request
490                .options
491                .start_to_close_timeout
492                .map(|seconds| attempt_started + Duration::from_secs(seconds));
493            let outcome = loop {
494                let heartbeat_deadline = request
495                    .options
496                    .heartbeat_timeout
497                    .map(|seconds| {
498                        reports
499                            .lock()
500                            .map(|reports| reports.last + Duration::from_secs(seconds))
501                    })
502                    .transpose()
503                    .map_err(|_| Error::WorkflowStatePoisoned)?;
504                let next_timeout = [
505                    (total_deadline, "schedule_to_close"),
506                    (attempt_deadline, "start_to_close"),
507                    (heartbeat_deadline, "heartbeat"),
508                ]
509                .into_iter()
510                .filter_map(|(deadline, kind)| deadline.map(|d| (d, kind)))
511                .min_by_key(|(deadline, _)| *deadline);
512                let timeout_kind = next_timeout
513                    .map(|(_, kind)| kind)
514                    .unwrap_or("start_to_close");
515                let timeout = async {
516                    match next_timeout {
517                        Some((deadline, _)) => tokio::time::sleep_until(deadline.into()).await,
518                        None => std::future::pending::<()>().await,
519                    }
520                };
521                tokio::pin!(timeout);
522                tokio::select! {
523                    biased;
524                    _ = self.local_worker_stopped() => return Ok(None),
525                    _ = &mut timeout => {
526                        if timeout_kind == "heartbeat" && heartbeat_extended(&reports, request.options.heartbeat_timeout)? { continue; }
527                        break Err((format!("local activity exceeded {timeout_kind} timeout"), "LocalActivityTimeout".to_owned(), false, Some(timeout_kind)));
528                    },
529                    result = &mut future => break result.map_err(local_error),
530                    _ = tokio::time::sleep(LEASE_RENEWAL_INTERVAL) => {
531                        tokio::select! {
532                            biased;
533                            _ = &mut timeout => {
534                                if timeout_kind == "heartbeat" && heartbeat_extended(&reports, request.options.heartbeat_timeout)? { continue; }
535                                break Err((format!("local activity exceeded {timeout_kind} timeout"), "LocalActivityTimeout".to_owned(), false, Some(timeout_kind)));
536                            },
537                            active = self.local_claim_is_active(task) => { if !active { return Ok(None); } }
538                        }
539                    }
540                }
541            };
542            drop(future);
543            drop(close);
544            let (heartbeats, overflowed) = {
545                let mut reports = reports.lock().map_err(|_| Error::WorkflowStatePoisoned)?;
546                (std::mem::take(&mut reports.reports), reports.overflowed)
547            };
548            heartbeat_count += heartbeats.len();
549            let outcome = if overflowed {
550                Err((
551                    "local activity exceeded 1000 heartbeat reports".into(),
552                    "LocalActivityHeartbeatLimit".into(),
553                    true,
554                    None,
555                ))
556            } else {
557                outcome
558            };
559            let mut report = json!({"attempt_number": number, "attempt_id": attempt_id,
560                "duration_ms": millis(attempt_started.elapsed()), "heartbeats": heartbeats});
561            let (result, failure) = match outcome {
562                Ok(result) => match encode_typed_envelope(&result, &task.payload_codec) {
563                    Ok(envelope) => {
564                        wire["result"] = envelope;
565                        (Some(result), None)
566                    }
567                    Err(error) => (
568                        None,
569                        Some((
570                            error.to_string(),
571                            "LocalActivityResultCodecError".into(),
572                            true,
573                            None,
574                        )),
575                    ),
576                },
577                Err(failure) => (None, Some(failure)),
578            };
579            if let Some((message, exception_type, mut non_retryable, timeout_kind)) = failure {
580                non_retryable |= policy
581                    .and_then(|policy| policy["non_retryable_error_types"].as_array())
582                    .is_some_and(|types| {
583                        types
584                            .iter()
585                            .any(|t| t.as_str() == Some(exception_type.as_str()))
586                    });
587                let status = if timeout_kind.is_some() {
588                    "timed_out"
589                } else {
590                    "failed"
591                };
592                report["outcome"] = json!(status);
593                let original = crate::failure_reporting::LocalText::new(&message);
594                report["message"] = json!(original.render(crate::failure_reporting::MESSAGE_BYTES));
595                messages.push((attempts.len(), original));
596                report["exception_type"] = json!(exception_type);
597                report["non_retryable"] = json!(non_retryable);
598                if let Some(kind) = timeout_kind {
599                    report["timeout_kind"] = json!(kind);
600                }
601                let retry = number < max_attempts
602                    && !non_retryable
603                    && timeout_kind != Some("schedule_to_close");
604                if retry {
605                    let backoff = local_retry_backoff_seconds(policy, number);
606                    report["retry_reason"] = json!(if timeout_kind.is_some() {
607                        "timeout"
608                    } else {
609                        "failure"
610                    });
611                    report["backoff_seconds"] = json!(backoff);
612                    attempts.push(report);
613                    let wake = Instant::now() + Duration::from_secs(backoff);
614                    while Instant::now() < wake {
615                        // An elapsed total deadline is reported as the next
616                        // terminal attempt, with no further callback invocation.
617                        if total_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
618                            break;
619                        }
620                        if !self.local_claim_is_active(task).await {
621                            return Ok(None);
622                        }
623                        let deadline = total_deadline.map_or(wake, |d| d.min(wake));
624                        tokio::time::sleep(
625                            deadline
626                                .saturating_duration_since(Instant::now())
627                                .min(LEASE_RENEWAL_INTERVAL),
628                        )
629                        .await;
630                    }
631                    continue;
632                }
633                attempts.push(report);
634                for field in [
635                    "outcome",
636                    "message",
637                    "exception_type",
638                    "non_retryable",
639                    "timeout_kind",
640                ] {
641                    if let Some(value) = attempts.last().unwrap().get(field) {
642                        wire[field] = value.clone();
643                    }
644                }
645                wire["attempts"] = json!(attempts);
646                if !self
647                    .admit_local_command(task, context, request.command_index, &mut wire, &messages)
648                    .await?
649                    || !self.local_claim_is_active(task).await
650                {
651                    return Ok(None);
652                }
653                // Match the terminal fields Server persists. Database-generated
654                // execution/failure IDs are only available on committed replay.
655                let mut payload = wire.clone();
656                if let Some(message) = payload["message"].as_str() {
657                    payload["message"] =
658                        json!(crate::failure_reporting::committed_message(message));
659                }
660                payload["attempt_number"] = json!(number);
661                payload["failure_category"] = json!(if timeout_kind.is_some() {
662                    "timeout"
663                } else {
664                    "application"
665                });
666                if timeout_kind.is_some() {
667                    payload["exception_class"] = payload["exception_type"].take();
668                    payload.as_object_mut().unwrap().remove("exception_type");
669                    payload.as_object_mut().unwrap().remove("non_retryable");
670                }
671                let event = HistoryEvent {
672                    event_type: if timeout_kind.is_some() {
673                        "ActivityTimedOut"
674                    } else {
675                        "ActivityFailed"
676                    }
677                    .into(),
678                    payload,
679                    raw: HashMap::new(),
680                };
681                let result =
682                    activity_outcome(&event, &task.payload_codec, Some(activity_type.clone()))?;
683                return Ok(Some((wire, result)));
684            }
685            report["outcome"] = json!("completed");
686            attempts.push(report);
687            wire["outcome"] = json!("completed");
688            wire["attempts"] = json!(attempts);
689            if !self
690                .admit_local_command(task, context, request.command_index, &mut wire, &messages)
691                .await?
692                || !self.local_claim_is_active(task).await
693            {
694                return Ok(None);
695            }
696            return Ok(Some((wire, Ok(result.unwrap()))));
697        }
698        unreachable!("validated local retry budget includes at least one attempt")
699    }
700}
701
702fn local_error(error: Error) -> (String, String, bool, Option<&'static str>) {
703    let failure = crate::ActivityError::from_error(&error, "RustActivityError");
704    (
705        failure.message,
706        failure.exception_type,
707        failure.non_retryable,
708        None,
709    )
710}
711
712fn heartbeat_extended(reports: &Arc<Mutex<Heartbeats>>, timeout: Option<u64>) -> Result<bool> {
713    let reports = reports.lock().map_err(|_| Error::WorkflowStatePoisoned)?;
714    Ok(timeout.is_some_and(|seconds| reports.last + Duration::from_secs(seconds) > Instant::now()))
715}
716
717/// The delay before retrying after attempt `number` (1-based). Once attempts
718/// outnumber the backoff list, the final entry repeats, as it does for
719/// Server-scheduled activities (durable-workflow/.github#219). No list means
720/// no delay.
721fn local_retry_backoff_seconds(policy: Option<&Value>, number: u64) -> u64 {
722    policy
723        .and_then(|p| p["backoff_seconds"].as_array())
724        .and_then(|list| {
725            let index = usize::try_from(number)
726                .ok()?
727                .min(list.len())
728                .checked_sub(1)?;
729            list.get(index)
730        })
731        .and_then(Value::as_u64)
732        .unwrap_or(0)
733}
734
735#[cfg(test)]
736mod local_retry_backoff_tests {
737    use super::local_retry_backoff_seconds;
738    use serde_json::json;
739
740    #[test]
741    fn repeats_the_final_entry_once_the_list_runs_out() {
742        let policy = json!({"max_attempts": 5, "backoff_seconds": [1, 2]});
743        let delays: Vec<u64> = (1..=4)
744            .map(|number| local_retry_backoff_seconds(Some(&policy), number))
745            .collect();
746        assert_eq!(delays, vec![1, 2, 2, 2]);
747    }
748
749    #[test]
750    fn no_list_or_no_policy_means_no_delay() {
751        let empty = json!({"max_attempts": 3, "backoff_seconds": []});
752        let missing = json!({"max_attempts": 3});
753        assert_eq!(local_retry_backoff_seconds(Some(&empty), 2), 0);
754        assert_eq!(local_retry_backoff_seconds(Some(&missing), 2), 0);
755        assert_eq!(local_retry_backoff_seconds(None, 1), 0);
756    }
757}