1use super::*;
2
3const MAX_ATTEMPTS: u64 = 100;
4const MAX_HEARTBEATS: usize = 1000;
5const LEASE_RENEWAL_INTERVAL: Duration = Duration::from_secs(1);
6
7#[derive(Clone, Debug, Default, PartialEq, Eq)]
12pub struct LocalActivityOptions {
13 pub retry_policy: Option<ActivityRetryPolicy>,
14 pub start_to_close_timeout: Option<Duration>,
15 pub schedule_to_close_timeout: Option<Duration>,
16 pub heartbeat_timeout: Option<Duration>,
17}
18
19impl LocalActivityOptions {
20 pub fn new() -> Self {
21 Self::default()
22 }
23 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
24 self.retry_policy = Some(policy);
25 self
26 }
27 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
28 self.start_to_close_timeout = Some(timeout);
29 self
30 }
31 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
32 self.schedule_to_close_timeout = Some(timeout);
33 self
34 }
35 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
36 self.heartbeat_timeout = Some(timeout);
37 self
38 }
39 fn into_activity_options(self) -> ActivityOptions {
40 ActivityOptions {
41 retry_policy: self.retry_policy,
42 start_to_close_timeout: self.start_to_close_timeout,
43 schedule_to_close_timeout: self.schedule_to_close_timeout,
44 heartbeat_timeout: self.heartbeat_timeout,
45 ..ActivityOptions::default()
46 }
47 }
48}
49
50pub(super) fn validate(options: &ValidatedActivityOptions) -> Result<()> {
51 let max_attempts = options
52 .retry_policy
53 .as_ref()
54 .and_then(|policy| policy["max_attempts"].as_u64())
55 .unwrap_or(1);
56 if max_attempts > MAX_ATTEMPTS {
57 return Err(Error::InvalidActivityOptions(ActivityOptionsError::new(
58 ActivityOptionsErrorKind::InvalidMaxAttempts,
59 Some("max_attempts"),
60 "local activities support at most 100 attempts",
61 )));
62 }
63 for seconds in [
64 options.start_to_close_timeout,
65 options.schedule_to_close_timeout,
66 options.heartbeat_timeout,
67 ]
68 .into_iter()
69 .flatten()
70 .chain(options.retry_policy.iter().flat_map(|p| {
71 p["backoff_seconds"]
72 .as_array()
73 .into_iter()
74 .flatten()
75 .filter_map(Value::as_u64)
76 })) {
77 if Instant::now()
78 .checked_add(Duration::from_secs(seconds))
79 .is_none()
80 {
81 return Err(Error::InvalidActivityOptions(ActivityOptionsError::new(
82 ActivityOptionsErrorKind::TimeoutOverflow,
83 Some("local_activity"),
84 "local timing exceeds the monotonic clock range",
85 )));
86 }
87 }
88 Ok(())
89}
90
91impl WorkflowContext {
92 pub fn local_activity<T: Serialize>(
98 &self,
99 activity_type: impl Into<String>,
100 args: T,
101 ) -> ActivityCall {
102 self.local_activity_with_options(activity_type, LocalActivityOptions::new(), args)
103 }
104
105 pub fn local_activity_with_options<T: Serialize>(
106 &self,
107 activity_type: impl Into<String>,
108 options: LocalActivityOptions,
109 args: T,
110 ) -> ActivityCall {
111 let mut call =
112 self.activity_with_options(activity_type, options.into_activity_options(), args);
113 call.local = true;
114 call
115 }
116
117 pub async fn local_activity_avro_value<T: Serialize>(
119 &self,
120 activity_type: impl Into<String>,
121 args: T,
122 ) -> Result<AvroValue> {
123 self.local_activity_avro_value_with_options(
124 activity_type,
125 LocalActivityOptions::new(),
126 args,
127 )
128 .await
129 }
130
131 pub async fn local_activity_avro_value_with_options<T: Serialize>(
132 &self,
133 activity_type: impl Into<String>,
134 options: LocalActivityOptions,
135 args: T,
136 ) -> Result<AvroValue> {
137 let mut call = self.local_activity_with_options(activity_type, options, args);
138 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
139 }
140
141 pub async fn local_activity_typed<I: Serialize, O: DeserializeOwned>(
142 &self,
143 activity_type: impl Into<String>,
144 args: I,
145 ) -> Result<O> {
146 self.local_activity_typed_with_options(activity_type, LocalActivityOptions::new(), args)
147 .await
148 }
149
150 pub async fn local_activity_typed_with_options<I: Serialize, O: DeserializeOwned>(
151 &self,
152 activity_type: impl Into<String>,
153 options: LocalActivityOptions,
154 args: I,
155 ) -> Result<O> {
156 let activity_type = activity_type.into();
157 let result = self
158 .local_activity_avro_value_with_options(activity_type.clone(), options, args)
159 .await?;
160 decode_handler_result(result, HandlerKind::Activity, &activity_type)
161 }
162}
163
164#[derive(Debug)]
165pub(super) struct Request {
166 pub command_index: usize,
167 pub options: ValidatedActivityOptions,
168 pub arguments: AvroValue,
169 pub result: Arc<Mutex<Option<ActivityOutcome>>>,
170}
171
172#[derive(Debug)]
173pub(super) struct Heartbeats {
174 active: bool,
175 started: Instant,
176 last: Instant,
177 reports: Vec<Value>,
178 capacity: usize,
179 overflowed: bool,
180}
181
182impl Heartbeats {
183 pub(super) fn captured_attempt() -> (Arc<Mutex<Self>>, impl Drop) {
184 let now = Instant::now();
185 let reports = Arc::new(Mutex::new(Self {
186 active: true,
187 started: now,
188 last: now,
189 reports: Vec::new(),
190 capacity: MAX_HEARTBEATS,
191 overflowed: false,
192 }));
193 let close = CloseAttempt(reports.clone());
194 (reports, close)
195 }
196
197 pub fn record<T: Serialize>(&mut self, details: T) -> Result<ActivityHeartbeatResponse> {
198 if !self.active {
199 return Ok(heartbeat_response(false));
200 }
201 if self.reports.len() >= self.capacity {
202 self.overflowed = true;
203 return Err(Error::WorkerLoop(
204 "local_activity_heartbeat_limit_exceeded: at most 1000 reports per local command"
205 .into(),
206 ));
207 }
208 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
209 let now = Instant::now();
210 self.reports.push(
211 json!({"elapsed_ms": millis(now.duration_since(self.started)), "details": details}),
212 );
213 self.last = now;
214 Ok(heartbeat_response(true))
215 }
216}
217
218fn heartbeat_response(active: bool) -> ActivityHeartbeatResponse {
219 ActivityHeartbeatResponse {
220 cancel_requested: false,
221 heartbeat_recorded: active,
222 can_continue: Some(active),
223 reason: (!active).then(|| "local_activity_attempt_closed".into()),
224 run_closed_reason: None,
225 run_closed_at: None,
226 lease_expires_at: None,
227 last_heartbeat_at: None,
228 }
229}
230
231struct CloseAttempt(Arc<Mutex<Heartbeats>>);
232impl Drop for CloseAttempt {
233 fn drop(&mut self) {
234 if let Ok(mut heartbeats) = self.0.lock() {
235 heartbeats.active = false;
236 }
237 }
238}
239
240fn millis(duration: Duration) -> u64 {
241 u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
242}
243
244impl Worker {
245 pub(super) async fn execute_workflow_with_local_activities(
246 &self,
247 task: WorkflowTask,
248 ) -> Result<Option<WorkflowTaskDecision>> {
249 #[cfg(feature = "telemetry")]
250 if let Some(telemetry) = &self.telemetry {
251 return telemetry
252 .run_observed(
253 "durable.workflow.task",
254 self.workflow_telemetry_fields(&task),
255 false,
256 self.workflow_observations(&task),
257 async {
258 let mut decision = self
259 .execute_workflow_with_local_activities_inner(task)
260 .await?;
261 if let Some(decision) = &mut decision {
262 self.stamp_observability_commands(&mut decision.commands);
263 }
264 Ok(decision)
265 },
266 )
267 .await;
268 }
269 self.execute_workflow_with_local_activities_inner(task)
270 .await
271 }
272
273 async fn execute_workflow_with_local_activities_inner(
274 &self,
275 task: WorkflowTask,
276 ) -> Result<Option<WorkflowTaskDecision>> {
277 let (task, context, mut future) = match self.prepare_workflow_task_execution(task, None)? {
278 PreparedWorkflowTask::Decision(decision) => return Ok(Some(decision)),
279 PreparedWorkflowTask::Execution {
280 task,
281 context,
282 future,
283 } => (task, context, future),
284 };
285 loop {
286 let outcome = Self::poll_workflow_future(&context, &mut future)?;
287 let requests = {
288 let mut state = context
289 .state
290 .lock()
291 .map_err(|_| Error::WorkflowStatePoisoned)?;
292 std::mem::take(&mut state.local_activity_requests)
293 };
294 if requests.is_empty() {
295 return self
296 .finish_workflow_task_execution(&task, &context, outcome)
297 .map(Some);
298 }
299 if !self.client.local_activities_enabled {
300 return Err(Error::WorkerLoop("local_activities_not_enabled: opt in with Worker::local_activities(true) before executing inline local work".into()));
301 }
302 if !outcome.is_pending() {
305 return Err(Error::WorkflowYieldedWithoutCommand);
306 }
307 for request in requests {
308 let Some((wire, result)) = self
309 .execute_local_activity(&task, &context, &request)
310 .await?
311 else {
312 return Ok(None);
315 };
316 let mut state = context
317 .state
318 .lock()
319 .map_err(|_| Error::WorkflowStatePoisoned)?;
320 state.commands[request.command_index] = wire;
321 *request
322 .result
323 .lock()
324 .map_err(|_| Error::WorkflowStatePoisoned)? = Some(result);
325 }
326 }
327 }
328
329 async fn local_claim_is_active(&self, task: &WorkflowTask) -> bool {
330 tokio::select! {
331 biased;
332 _ = self.local_worker_stopped() => false,
333 receipt = self.client.heartbeat_workflow_task_with_protocol(task, None, MINIMUM_WORKER_PROTOCOL_VERSION) => {
334 receipt.is_ok_and(|receipt| receipt.cancellation_request.is_none())
335 }
336 }
337 }
338
339 async fn local_worker_stopped(&self) {
340 match &self.client.worker_storage_admission {
341 Some(admission) => wait_for_worker_stop(&admission.stop).await,
342 None => std::future::pending::<()>().await,
343 }
344 }
345
346 async fn externalize_local_command(
347 &self,
348 task: &WorkflowTask,
349 context: &WorkflowContext,
350 index: usize,
351 wire: &mut Value,
352 messages: &[(usize, crate::failure_reporting::LocalText)],
353 ) -> Result<()> {
354 let mut commands = context
355 .state
356 .lock()
357 .map_err(|_| Error::WorkflowStatePoisoned)?
358 .commands
359 .clone();
360 commands[index] = wire.clone();
361 let mut body = json!({"lease_owner": task.lease_owner, "workflow_task_attempt": task.workflow_task_attempt,
362 "commands": commands});
363 let path = format!("/worker/workflow-tasks/{}/complete", task.task_id);
364 let limit = self
365 .client
366 .known_failure_request_limit(RequestProtocol::DefaultWorker)?;
367 crate::failure_reporting::fit_local(&mut body, index, messages, limit)?;
368 self.client
369 .externalize_runtime_payloads(&mut body, &path, RequestProtocol::DefaultWorker)
370 .await?;
371 let limit = self
372 .client
373 .known_failure_request_limit(RequestProtocol::DefaultWorker)?;
374 crate::failure_reporting::fit_local(&mut body, index, messages, limit)?;
375 *wire = body["commands"][index].take();
376 Ok(())
377 }
378
379 async fn admit_local_command(
380 &self,
381 task: &WorkflowTask,
382 context: &WorkflowContext,
383 index: usize,
384 wire: &mut Value,
385 messages: &[(usize, crate::failure_reporting::LocalText)],
386 ) -> Result<bool> {
387 match self
388 .externalize_local_command(task, context, index, wire, messages)
389 .await
390 {
391 Ok(()) => Ok(true),
392 Err(error @ Error::FailureReporting(_)) => Err(error),
393 Err(_) => Ok(false),
394 }
395 }
396
397 async fn execute_local_activity(
398 &self,
399 task: &WorkflowTask,
400 context: &WorkflowContext,
401 request: &Request,
402 ) -> Result<Option<(Value, ActivityOutcome)>> {
403 let mut wire = context
404 .state
405 .lock()
406 .map_err(|_| Error::WorkflowStatePoisoned)?
407 .commands[request.command_index]
408 .clone();
409 if !self
411 .admit_local_command(task, context, request.command_index, &mut wire, &[])
412 .await?
413 {
414 return Ok(None);
415 }
416 let activity_type = wire["activity_type"]
417 .as_str()
418 .unwrap_or_default()
419 .to_owned();
420 let policy = request.options.retry_policy.as_ref();
421 let max_attempts = policy
422 .and_then(|policy| policy["max_attempts"].as_u64())
423 .unwrap_or(1);
424 let started = Instant::now();
425 let total_deadline = request
426 .options
427 .schedule_to_close_timeout
428 .map(|seconds| started + Duration::from_secs(seconds));
429 let mut attempts = Vec::new();
430 let mut messages = Vec::new();
431 let mut heartbeat_count = 0;
432 for number in 1..=max_attempts {
433 if !self.local_claim_is_active(task).await {
434 return Ok(None);
435 }
436 let attempt_started = Instant::now();
437 let reports = Arc::new(Mutex::new(Heartbeats {
438 active: true,
439 started: attempt_started,
440 last: attempt_started,
441 reports: Vec::new(),
442 capacity: MAX_HEARTBEATS - heartbeat_count,
443 overflowed: false,
444 }));
445 let close = CloseAttempt(reports.clone());
446 let attempt_id = format!(
447 "{:x}",
448 Sha256::digest(
449 format!(
450 "{}\0{}\0{}\0{}",
451 task.task_id, task.workflow_task_attempt, request.command_index, number
452 )
453 .as_bytes()
454 )
455 );
456 let ctx = ActivityContext {
457 client: self.client.clone(),
458 task_id: task.task_id.clone(),
459 activity_attempt_id: attempt_id.clone(),
460 lease_owner: task.lease_owner.clone().unwrap_or_default(),
461 activity_type: activity_type.clone(),
462 attempt_number: number,
463 retry_info: crate::ActivityRetryInfo::new(number, None),
464 activity_execution_id: None,
465 workflow_id: task.workflow_id.clone(),
466 run_id: task.run_id.clone(),
467 task_queue: self.task_queue.clone(),
468 worker_id: self.worker_id.clone(),
469 claim_guard: None,
470 local_heartbeats: Some(reports.clone()),
471 worker_session: None,
472 };
473 let callback = self
474 .activities
475 .get(&activity_type)
476 .filter(|_| !total_deadline.is_some_and(|deadline| Instant::now() >= deadline));
477 let mut future: ActivityFuture = match callback {
478 Some(callback) => {
479 self.activity_callback(callback, ctx, request.arguments.clone(), true)
480 }
481 None if total_deadline.is_some_and(|deadline| Instant::now() >= deadline) => {
482 Box::pin(std::future::pending())
483 }
484 None => {
485 let unknown = activity_type.clone();
486 Box::pin(async move { Err(Error::ActivityNotRegistered(unknown)) })
487 }
488 };
489 let attempt_deadline = request
490 .options
491 .start_to_close_timeout
492 .map(|seconds| attempt_started + Duration::from_secs(seconds));
493 let outcome = loop {
494 let heartbeat_deadline = request
495 .options
496 .heartbeat_timeout
497 .map(|seconds| {
498 reports
499 .lock()
500 .map(|reports| reports.last + Duration::from_secs(seconds))
501 })
502 .transpose()
503 .map_err(|_| Error::WorkflowStatePoisoned)?;
504 let next_timeout = [
505 (total_deadline, "schedule_to_close"),
506 (attempt_deadline, "start_to_close"),
507 (heartbeat_deadline, "heartbeat"),
508 ]
509 .into_iter()
510 .filter_map(|(deadline, kind)| deadline.map(|d| (d, kind)))
511 .min_by_key(|(deadline, _)| *deadline);
512 let timeout_kind = next_timeout
513 .map(|(_, kind)| kind)
514 .unwrap_or("start_to_close");
515 let timeout = async {
516 match next_timeout {
517 Some((deadline, _)) => tokio::time::sleep_until(deadline.into()).await,
518 None => std::future::pending::<()>().await,
519 }
520 };
521 tokio::pin!(timeout);
522 tokio::select! {
523 biased;
524 _ = self.local_worker_stopped() => return Ok(None),
525 _ = &mut timeout => {
526 if timeout_kind == "heartbeat" && heartbeat_extended(&reports, request.options.heartbeat_timeout)? { continue; }
527 break Err((format!("local activity exceeded {timeout_kind} timeout"), "LocalActivityTimeout".to_owned(), false, Some(timeout_kind)));
528 },
529 result = &mut future => break result.map_err(local_error),
530 _ = tokio::time::sleep(LEASE_RENEWAL_INTERVAL) => {
531 tokio::select! {
532 biased;
533 _ = &mut timeout => {
534 if timeout_kind == "heartbeat" && heartbeat_extended(&reports, request.options.heartbeat_timeout)? { continue; }
535 break Err((format!("local activity exceeded {timeout_kind} timeout"), "LocalActivityTimeout".to_owned(), false, Some(timeout_kind)));
536 },
537 active = self.local_claim_is_active(task) => { if !active { return Ok(None); } }
538 }
539 }
540 }
541 };
542 drop(future);
543 drop(close);
544 let (heartbeats, overflowed) = {
545 let mut reports = reports.lock().map_err(|_| Error::WorkflowStatePoisoned)?;
546 (std::mem::take(&mut reports.reports), reports.overflowed)
547 };
548 heartbeat_count += heartbeats.len();
549 let outcome = if overflowed {
550 Err((
551 "local activity exceeded 1000 heartbeat reports".into(),
552 "LocalActivityHeartbeatLimit".into(),
553 true,
554 None,
555 ))
556 } else {
557 outcome
558 };
559 let mut report = json!({"attempt_number": number, "attempt_id": attempt_id,
560 "duration_ms": millis(attempt_started.elapsed()), "heartbeats": heartbeats});
561 let (result, failure) = match outcome {
562 Ok(result) => match encode_typed_envelope(&result, &task.payload_codec) {
563 Ok(envelope) => {
564 wire["result"] = envelope;
565 (Some(result), None)
566 }
567 Err(error) => (
568 None,
569 Some((
570 error.to_string(),
571 "LocalActivityResultCodecError".into(),
572 true,
573 None,
574 )),
575 ),
576 },
577 Err(failure) => (None, Some(failure)),
578 };
579 if let Some((message, exception_type, mut non_retryable, timeout_kind)) = failure {
580 non_retryable |= policy
581 .and_then(|policy| policy["non_retryable_error_types"].as_array())
582 .is_some_and(|types| {
583 types
584 .iter()
585 .any(|t| t.as_str() == Some(exception_type.as_str()))
586 });
587 let status = if timeout_kind.is_some() {
588 "timed_out"
589 } else {
590 "failed"
591 };
592 report["outcome"] = json!(status);
593 let original = crate::failure_reporting::LocalText::new(&message);
594 report["message"] = json!(original.render(crate::failure_reporting::MESSAGE_BYTES));
595 messages.push((attempts.len(), original));
596 report["exception_type"] = json!(exception_type);
597 report["non_retryable"] = json!(non_retryable);
598 if let Some(kind) = timeout_kind {
599 report["timeout_kind"] = json!(kind);
600 }
601 let retry = number < max_attempts
602 && !non_retryable
603 && timeout_kind != Some("schedule_to_close");
604 if retry {
605 let backoff = local_retry_backoff_seconds(policy, number);
606 report["retry_reason"] = json!(if timeout_kind.is_some() {
607 "timeout"
608 } else {
609 "failure"
610 });
611 report["backoff_seconds"] = json!(backoff);
612 attempts.push(report);
613 let wake = Instant::now() + Duration::from_secs(backoff);
614 while Instant::now() < wake {
615 if total_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
618 break;
619 }
620 if !self.local_claim_is_active(task).await {
621 return Ok(None);
622 }
623 let deadline = total_deadline.map_or(wake, |d| d.min(wake));
624 tokio::time::sleep(
625 deadline
626 .saturating_duration_since(Instant::now())
627 .min(LEASE_RENEWAL_INTERVAL),
628 )
629 .await;
630 }
631 continue;
632 }
633 attempts.push(report);
634 for field in [
635 "outcome",
636 "message",
637 "exception_type",
638 "non_retryable",
639 "timeout_kind",
640 ] {
641 if let Some(value) = attempts.last().unwrap().get(field) {
642 wire[field] = value.clone();
643 }
644 }
645 wire["attempts"] = json!(attempts);
646 if !self
647 .admit_local_command(task, context, request.command_index, &mut wire, &messages)
648 .await?
649 || !self.local_claim_is_active(task).await
650 {
651 return Ok(None);
652 }
653 let mut payload = wire.clone();
656 if let Some(message) = payload["message"].as_str() {
657 payload["message"] =
658 json!(crate::failure_reporting::committed_message(message));
659 }
660 payload["attempt_number"] = json!(number);
661 payload["failure_category"] = json!(if timeout_kind.is_some() {
662 "timeout"
663 } else {
664 "application"
665 });
666 if timeout_kind.is_some() {
667 payload["exception_class"] = payload["exception_type"].take();
668 payload.as_object_mut().unwrap().remove("exception_type");
669 payload.as_object_mut().unwrap().remove("non_retryable");
670 }
671 let event = HistoryEvent {
672 event_type: if timeout_kind.is_some() {
673 "ActivityTimedOut"
674 } else {
675 "ActivityFailed"
676 }
677 .into(),
678 payload,
679 raw: HashMap::new(),
680 };
681 let result =
682 activity_outcome(&event, &task.payload_codec, Some(activity_type.clone()))?;
683 return Ok(Some((wire, result)));
684 }
685 report["outcome"] = json!("completed");
686 attempts.push(report);
687 wire["outcome"] = json!("completed");
688 wire["attempts"] = json!(attempts);
689 if !self
690 .admit_local_command(task, context, request.command_index, &mut wire, &messages)
691 .await?
692 || !self.local_claim_is_active(task).await
693 {
694 return Ok(None);
695 }
696 return Ok(Some((wire, Ok(result.unwrap()))));
697 }
698 unreachable!("validated local retry budget includes at least one attempt")
699 }
700}
701
702fn local_error(error: Error) -> (String, String, bool, Option<&'static str>) {
703 let failure = crate::ActivityError::from_error(&error, "RustActivityError");
704 (
705 failure.message,
706 failure.exception_type,
707 failure.non_retryable,
708 None,
709 )
710}
711
712fn heartbeat_extended(reports: &Arc<Mutex<Heartbeats>>, timeout: Option<u64>) -> Result<bool> {
713 let reports = reports.lock().map_err(|_| Error::WorkflowStatePoisoned)?;
714 Ok(timeout.is_some_and(|seconds| reports.last + Duration::from_secs(seconds) > Instant::now()))
715}
716
717fn local_retry_backoff_seconds(policy: Option<&Value>, number: u64) -> u64 {
722 policy
723 .and_then(|p| p["backoff_seconds"].as_array())
724 .and_then(|list| {
725 let index = usize::try_from(number)
726 .ok()?
727 .min(list.len())
728 .checked_sub(1)?;
729 list.get(index)
730 })
731 .and_then(Value::as_u64)
732 .unwrap_or(0)
733}
734
735#[cfg(test)]
736mod local_retry_backoff_tests {
737 use super::local_retry_backoff_seconds;
738 use serde_json::json;
739
740 #[test]
741 fn repeats_the_final_entry_once_the_list_runs_out() {
742 let policy = json!({"max_attempts": 5, "backoff_seconds": [1, 2]});
743 let delays: Vec<u64> = (1..=4)
744 .map(|number| local_retry_backoff_seconds(Some(&policy), number))
745 .collect();
746 assert_eq!(delays, vec![1, 2, 2, 2]);
747 }
748
749 #[test]
750 fn no_list_or_no_policy_means_no_delay() {
751 let empty = json!({"max_attempts": 3, "backoff_seconds": []});
752 let missing = json!({"max_attempts": 3});
753 assert_eq!(local_retry_backoff_seconds(Some(&empty), 2), 0);
754 assert_eq!(local_retry_backoff_seconds(Some(&missing), 2), 0);
755 assert_eq!(local_retry_backoff_seconds(None, 1), 0);
756 }
757}